+
+
+
+ ![](\"../../../images/rag_image.png\")
\n",
+ "\n",
+ "
+
+
+
+
+
+# -*- coding: UTF-8 -*-
+
+import collections
+import copy
+import functools
+import logging
+
+from urllib.parse import urlparse
+
+from . import __version__
+from .types import IndexType, MetricType, Status
+from .check import check_pass_param, is_legal_host, is_legal_port
+from .pool import ConnectionPool, SingleConnectionPool, SingletonThreadPool
+from .exceptions import ParamError, DeprecatedError
+
+from ..settings import DefaultConfig as config
+
+LOGGER = logging.getLogger(__name__)
+
+
+def deprecated(func):
+ @functools.wraps(func)
+ def inner(*args, **kwargs):
+ error_str = "Function {} has been deprecated".format(func.__name__)
+ LOGGER.error(error_str)
+ raise DeprecatedError(error_str)
+
+ return inner
+
+
+def check_connect(func):
+ @functools.wraps(func)
+ def inner(self, *args, **kwargs):
+ return func(self, *args, **kwargs)
+
+ return inner
+
+
+def _pool_args(**kwargs):
+ pool_kwargs = dict()
+ for k, v in kwargs.items():
+ if k in ("pool_size", "wait_timeout", "handler", "try_connect", "pre_ping", "max_retry"):
+ pool_kwargs[k] = v
+
+ return pool_kwargs
+
+
+def _set_uri(host, port, uri, handler="GRPC"):
+ default_port = config.GRPC_PORT if handler == "GRPC" else config.HTTP_PORT
+ default_uri = config.GRPC_URI if handler == "GRPC" else config.HTTP_URI
+
+ if host is not None:
+ _port = port if port is not None else default_port
+ _host = host
+ if handler == "HTTP":
+ _proto = "https" if _port == 443 else "http"
+ else:
+ _proto = "tcp"
+ elif port is None:
+ try:
+ _uri = urlparse(uri) if uri else urlparse(default_uri)
+ _host = _uri.hostname
+ _port = _uri.port
+ _proto = _uri.scheme
+ except (AttributeError, ValueError, TypeError) as e:
+ raise ParamError("uri is illegal: {}".format(e))
+ else:
+ raise ParamError("Param is not complete. Please invoke as follow:\n"
+ "\t(host = ${HOST}, port = ${PORT})\n"
+ "\t(uri = ${URI})\n")
+
+ if not is_legal_host(_host) or not is_legal_port(_port):
+ raise ParamError("host {} or port {} is illegal".format(_host, _port))
+
+ return "{}://{}:{}".format(str(_proto), str(_host), str(_port))
+
+
+[docs]class Milvus:
+ def __init__(self, host=None, port=None, handler="GRPC", pool="SingletonThread", **kwargs):
+ """Constructor method
+ """
+ self._name = kwargs.get('name', None)
+ self._uri = None
+ self._status = None
+ self._connected = False
+ self._handler = handler
+
+ #
+ self._conn = None
+
+ _uri = kwargs.get('uri', None)
+ pool_uri = _set_uri(host, port, _uri, self._handler)
+ pool_kwargs = _pool_args(handler=handler, **kwargs)
+ # self._pool = SingleConnectionPool(pool_uri, **pool_kwargs)
+ if pool == "QueuePool":
+ self._pool = ConnectionPool(pool_uri, **pool_kwargs)
+ elif pool == "SingletonThread":
+ self._pool = SingletonThreadPool(pool_uri, **pool_kwargs)
+ elif pool == "Singleton":
+ self._pool = SingleConnectionPool(pool_uri, **pool_kwargs)
+ else:
+ raise ParamError("Unknown pool value: {}".format(pool))
+
+ # store extra key-words arguments
+ self._kw = kwargs
+ self._hooks = collections.defaultdict()
+
+ def __enter__(self):
+ self._conn = self._pool.fetch()
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self._conn.close()
+ self._conn = None
+
+ def __del__(self):
+ return self.close()
+
+ def _connection(self):
+ return self._pool.fetch()
+
+ @deprecated
+ def set_hook(self, **kwargs):
+ """
+ Deprecated
+ """
+ # TODO: may remove it.
+ if self._stub:
+ self._stub.set_hook(**kwargs)
+ else:
+ self._hooks.update(kwargs)
+
+ @property
+ def name(self):
+ return self._name
+
+ @property
+ def handler(self):
+ return self._handler
+
+ @deprecated
+ def connect(self, host=None, port=None, uri=None, timeout=2):
+ """
+ Deprecated
+ """
+ if self.connected() and self._connected:
+ return Status(message="You have already connected {} !".format(self._uri),
+ code=Status.CONNECT_FAILED)
+
+ if self._stub is None:
+ self._init(host, port, uri, handler=self._handler)
+
+ if self.ping(timeout):
+ self._status = Status(message="Connected")
+ self._connected = True
+ return self._status
+
+ return Status()
+
+ @deprecated
+ def connected(self):
+ """
+ Deprecated
+ """
+ return self._status and self._status.OK()
+
+ @deprecated
+ def disconnect(self):
+ """
+ Deprecated
+ """
+ pass
+
+ def close(self):
+ """
+ Close client instance
+ """
+ self._pool = None
+
+ def client_version(self):
+ """
+ Returns the version of the client.
+
+ :return: Version of the client.
+
+ :rtype: (str)
+ """
+ return __version__
+
+ def server_status(self, timeout=30):
+ """
+ Returns the status of the Milvus server.
+
+ :return:
+ Status: Whether the operation is successful.
+
+ str : Status of the Milvus server.
+
+ :rtype: (Status, str)
+ """
+ return self._cmd("status", timeout)
+
+ def server_version(self, timeout=30):
+ """
+ Returns the version of the Milvus server.
+
+ :return:
+ Status: Whether the operation is successful.
+
+ str : Version of the Milvus server.
+
+ :rtype: (Status, str)
+ """
+
+ return self._cmd("version", timeout)
+
+ @check_connect
+ def _cmd(self, cmd, timeout=30):
+ check_pass_param(cmd=cmd)
+
+ with self._connection() as handler:
+ return handler._cmd(cmd, timeout)
+
+[docs] @check_connect
+ def create_collection(self, param, timeout=30):
+ """
+ Creates a collection.
+
+ :type param: dict
+ :param param: Information needed to create a collection. It contains items:
+
+ * *collection_name* (``str``) -- Collection name.
+ * *dimension* (``int``) -- Dimension of embeddings stored in collection.
+ * *index_file_size* (``int``) -- Segment size. See
+ `Storage Concepts <https://milvus.io/docs/v1.0.0/storage_concept.md>`_.
+ * *metric_type* (``MetricType``) -- Distance Metrics type. Valued form
+ :class:`~milvus.MetricType`. See
+ `Distance Metrics <https://milvus.io/docs/v1.0.0/metric.md>`_.
+
+
+ A demo is as follow:
+
+ .. code-block:: python
+
+ param={'collection_name': 'name',
+ 'dimension': 16,
+ 'index_file_size': 1024 # Optional, default 1024，
+ 'metric_type': MetricType.L2 # Optional, default MetricType.L2
+ }
+
+
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ if not isinstance(param, dict):
+ raise ParamError('Param type incorrect, expect {} but get {} instead'
+ .format(type(dict), type(param)))
+
+ collection_param = copy.deepcopy(param)
+
+ if 'collection_name' not in collection_param:
+ raise ParamError('collection_name is required')
+ collection_name = collection_param["collection_name"]
+ collection_param.pop('collection_name')
+
+ if 'dimension' not in collection_param:
+ raise ParamError('dimension is required')
+ dim = collection_param["dimension"]
+ collection_param.pop("dimension")
+
+ index_file_size = collection_param.get('index_file_size', 1024)
+ collection_param.pop('index_file_size', None)
+
+ metric_type = collection_param.get('metric_type', MetricType.L2)
+ collection_param.pop('metric_type', None)
+
+ check_pass_param(collection_name=collection_name, dimension=dim,
+ index_file_size=index_file_size, metric_type=metric_type)
+
+ with self._connection() as handler:
+ return handler.create_collection(collection_name, dim, index_file_size,
+ metric_type, collection_param, timeout)
+
+[docs] @check_connect
+ def has_collection(self, collection_name, timeout=30):
+ """
+
+ Checks whether a collection exists.
+
+ :param collection_name: Name of the collection to check.
+ :type collection_name: str
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status and the flag indicating if collection exists. Succeed
+ if `Status.OK()` is `True`. If status is not OK, the flag is always `False`.
+ :rtype: Status, bool
+
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.has_collection(collection_name, timeout)
+
+[docs] @check_connect
+ def get_collection_info(self, collection_name, timeout=30):
+ """
+ Returns information of a collection.
+
+ :type collection_name: str
+ :param collection_name: Name of the collection to describe.
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status and collection information. Succeed if `Status.OK()`
+ is `True`. If status is not OK, the returned information is always `None`.
+ :rtype: Status, CollectionSchema
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.describe_collection(collection_name, timeout)
+
+[docs] @check_connect
+ def count_entities(self, collection_name, timeout=30):
+ """
+ Returns the number of vectors in a collection.
+
+ :type collection_name: str
+ :param collection_name: target table name.
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status and row count. Succeed if `Status.OK()` is `True`.
+ If status is not OK, the returned value of is always `None`.
+ :rtype: Status, int
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.count_collection(collection_name, timeout)
+
+[docs] @check_connect
+ def list_collections(self, timeout=30):
+ """
+ Returns collection list.
+
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status and collection name list. Succeed if `Status.OK()` is `True`.
+ If status is not OK, the returned name list is always `[]`.
+ :rtype: Status, list[str]
+ """
+ with self._connection() as handler:
+ return handler.show_collections(timeout)
+
+[docs] @check_connect
+ def get_collection_stats(self, collection_name, timeout=30):
+ """
+
+ Returns collection statistics information.
+
+ :type collection_name: str
+ :param collection_name: target table name.
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status and collection statistics information. Succeed
+ if `Status.OK()` is `True`. If status is not OK, the returned information
+ is always `[]`.
+ :rtype: Status, dict
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.show_collection_info(collection_name, timeout)
+
+[docs] @check_connect
+ def load_collection(self, collection_name, partition_tags=None, timeout=None):
+ """
+ Loads a collection for caching.
+
+ :param collection_name: collection to load
+ :type collection_name: str
+ :param partition_tags: partition tag list. `None` indicates to load whole collection,
+ otherwise to load specified partitions.
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.preload_collection(collection_name, partition_tags, timeout)
+
+ @check_connect
+ def reload_segments(self, collection_name, segment_ids, timeout=None):
+ """
+ Reloads segment DeletedDocs data to cache. This API is not recommended for users.
+
+ :param collection_name: Name of the collection being deleted
+ :type collection_name: str
+ :param segment_ids: Segment IDs.
+ :type segment_ids: list[str]
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.reload_segments(collection_name, segment_ids, timeout)
+
+[docs] @check_connect
+ def drop_collection(self, collection_name, timeout=30):
+ """
+ Deletes a collection by name.
+
+ :param collection_name: Name of the collection being deleted
+ :type collection_name: str
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.drop_collection(collection_name, timeout)
+
+[docs] @check_connect
+ def insert(self, collection_name, records, ids=None, partition_tag=None,
+ params=None, timeout=None, **kwargs):
+ """
+ Insert vectors to a collection.
+
+ :type collection_name: str
+ :param collection_name: Name of the collection to insert vectors to.
+ :param ids: ID list. `None` indicates ID is generated by server system. Note that if the
+ first time when insert() is invoked ids is not passed into this method, each
+ of the rest time when inset() is invoked ids is not permitted to pass,
+ otherwise server will return an error and the insertion process will fail.
+ And vice versa.
+ :type ids: list[int]
+ :param records: List of vectors to insert.
+ :type records: list[list[float]]
+ :param partition_tag: Tag of a partition.
+ :type partition_tag: str or None. If partition_tag is None, vectors will be inserted to the
+ default partition `_default`.
+ :param params: Insert param. Reserved.
+ :type params: dict
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+ :param kwargs:
+ * *_async* (``bool``) --
+ Indicate if invoke asynchronously. When value is true, method returns a InsertFuture
+ object; otherwise, method returns results from server.
+ * *_callback* (``function``) --
+ The callback function which is invoked after server response successfully. It only
+ takes effect when _async is set to True.
+
+ :return: The operation status and IDs of inserted entities. Succeed if `Status.OK()`
+ is `True`. If status is not OK, the returned IDs is always `[]`.
+ :rtype: Status, list[int]
+ """
+ if kwargs.get("insert_param", None) is not None:
+ with self._connection() as handler:
+ return handler.insert(None, None, timeout=timeout, **kwargs)
+
+ check_pass_param(collection_name=collection_name, records=records)
+ _ = partition_tag is not None and check_pass_param(partition_tag=partition_tag)
+ if ids is not None:
+ check_pass_param(ids=ids)
+ if len(records) != len(ids):
+ raise ParamError("length of vectors do not match that of ids")
+
+ params = params or dict()
+ if not isinstance(params, dict):
+ raise ParamError("Params must be a dictionary type")
+ with self._connection() as handler:
+ return handler.insert(collection_name, records, ids, partition_tag,
+ params, timeout, **kwargs)
+
+[docs] def get_entity_by_id(self, collection_name, ids, timeout=None):
+ """
+ Returns raw vectors according to ids.
+
+ :param collection_name: Name of the collection
+ :type collection_name: str
+
+ :param ids: list of vector id
+ :type ids: list
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status and entities. Succeed if `Status.OK()` is `True`.
+ If status is not OK, the returned entities is always `[]`.
+ :rtype: Status, list[list[float]]
+ """
+ check_pass_param(collection_name=collection_name, ids=ids)
+
+ with self._connection() as handler:
+ return handler.get_vectors_by_ids(collection_name, ids, timeout=timeout)
+
+[docs] @check_connect
+ def list_id_in_segment(self, collection_name, segment_name, timeout=None):
+ """
+ Get IDs of entity stored in the specified segment.
+
+ :param collection_name: Collection the segment belongs to.
+ :type collection_name: str
+ :param segment_name: Segment name.
+ :type segment_name: str
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status and entity IDs. Succeed if `Status.OK()` is `True`.
+ If status is not OK, the returned IDs is always `[]`.
+ :rtype: Status, list[int]
+ """
+ check_pass_param(collection_name=collection_name)
+ check_pass_param(collection_name=segment_name)
+ with self._connection() as handler:
+ return handler.get_vector_ids(collection_name, segment_name, timeout)
+
+[docs] @check_connect
+ def create_index(self, collection_name, index_type=None, params=None, timeout=None, **kwargs):
+ """
+ Creates index for a collection.
+
+ :param collection_name: Collection used to create index.
+ :type collection_name: str
+ :param index_type: index params. See `index params <param.html>`_ for supported indexes.
+ :type index_type: IndexType
+ :param params: Index param. See `index params <param.html>`_ for detailed index param of
+ supported indexes.
+ :type params: dict
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+ :param kwargs:
+ * *_async* (``bool``) --
+ Indicate if invoke asynchronously. When value is true, method returns a IndexFuture
+ object; otherwise, method returns results from server.
+ * *_callback* (``function``) --
+ The callback function which is invoked after server response successfully. It only
+ takes effect when _async is set to True.
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ _index_type = IndexType.FLAT if index_type is None else index_type
+ check_pass_param(collection_name=collection_name, index_type=_index_type)
+
+ params = params or dict()
+ if not isinstance(params, dict):
+ raise ParamError("Params must be a dictionary type")
+ with self._connection() as handler:
+ return handler.create_index(collection_name, _index_type, params, timeout, **kwargs)
+
+[docs] @check_connect
+ def get_index_info(self, collection_name, timeout=30):
+ """
+ Show index information of a collection.
+
+ :type collection_name: str
+ :param collection_name: table name been queried
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status and index info. Succeed if `Status.OK()` is `True`.
+ If status is not OK, the returned index info is always `None`.
+ :rtype: Status, IndexParam
+
+ """
+ check_pass_param(collection_name=collection_name)
+
+ with self._connection() as handler:
+ return handler.describe_index(collection_name, timeout)
+
+[docs] @check_connect
+ def drop_index(self, collection_name, timeout=30):
+ """
+ Removes an index.
+
+ :param collection_name: target collection name.
+ :type collection_name: str
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ check_pass_param(collection_name=collection_name)
+
+ with self._connection() as handler:
+ return handler.drop_index(collection_name, timeout)
+
+[docs] @check_connect
+ def create_partition(self, collection_name, partition_tag, timeout=30):
+ """
+ create a partition for a collection.
+
+ :param collection_name: Name of the collection.
+ :type collection_name: str
+ :param partition_tag: Name of the partition tag.
+ :type partition_tag: str
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+
+ """
+ check_pass_param(collection_name=collection_name, partition_tag=partition_tag)
+ with self._connection() as handler:
+ return handler.create_partition(collection_name, partition_tag, timeout)
+
+[docs] @check_connect
+ def has_partition(self, collection_name, partition_tag, timeout=30):
+ """
+ Check if specified partition exists.
+
+ :param collection_name: target table name.
+ :type collection_name: str
+ :param partition_tag: partition tag.
+ :type partition_tag: str
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :returns: The operation status and a flag indicating if partition exists. Succeed
+ if `Status.OK()` is `True`. If status is not ok, the flag is always `False`.
+ :rtype: Status, bool
+
+ """
+ check_pass_param(collection_name=collection_name, partition_tag=partition_tag)
+ with self._connection() as handler:
+ return handler.has_partition(collection_name, partition_tag, timeout)
+
+[docs] @check_connect
+ def list_partitions(self, collection_name, timeout=30):
+ """
+ Show all partitions in a collection.
+
+ :param collection_name: target table name.
+ :type collection_name: str
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :returns: The operation status and partition list. Succeed if `Status.OK()` is `True`.
+ If status is not OK, returned partition list is `[]`.
+ :rtype: Status, list[PartitionParam]
+
+ """
+ check_pass_param(collection_name=collection_name)
+
+ with self._connection() as handler:
+ return handler.show_partitions(collection_name, timeout)
+
+[docs] @check_connect
+ def drop_partition(self, collection_name, partition_tag, timeout=30):
+ """
+ Deletes a partition in a collection.
+
+ :param collection_name: Collection name.
+ :type collection_name: str
+ :param partition_tag: Partition name.
+ :type partition_tag: str
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+
+ """
+ check_pass_param(collection_name=collection_name, partition_tag=partition_tag)
+ with self._connection() as handler:
+ return handler.drop_partition(collection_name, partition_tag, timeout)
+
+[docs] @check_connect
+ def search(self, collection_name, top_k, query_records, partition_tags=None,
+ params=None, timeout=None, **kwargs):
+ """
+ Search vectors in a collection.
+
+ :param collection_name: Name of the collection.
+ :type collection_name: str
+ :param top_k: number of vectors which is most similar with query vectors
+ :type top_k: int
+ :param query_records: vectors to query
+ :type query_records: list[list[float32]]
+ :param partition_tags: tags to search. `None` indicates to search in whole collection.
+ :type partition_tags: list
+ :param params: Search params. The params is related to index type the collection is built.
+ See `index params <param.html>`_ for more detailed information.
+ :type params: dict
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+ :param kwargs:
+ * *_async* (``bool``) --
+ Indicate if invoke asynchronously. When value is true, method returns a SearchFuture
+ object; otherwise, method returns results from server.
+ * *_callback* (``function``) --
+ The callback function which is invoked after server response successfully. It only
+ takes effect when _async is set to True.
+
+ :returns: The operation status and search result. See <a>here</a> to find how to handle
+ search result. Succeed if `Status.OK()` is `True`. If status is not OK,
+ results is always `None`.
+ :rtype: Status, TopKQueryResult
+ """
+ check_pass_param(collection_name=collection_name, topk=top_k, records=query_records)
+ if partition_tags is not None:
+ check_pass_param(partition_tag_array=partition_tags)
+
+ params = dict() if params is None else params
+ if not isinstance(params, dict):
+ raise ParamError("Params must be a dictionary type")
+ with self._connection() as handler:
+ return handler.search(collection_name, top_k, query_records,
+ partition_tags, params, timeout, **kwargs)
+
+ @check_connect
+ def search_in_segment(self, collection_name, file_ids, query_records, top_k,
+ params=None, timeout=None, **kwargs):
+ """
+ Searches for vectors in specific segments of a collection.
+ This API is not recommended for users.
+
+ The Milvus server stores vector data into multiple files. Searching for vectors in specific
+ files is a method used in Mishards. Obtain more detail about Mishards, see
+ `Mishards <https://github.com/milvus-io/milvus/tree/master/shards>`_.
+
+ :param collection_name: table name been queried
+ :type collection_name: str
+ :param file_ids: Specified files id array
+ :type file_ids: list[str] or list[int]
+ :param query_records: all vectors going to be queried
+ :type query_records: list[list[float]]
+ :param top_k: how many similar vectors will be searched
+ :type top_k: int
+ :param params: Search params. The params is related to index type the collection is built.
+ See <a></a> for more detailed information.
+ :type params: dict
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+ :param kwargs:
+ * *_async* (``bool``) --
+ Indicate if invoke asynchronously. When value is true, method returns a SearchFuture
+ object; otherwise, method returns results from server.
+ * *_callback* (``function``) --
+ The callback function which is invoked after server response successfully. It only
+ takes effect when _async is set to True.
+
+ :returns: The operation status and search result. See <a>here</a> to find how to handle
+ search result. Succeed if `Status.OK()` is `True`. If status is not OK, results
+ is always `None`.
+ :rtype: Status, TopKQueryResult
+ """
+ check_pass_param(collection_name=collection_name, topk=top_k,
+ records=query_records, ids=file_ids)
+
+ params = dict() if params is None else params
+ if not isinstance(params, dict):
+ raise ParamError("Params must be a dictionary type")
+ with self._connection() as handler:
+ return handler.search_in_files(collection_name, file_ids,
+ query_records, top_k, params, timeout, **kwargs)
+
+[docs] @check_connect
+ def delete_entity_by_id(self, collection_name, id_array, timeout=None):
+ """
+ Deletes vectors in a collection by vector ID.
+
+ :param collection_name: Name of the collection.
+ :type collection_name: str
+ :param id_array: list of vector id
+ :type id_array: list[int]
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :returns: The operation status. If the specified ID doesn't exist, Milvus server skip it
+ and try to delete next entities, which is regard as one successful operation.
+ Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ check_pass_param(collection_name=collection_name, ids=id_array)
+ with self._connection() as handler:
+ return handler.delete_by_id(collection_name, id_array, timeout)
+
+[docs] @check_connect
+ def flush(self, collection_name_array=None, timeout=None, **kwargs):
+ """
+ Flushes vector data in one collection or multiple collections to disk.
+
+ :type collection_name_array: list
+ :param collection_name_array: Name of one or multiple collections to flush.
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+ :param kwargs:
+ * *_async* (``bool``) --
+ Indicate if invoke asynchronously. When value is true, method returns a FlushFuture
+ object; otherwise, method returns results from server.
+ * *_callback* (``function``) --
+ The callback function which is invoked after server response successfully. It only
+ takes effect when _async is set to True.
+
+ :returns: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+
+ if collection_name_array in (None, []):
+ with self._connection() as handler:
+ return handler.flush([], timeout)
+
+ if not isinstance(collection_name_array, list):
+ raise ParamError("Collection name array must be type of list")
+
+ if len(collection_name_array) <= 0:
+ raise ParamError("Collection name array is not allowed to be empty")
+
+ for name in collection_name_array:
+ check_pass_param(collection_name=name)
+ with self._connection() as handler:
+ return handler.flush(collection_name_array, timeout, **kwargs)
+
+[docs] @check_connect
+ def compact(self, collection_name, timeout=None, **kwargs):
+ """
+ Compacts segments in a collection. This function is recommended after deleting vectors.
+
+ :type collection_name: str
+ :param collection_name: Name of the collections to compact.
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+ :param kwargs:
+ * *_async* (``bool``) --
+ Indicate if invoke asynchronously. When value is true, method returns a CompactFuture
+ object; otherwise, method returns results from server.
+ * *_callback* (``function``) --
+ The callback function which is invoked after server response successfully. It only
+ takes effect when _async is set to True.
+
+ :returns: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.compact(collection_name, timeout, **kwargs)
+
+ def get_config(self, parent_key, child_key):
+ """
+ Gets Milvus configurations.
+
+ """
+ cmd = "get_config {}.{}".format(parent_key, child_key)
+
+ return self._cmd(cmd)
+
+ def set_config(self, parent_key, child_key, value):
+ """
+ Sets Milvus configurations.
+
+ """
+ cmd = "set_config {}.{} {}".format(parent_key, child_key, value)
+
+ return self._cmd(cmd)
+
+
+
+
+
+
+from enum import IntEnum
+
+
+class Status:
+ """
+ :attribute code: int (optional) default as ok
+
+ :attribute message: str (optional) current status message
+ """
+
+ SUCCESS = 0
+ UNEXPECTED_ERROR = 1
+ CONNECT_FAILED = 2
+ PERMISSION_DENIED = 3
+ COLLECTION_NOT_EXISTS = 4
+ ILLEGAL_ARGUMENT = 5
+ ILLEGAL_RANGE = 6
+ ILLEGAL_DIMENSION = 7
+ ILLEGAL_INDEX_TYPE = 8
+ ILLEGAL_COLLECTION_NAME = 9
+ ILLEGAL_TOPK = 10
+ ILLEGAL_ROWRECORD = 11
+ ILLEGAL_VECTOR_ID = 12
+ ILLEGAL_SEARCH_RESULT = 13
+ FILE_NOT_FOUND = 14
+ META_FAILED = 15
+ CACHE_FAILED = 16
+ CANNOT_CREATE_FOLDER = 17
+ CANNOT_CREATE_FILE = 18
+ CANNOT_DELETE_FOLDER = 19
+ CANNOT_DELETE_FILE = 20
+ BUILD_INDEX_ERROR = 21
+ ILLEGAL_NLIST = 22
+ ILLEGAL_METRIC_TYPE = 23
+ OUT_OF_MEMORY = 24
+
+ def __init__(self, code=SUCCESS, message="Success"):
+ self.code = code
+ self.message = message
+
+ def __repr__(self):
+ attr_list = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.items()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(attr_list))
+
+ def __eq__(self, other):
+ """
+ Make Status comparable with self by code
+ """
+ if isinstance(other, int):
+ return self.code == other
+
+ return isinstance(other, self.__class__) and self.code == other.code
+
+ def __ne__(self, other):
+ return self != other
+
+ def OK(self):
+ return self.code == Status.SUCCESS
+
+
+[docs]class IndexType(IntEnum):
+ """Index type enum.
+ """
+
+ #: Invalid index type.
+ INVALID = 0
+
+ #: FLAT index. See `FLAT <https://milvus.io/docs/v1.0.0/index.md#FLAT>`_.
+ FLAT = 1
+
+ #: IVF(Inverted File) FLAT index.
+ #: See `IVF_FLAT <https://milvus.io/docs/v1.0.0/index.md#IVF_FLAT>`_.
+ IVF_FLAT = 2
+
+ #: IVF SQ8 index. See `IVF_SQ8 <https://milvus.io/docs/v1.0.0/index.md#IVF_SQ8>`_.
+ IVF_SQ8 = 3
+
+ #: RNSG(Refined NSG) index. See `RNSG <https://milvus.io/docs/v1.0.0/index.md#RNSG>`_.
+ RNSG = 4
+
+ #: IVF SQ8 Hybrid index. See `IVF_SQ8H <https://milvus.io/docs/v1.0.0/index.md#IVF_SQ8H>`_.
+ IVF_SQ8H = 5
+
+ #: IVF PQ index. See `IVF_PQ <https://milvus.io/docs/v1.0.0/index.md#IVF_PQ>`_.
+ IVF_PQ = 6
+
+ #: HNSW index. See `HNSW <https://milvus.io/docs/v1.0.0/index.md#HNSW>`_.
+ HNSW = 11
+
+ #: ANNOY index. See `ANNOY <https://milvus.io/docs/v1.0.0/index.md#ANNOY>`_.
+ ANNOY = 12
+
+ #: Alternative name for `IVF_FLAT`. Reserved for compatibility.
+ IVFLAT = IVF_FLAT
+ #: Alternative name for `IVF_SQ8H`. Reserved for compatibility.
+ IVF_SQ8_H = IVF_SQ8H
+
+ def __repr__(self):
+ return "<{}: {}>".format(self.__class__.__name__, self._name_)
+
+ def __str__(self):
+ return self._name_
+
+
+[docs]class MetricType(IntEnum):
+ """Metric type enum.
+ """
+
+ #: Invalid metric type.
+ INVALID = 0
+
+ #: Euclidean distance. A metric for float vectors.
+ #: See `Euclidean distance <https://milvus.io/docs/v1.0.0/metric.md#Euclidean-distance-L2>`_.
+ L2 = 1
+
+ #: Inner product. A metric for float vectors.
+ #: See `Inner Product <https://milvus.io/docs/v1.0.0/metric.md#Inner-product-IP>`_.
+ IP = 2
+
+ #: Hamming distance. A metric for binary vectors.
+ #: See `Hamming distance <https://milvus.io/docs/v1.0.0/metric.md#Hamming-distance>`_.
+ HAMMING = 3
+
+ #: Jaccard distance. A metric for binary vectors.
+ #: See `Jaccard distance <https://milvus.io/docs/v1.0.0/metric.md#Jaccard-distance>`_.
+ JACCARD = 4
+
+ #: Tanimoto distance. A metric for binary vectors.
+ #: See `Tanimoto distance <https://milvus.io/docs/v1.0.0/metric.md#Tanimoto-distance>`_.
+ TANIMOTO = 5
+
+ #: Superstructure. A metric for binary vectors,
+ #: only support :attr:`~milvus.IndexType.FLAT` index.
+ #: See `Superstructure <https://milvus.io/docs/v1.0.0/metric.md#Superstructure>`_.
+ SUBSTRUCTURE = 6
+
+ #: Substructure. A metric for binary vectors, only support :attr:`~milvus.IndexType.FLAT` index.
+ #: See `Substructure <https://milvus.io/docs/v1.0.0/metric.md#Substructure>`_.
+ SUPERSTRUCTURE = 7
+
+ def __repr__(self):
+ return "<{}: {}>".format(self.__class__.__name__, self._name_)
+
+ def __str__(self):
+ return self._name_
+
+
+
+
+
+ This documentation is generated using the Sphinx documentation generator. The source files for the documentation are +located in the doc/ directory of the pymilvus distribution. To generate the docs locally run the following command +under directory doc/:
+$ make html
+The documentation should be generated under directory build/html.
+To preview it, you can open index.html in your browser.
+Or run a web server in that directory:
+$ python3 -m http.server
+Then open your browser to http://localhost:8000.
+Section author: Bosszou@milvus
+
+
+
+
+
+ API |
+Description |
+
|---|---|
| + | Creates a collection. |
+
| + | Checks if a collection exists. |
+
| + | Gets information of a specified collection. |
+
| + | Gets the number of entities in a collection. |
+
| + | Gets a list of collections. |
+
| + | Gets collection statistics. |
+
| + | Loads a specified collection from storage to memory. |
+
| + | Removes a collection. |
+
| + | Inserts entities to a specified collection. |
+
| + | Gets entities by entity ID. |
+
| + | Gets the list of ids in a specified segment. |
+
| + | Creates an index for a specified field. |
+
| + | Get index information. |
+
| + | Removes index from a specified field. |
+
| + | Creates a partition in a specified collection. |
+
| + | Checks if a partition exists in a specified collection. |
+
| + | Gets a partition list from a specified collection. |
+
| + | Removes a specified partition from a collection. |
+
| + | Searches for approximate nearest entities. |
+
| + | Deletes entities by entity ID. |
+
| + | Flushes collection data from memory to storage. |
+
| + | Compacts a specified collection. |
+
milvus.Milvus(host=None, port=None, handler='GRPC', pool='SingletonThread', **kwargs)[source]¶create_collection(param, timeout=30)[source]¶Creates a collection.
+param (dict) –
Information needed to create a collection. It contains items:
+collection_name (str) – Collection name.
dimension (int) – Dimension of embeddings stored in collection.
index_file_size (int) – Segment size. See
+Storage Concepts.
metric_type (MetricType) – Distance Metrics type. Valued form
+MetricType. See
+Distance Metrics.
A demo is as follow:
+param={'collection_name': 'name',
+ 'dimension': 16,
+ 'index_file_size': 1024 # Optional, default 1024，
+ 'metric_type': MetricType.L2 # Optional, default MetricType.L2
+ }
+timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status. Succeed if Status.OK() is True.
+Status
+has_collection(collection_name, timeout=30)[source]¶Checks whether a collection exists.
+collection_name (str) – Name of the collection to check.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status and the flag indicating if collection exists. Succeed +if Status.OK() is True. If status is not OK, the flag is always False.
+Status, bool
+get_collection_info(collection_name, timeout=30)[source]¶Returns information of a collection.
+collection_name (str) – Name of the collection to describe.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status and collection information. Succeed if Status.OK() +is True. If status is not OK, the returned information is always None.
+Status, CollectionSchema
+count_entities(collection_name, timeout=30)[source]¶Returns the number of vectors in a collection.
+collection_name (str) – target table name.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status and row count. Succeed if Status.OK() is True. +If status is not OK, the returned value of is always None.
+Status, int
+list_collections(timeout=30)[source]¶Returns collection list.
+timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
+The operation status and collection name list. Succeed if Status.OK() is True. +If status is not OK, the returned name list is always [].
+Status, list[str]
+get_collection_stats(collection_name, timeout=30)[source]¶Returns collection statistics information.
+collection_name (str) – target table name.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status and collection statistics information. Succeed +if Status.OK() is True. If status is not OK, the returned information +is always [].
+Status, dict
+load_collection(collection_name, partition_tags=None, timeout=None)[source]¶Loads a collection for caching.
+collection_name (str) – collection to load
partition_tags – partition tag list. None indicates to load whole collection, +otherwise to load specified partitions.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status. Succeed if Status.OK() is True.
+Status
+drop_collection(collection_name, timeout=30)[source]¶Deletes a collection by name.
+collection_name (str) – Name of the collection being deleted
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status. Succeed if Status.OK() is True.
+Status
+insert(collection_name, records, ids=None, partition_tag=None, params=None, timeout=None, **kwargs)[source]¶Insert vectors to a collection.
+collection_name (str) – Name of the collection to insert vectors to.
ids (list[int]) – ID list. None indicates ID is generated by server system. Note that if the +first time when insert() is invoked ids is not passed into this method, each +of the rest time when inset() is invoked ids is not permitted to pass, +otherwise server will return an error and the insertion process will fail. +And vice versa.
records (list[list[float]]) – List of vectors to insert.
partition_tag (str or None. If partition_tag is None, vectors will be inserted to the +default partition _default.) – Tag of a partition.
params (dict) – Insert param. Reserved.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
kwargs –
_async (bool) –
+Indicate if invoke asynchronously. When value is true, method returns a InsertFuture
+object; otherwise, method returns results from server.
_callback (function) –
+The callback function which is invoked after server response successfully. It only
+takes effect when _async is set to True.
The operation status and IDs of inserted entities. Succeed if Status.OK() +is True. If status is not OK, the returned IDs is always [].
+Status, list[int]
+get_entity_by_id(collection_name, ids, timeout=None)[source]¶Returns raw vectors according to ids.
+collection_name (str) – Name of the collection
ids (list) – list of vector id
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status and entities. Succeed if Status.OK() is True. +If status is not OK, the returned entities is always [].
+Status, list[list[float]]
+list_id_in_segment(collection_name, segment_name, timeout=None)[source]¶Get IDs of entity stored in the specified segment.
+collection_name (str) – Collection the segment belongs to.
segment_name (str) – Segment name.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status and entity IDs. Succeed if Status.OK() is True. +If status is not OK, the returned IDs is always [].
+Status, list[int]
+create_index(collection_name, index_type=None, params=None, timeout=None, **kwargs)[source]¶Creates index for a collection.
+collection_name (str) – Collection used to create index.
index_type (IndexType) – index params. See index params for supported indexes.
params (dict) –
Index param. See index params for detailed index param of +supported indexes.
+timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
kwargs –
_async (bool) –
+Indicate if invoke asynchronously. When value is true, method returns a IndexFuture
+object; otherwise, method returns results from server.
_callback (function) –
+The callback function which is invoked after server response successfully. It only
+takes effect when _async is set to True.
The operation status. Succeed if Status.OK() is True.
+Status
+get_index_info(collection_name, timeout=30)[source]¶Show index information of a collection.
+collection_name (str) – table name been queried
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status and index info. Succeed if Status.OK() is True. +If status is not OK, the returned index info is always None.
+Status, IndexParam
+drop_index(collection_name, timeout=30)[source]¶Removes an index.
+collection_name (str) – target collection name.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status. Succeed if Status.OK() is True.
+Status
+create_partition(collection_name, partition_tag, timeout=30)[source]¶create a partition for a collection.
+collection_name (str) – Name of the collection.
partition_tag (str) – Name of the partition tag.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status. Succeed if Status.OK() is True.
+Status
+has_partition(collection_name, partition_tag, timeout=30)[source]¶Check if specified partition exists.
+collection_name (str) – target table name.
partition_tag (str) – partition tag.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status and a flag indicating if partition exists. Succeed +if Status.OK() is True. If status is not ok, the flag is always False.
+Status, bool
+list_partitions(collection_name, timeout=30)[source]¶Show all partitions in a collection.
+collection_name (str) – target table name.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status and partition list. Succeed if Status.OK() is True. +If status is not OK, returned partition list is [].
+Status, list[PartitionParam]
+drop_partition(collection_name, partition_tag, timeout=30)[source]¶Deletes a partition in a collection.
+collection_name (str) – Collection name.
partition_tag (str) – Partition name.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status. Succeed if Status.OK() is True.
+Status
+search(collection_name, top_k, query_records, partition_tags=None, params=None, timeout=None, **kwargs)[source]¶Search vectors in a collection.
+collection_name (str) – Name of the collection.
top_k (int) – number of vectors which is most similar with query vectors
query_records (list[list[float32]]) – vectors to query
partition_tags (list) – tags to search. None indicates to search in whole collection.
params (dict) –
Search params. The params is related to index type the collection is built. +See index params for more detailed information.
+timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
kwargs –
_async (bool) –
+Indicate if invoke asynchronously. When value is true, method returns a SearchFuture
+object; otherwise, method returns results from server.
_callback (function) –
+The callback function which is invoked after server response successfully. It only
+takes effect when _async is set to True.
The operation status and search result. See <a>here</a> to find how to handle +search result. Succeed if Status.OK() is True. If status is not OK, +results is always None.
+Status, TopKQueryResult
+delete_entity_by_id(collection_name, id_array, timeout=None)[source]¶Deletes vectors in a collection by vector ID.
+collection_name (str) – Name of the collection.
id_array (list[int]) – list of vector id
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status. If the specified ID doesn’t exist, Milvus server skip it +and try to delete next entities, which is regard as one successful operation. +Succeed if Status.OK() is True.
+Status
+flush(collection_name_array=None, timeout=None, **kwargs)[source]¶Flushes vector data in one collection or multiple collections to disk.
+collection_name_array (list) – Name of one or multiple collections to flush.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
kwargs –
_async (bool) –
+Indicate if invoke asynchronously. When value is true, method returns a FlushFuture
+object; otherwise, method returns results from server.
_callback (function) –
+The callback function which is invoked after server response successfully. It only
+takes effect when _async is set to True.
The operation status. Succeed if Status.OK() is True.
+Status
+compact(collection_name, timeout=None, **kwargs)[source]¶Compacts segments in a collection. This function is recommended after deleting vectors.
+collection_name (str) – Name of the collections to compact.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
kwargs –
_async (bool) –
+Indicate if invoke asynchronously. When value is true, method returns a CompactFuture
+object; otherwise, method returns results from server.
_callback (function) –
+The callback function which is invoked after server response successfully. It only
+takes effect when _async is set to True.
The operation status. Succeed if Status.OK() is True.
+Status
+milvus.MetricType[source]¶Metric type enum.
+INVALID = 0¶Invalid metric type.
+L2 = 1¶Euclidean distance. A metric for float vectors. +See Euclidean distance.
+IP = 2¶Inner product. A metric for float vectors. +See Inner Product.
+HAMMING = 3¶Hamming distance. A metric for binary vectors. +See Hamming distance.
+JACCARD = 4¶Jaccard distance. A metric for binary vectors. +See Jaccard distance.
+TANIMOTO = 5¶Tanimoto distance. A metric for binary vectors. +See Tanimoto distance.
+SUBSTRUCTURE = 6¶Superstructure. A metric for binary vectors,
+only support FLAT index.
+See Superstructure.
SUPERSTRUCTURE = 7¶Substructure. A metric for binary vectors, only support FLAT index.
+See Substructure.
+
+
+
+
+ v1.0.2(Developing)
+v1.0.1
+Remove unused hybrid APIs
Section author: Bosszou@milvus
+
+
+
+
+
+ Contributing is warmly welcomed. You can contribute to PyMilvus project by opening issues and submitting pull +requests on PyMilvus Github page.
+To request a new feature, report a bug or ask a question, it’s recommended for you to open an issue.
+You can tell us why you need it and we will decide whether to implement it soon. +If we think it’s a good improvement, we will make it a feature request and start to work on it. It’s +also welcomed for you to open an issue with your PR as a solution.
+You need to tell us as much information as possible, better start with our +bug report template. +With information, we can reproduce the bug easily and solve it later.
+It’s welcomed to ask any questions about PyMilvus and Milvus, we are pleased to communicate with you.
+If you have improvements to PyMilvus, please submit pull requests(PR) to master, see workflow below.
+PR for codes, you need to tell us why we need it, mentioning an existing issue would be better.
+PR for docs, you also need to tell us why we need it.
+Your PRs will be reviewed and checked, merged into our project if approved.
+This is a brief instruction of Github workflow for beginners.
+Fork the PyMilvus repository on Github.
Clone your fork to your local machine with git clone git@github.com:<your_user_name>/pymilvus.git.
Create a new branch with git checkout -b my_working_branch.
Make your changes, commit, then push to your forked repository.
Visit Github and make you PR.
If you already have an existing local repository, always update it before you start to make changes like below:
+$ git remote add upstream git@github.com:milvus-io/pymilvus.git
+$ git checkout master
+$ git pull upstream master
+$ git checkout -b my_working_branch
+1. Update CHANGELOG.md
+If any improvement or feature being added, you are recommended to open a new issue(if not exist) then +record your change in file CHANGELOG.md. The format is: +- #{GitHub issue number} - {Brief description for your change}
+2. Add unit tests for your codes
+To run unit test in github action, you need make sure the last commit message of PR starts with “[ci]”. +If you want to run unit test locally, under root folder of Pymilvus project run pytest –ip=${IP} –port=${PORT}.
+3. Pass pylint check
+In the root directory, run pylint --rcfile=pylint.conf milvus/client to make sure the rate is 10.
4. For documentations
+You need to enter the doc directory and run make html, please refer to
+About this documentations.
Section author: Yangxuan@milvus
+
+
+
+
+
+ Make sure to set the environment variable GRPC_ENABLE_FORK_SUPPORT=1.
+For reference, see this post.
Try installing PyMilvus in a Conda environment.
+Section author: Yangxuan@milvus
+
+
+
+
+
+ | + |
| + | + |
| + | + |
| + | + |
| + | + |
| + | + |
| + | + |
| + |
| + | + |
| + |
|
+
| + |
| + | + |
| + |
+
+
+
+
+ PyMilvus is a python SDK for Milvus and is a recommended way to work with Milvus. This documentation covers +every thing you need to know about PyMilvus.
+Instructions on how to install PyMilvus.
+A quick start to use PyMilvus.
+The complete API documentation.
+Index and relevant parameters.
+How to deal with search results.
+Changes in the latest PyMilvus.
+Method of contribution, bug shooting and contribution guide.
+Some questions that come up often.
+How this documentation is generated.
+Section author: Bosszou@milvus, +Godchen@milvus, +Yangxuan@milvus
+
+
+
+
+
+ PyMilvus is in the Python Package Index.
+PyMilvus only support python3(>= 3.6), usually, it’s ok to install PyMilvus like below.
+$ python3 -m pip install pymilvus
+It’s recommended to use PyMilvus in a virtual environment, using virtual environment allows you to avoid
+installing Python packages globally which could break system tools or other projects.
+We use virtualenv as an example to demonstrate how to install and using PyMilvus in a virtual environment.
+See virtualenv for more information about why and how.
$ python3 -m pip install virtualenv
+$ virtualenv venv
+$ source venv/bin/activate
+(venv) $ pip install pymilvus
+If you want to exit the virtualenv venv, you can use deactivate.
(venv) $ deactivate
+$
+Here we assume you are already in a virtual environment.
+Suitable PyMilvus version depends on Milvus version you are using. See install pymilvus for recommended pymilvus version.
+If you want to install a specific version of PyMilvus:
+(venv) $ pip install pymilvus==1.0.1
+If you want to upgrade PyMilvus into the latest version published:
+(venv) $ pip install --upgrade pymilvus
+This will install the latest PyMilvus into your virtual environment.
+(venv) $ pip install git+https://github.com/milvus-io/pymilvus.git
+Your installation is correct if the following command in the Python shell doesn’t raise an exception.
+(venv) $ python -c "from milvus import Milvus, DataType"
+Section author: Yangxuan@milvus
+
+
+
+
+
+ Milvus support to create index to accelerate vector approximate search.
+To learn how to create an index by python client, see method create_index() and +index example .
+For more detailed information about indexes, please refer to Milvus documentation index chapter.
+To learn how to choose an appropriate index for your application scenarios, please read How to Select an Index in Milvus.
+To learn how to choose an appropriate index for a metric, see Distance Metrics.
+If FLAT index is used, the vectors are stored in an array of float/binary data without any compression. during +searching vectors, all indexed vectors are decoded sequentially and compared to the query vectors.
+FLAT index provides 100% query recall rate. Compared to other indexes, it is the most efficient indexing method +when the number of queries is small.
+The inserted and index-inbuilt vectors and index-dropped vectors are regard as built with FLAT.
building parameters: +N/A
# FLAT
+client.create_index(collection_name, IndexType.FLAT)
+search parameters: +N/A
# FLAT
+client.search(collection_name,
+ 1,
+ query_vectors
+)
+IVF (Inverted File) is an index type based on quantization. It divides the points in space into nlist
+units by clustering method. During searching vectors, it compares the distances between the target vector
+and the center of all the units, and then select the nprobe nearest unit. Then, it compares all the vectors
+in these selected cells to get the final result.
IVF_FLAT is the most basic IVF index, and the encoded data stored in each unit is consistent with the original data.
+building parameters:
+nlist: Number of cluster units.
+# IVF_FLAT
+client.create_index(collection_name, IndexType.IVF_FLAT, {
+ "nlist": 100 # int. 1~65536
+})
+search parameters:
+nprobe: Number of inverted file cell to probe.
+# IVF_FLAT
+client.search(collection_name,
+ 1,
+ query_vectors,
+ params={
+ "nprobe": 8 # int. 1~nlist(cpu), 1~min[2048, nlist](gpu)
+ }
+)
+PQ (Product Quantization) uniformly decomposes the original high-dimensional vector space into
+Cartesian products of m low-dimensional vector spaces, and then quantizes the decomposed low-dimensional
+vector spaces. In the end, each vector is stored in m × nbits bits. Instead of calculating the distances
+between the target vector and the center of all the units, product quantization enables the calculation of
+distances between the target vector, and the clustering center of each low-dimensional space and greatly reduces
+the time complexity and space complexity of the algorithm.
IVF_PQ performs IVF index clustering, and then quantizes the product of vectors. Its index file is even +smaller than IVF_SQ8, but it also causes a loss of accuracy during searching.
+building parameters:
+nlist: Number of cluster units.
+m: Number of factors of product quantization. CPU-only Milvus: m ≡ dim (mod m); GPU-enabled Milvus: m ∈ {1, 2, 3, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 64, 96}, and (dim / m) ∈ {1, 2, 3, 4, 6, 8, 10, 12, 16, 20, 24, 28, 32}. (m x 1024) ≥ MaxSharedMemPerBlock of your graphics card.
nbits: Number of bits in which each low-dimensional vector is stored.
+# IVF_PQ
+client.create_index(collection_name,
+ IndexType.IVF_PQ,
+ {
+ "nlist": 100, # int. 1~65536
+ "m": 8 # int. 1~16. 8 by default
+ }
+)
+search parameters:
+nprobe: Number of inverted file cell to probe.
+# IVF_PQ
+client.search(collection_name,
+ 1,
+ query_vectors,
+ params={
+ "nprobe": 8 # int. 1~nlist(cpu), 1~min[2048, nlist](gpu)
+ }
+)
+IVF_SQ8 does scalar quantization for each vector placed in the unit based on IVF. Scalar quantization +converts each dimension of the original vector from a 4-byte floating-point number to a 1-byte unsigned integer, +so the IVF_SQ8 index file occupies much less space than the IVF_FLAT index file. +However, scalar quantization results in a loss of accuracy during searching vectors.
+building parameters:
+nlist: Number of cluster units.
+# IVF_SQ8
+client.create_index(collection_name,
+ IndexType.IVF_SQ8,
+ {
+ "nlist": 100 # int. 1~65536
+ }
+)
+search parameters:
+nprobe: Number of inverted file cell to probe.
+# IVF_SQ8
+client.search(collection_name,
+ 1,
+ query_vectors,
+ params={
+ "nprobe": 8 # int. 1~nlist(cpu), 1~min[2048, nlist](gpu)
+ }
+)
+Optimized version of IVF_SQ8 that requires both CPU and GPU to work. Unlike IVF_SQ8, IVF_SQ8_H uses a GPU-based +coarse quantizer, which greatly reduces time to quantize.
+IVF_SQ8H is an IVF_SQ8 index that optimizes query execution.
+The query method is as follows:
+If nq ≥ gpu_search_threshold, GPU handles the entire query task.
If nq < gpu_search_threshold, GPU handles the task of retrieving the nprobe nearest unit in the IVF
+index file, and CPU handles the rest.
building parameters:
+nlist: Number of cluster units.
+# IVF_SQ8_H
+client.create_index(collection_name,
+ IndexType.IVF_SQ8_H,
+ {
+ "nlist": 100 # int. 1~65536
+ }
+)
+search parameters:
+nprobe: Number of inverted file cell to probe.
+# IVF_SQ8_H
+client.search(collection_name,
+ 1,
+ query_vectors,
+ params={
+ "nprobe": 8 # int. 1~nlist(cpu), 1~min[2048, nlist](gpu)
+ }
+)
+ANNOY (Approximate Nearest Neighbors Oh Yeah) is an index that uses a hyperplane to divide a +high-dimensional space into multiple subspaces, and then stores them in a tree structure.
+When searching for vectors, ANNOY follows the tree structure to find subspaces closer to the target vector,
+and then compares all the vectors in these subspaces (The number of vectors being compared should not be
+less than search_k) to obtain the final result. Obviously, when the target vector is close to the edge of
+a certain subspace, sometimes it is necessary to greatly increase the number of searched subspaces to obtain
+a high recall rate. Therefore, ANNOY uses n_trees different methods to divide the whole space, and searches
+all the dividing methods simultaneously to reduce the probability that the target vector is always at the edge of the subspace.
building parameters:
+n_trees: The number of methods of space division.
+# ANNOY
+client.create_index(collection_name,
+ IndexType.ANNOY,
+ {
+ "n_trees": 8 # int. 1~1024
+ }
+)
+search parameters:
+search_k: The number of nodes to search. -1 means 5% of the whole data.
+# ANNOY
+client.search(collection_name,
+ 1,
+ query_vectors,
+ params={
+ "search_k": -1 # int. {-1} U [top_k, n*n_trees], n represents vectors count.
+ }
+)
+HNSW (Hierarchical Navigable Small World Graph) is a graph-based indexing algorithm. It builds a +multi-layer navigation structure for an image according to certain rules. In this structure, the upper +layers are more sparse and the distances between nodes are farther; the lower layers are denser and +he distances between nodes are closer. The search starts from the uppermost layer, finds the node closest +to the target in this layer, and then enters the next layer to begin another search. After multiple iterations, +it can quickly approach the target position.
+In order to improve performance, HNSW limits the maximum degree of nodes on each layer of the graph to M.
+In addition, you can use efConstruction (when building index) or ef (when searching targets) to specify a search range.
building parameters:
+M: Maximum degree of the node.
+efConstruction: Take the effect in stage of index construction.
+# HNSW
+client.create_index(collection_name,
+ IndexType.HNSW,
+ {
+ "M": 16, # int. 4~64
+ "efConstruction": 40 # int. 8~512
+ }
+)
+search parameters:
+ef: Take the effect in stage of search scope, should be larger than top_k.
# HNSW
+client.search(collection_name,
+ 1,
+ query_vectors,
+ params={
+ "ef": 64 # int. top_k~32768
+ }
+)
+RNSG (Refined Navigating Spreading-out Graph) is a graph-based indexing algorithm. It sets the center
+position of the whole image as a navigation point, and then uses a specific edge selection strategy to control
+the out-degree of each point (less than or equal to out_degree). Therefore, it can reduce memory usage and
+quickly locate the target position nearby during searching vectors.
The graph construction process of NSG is as follows:
+Find knng nearest neighbors for each point.
Iterate at least search_length times based on knng nearest neighbor nodes to select candidate_pool_size possible nearest neighbor nodes.
Construct the out-edge of each point in the selected candidate_pool_size nodes according to the edge selection strategy.
The query process is similar to the graph building process. It starts from the navigation point and iterates at least search_length times to get the final result.
building parameters:
+search_length: Number of query iterations.
+out_degree: Maximum out-degree of the node.
+candidate_pool_size: Candidate pool size of the node.
+knng: Number of nearest neighbors
+# RNSG
+client.create_index(collection_name,
+ IndexType.RNSG,
+ {
+ "search_length": 60, # int. 10~300
+ "out_degree": 30, # int. 5~300
+ "candidate_pool_size": 300, # int. 50~1000
+ "knng": 50 # int. 5~300
+ }
+)
+search parameters:
+++search_length: Number of query iterations
+
# RNSG
+client.search(collection_name,
+ 1,
+ query_vectors,
+ params={
+ "search_length": 100 # int. 10~300
+ }
+)
+Section author: Godchen@milvus
+
+
+
+
+
+ The invocation of search() is like this:
+>>> results = client.search('demo', query_vectors, topk)
+The result object can be used as a 2-D array. results[i] (0 <= i < len(results)) represents topk results of i-th query +vector, and results[i][j] (0 <= j < len( results[i] )) represents j-th result of i-th query vector. To get result id and distance, +you can invoke like this:
+>>> id = results[i][j].id
+>>> distance = results[i][j].distance
+The results object can be iterated, so you can traverse the results with two-level loop:
+>>> for raw_result in results:
+... for result in raw_result:
+... id = result.id # result id
+... distance = result.distance
+Meanwhile, the results object provide attributes to separately access result id array id_array and distance array distance_array, +so you can traverse the results like this:
+>>> for ids, distances in zip(results.id_array, results.distance_array):
+... for id_, dis_ in zip(ids, distances):
+... print(f"id = {id_}, distance = {dis_}")
+Section author: Bosszou@milvus
+
+
+
+
+
+
+
+
+
+
+ This is a basic introduction to Milvus by PyMilvus.
+For a runnable python script, +checkout example.py on PyMilvus Github, +or hello milvus on Milvus official website. It’s a good recommended +start to get started with Milvus and PyMilvus as well.
+Note
+Here we use float vectors as example vector field data, if you want to learn example about binary vectors, see +binary vector example.
+Before we start, there are some prerequisites.
+Make sure that:
+You have a running Milvus instance.
PyMilvus is correctly installed.
First of all, we need to import PyMilvus.
+>>> from milvus import Milvus, DataType, MetricType
+Then, we can make connection with Milvus server. +By default Milvus runs on localhost in port 19530, so you can use default value to connect to Milvus.
+>>> host = '127.0.0.1'
+>>> port = '19530'
+>>> client = Milvus(host, port)
+After connecting, we can communicate with Milvus in the following ways. If you are confused about the +terminology, see Milvus Terminology for explanations.
+Now let’s create a new collection. Before we start, we can list all the collections already exist. For a brand +new Milvus running instance, the result should be empty.
+>>> client.list_collections()
+(Status(code=0, message='Show collections successfully!'), [])
+To create collection, we need to provide collection parameters.
+collection_param consists of 4 components, they are collection_name, dimension, index_file_size
+and metric_type.
The name of collection should be a unique string to collections already exist.
+For a float vector, dimension should be equal to the length of a vector; for a binary vector, dimension should +be equal to bit size of a vector.
+Milvus controls the size of data segment according to the index_file_size, you can refer to +Storage Concepts for more information about segments and index_file_size.
+Milvus compute distance between two vectors, you can refer to Distance Metrics +for more information.
+Now we can create a collection:
+>>> collection_name = 'demo_film_tutorial'
+>>> collection_param = {
+... "collection_name": collection_name,
+... "dimension": 8,
+... "index_file_size": 2048,
+... "metric_type": MetricType.L2
+... }
+>>> client.create_collection(collection_param)
+Status(code=0, message='Create collection successfully!')
+Then you can list collections and ‘demo_film_tutorial’ will be in the result.
+>>> client.list_collections()
+(Status(code=0, message='Show collections successfully!'), ['demo_film_tutorial'])
+You can also get info of the collection.
+>>> status, info = client.get_collection_info(collection_name)
+>>> info
+CollectionSchema(collection_name='demo_film_tutorial', dimension=8, index_file_size=2048, metric_type=<MetricType: L2>)
+The attributes of collection can be extracted from info.
+>>> info.collection_name
+'demo_film_tutorial'
+>>> info.dimension
+8
+>>> info.index_file_size
+2048
+>>> info.metric_type
+<MetricType: L2>
+This tutorial is a basic intro tutorial, building index won’t be covered by this tutorial. +If you want to go further into Milvus with indexes, it’s recommended to check our +index examples.
+If you’re already known about indexes from index examples, and you want a full lists of params supported
+by PyMilvus, you check out Index
+chapter of the PyMilvus documentation.
Further more, if you want to get a thorough view of indexes, check our official website for +Vector Index.
+If you don’t create a partition, there will be a default one called “_default”, all the entities will be
+inserted into the “_default” partition. You can check it by list_partitions()
>>> client.list_partitions(collection_name)
+(Status(code=0, message='Success'), [(collection_name='demo_film_tutorial', tag='_default')])
+You can provide a partition tag to create a new partition.
+>>> client.create_partition(collection_name, "films")
+Status(code=0, message='OK')
+>>> client.list_partitions(collection_name)
+(Status(code=0, message='Success'), [(collection_name='demo_film_tutorial', tag='_default'), (collection_name='demo_film_tutorial', tag='films')])
+An entity is a group of fields that corresponds to real world objects. In current version, Milvus only contains a vector field. +Here is an example of 3 entities structured in list of list.
+>>> import random
+>>> entities = [[random.random() for _ in range(8)] for _ in range(3)]
+>>>> status, ids = client.insert(collection_name, entities)
+If the entities inserted successfully, ids we provided will be returned.
>>> ids
+[1615279498011637000, 1615279498011637001, 1615279498011637002]
+Or you can also provide entity ids
+>>> entity_ids = [0, 1, 2]
+>>> status, ids = client.insert(collection_name, entities, entity_ids)
+Warning
+If the first time when insert() is invoked ids is not passed into this method, each of the rest time +when inset() is invoked ids is not permitted to pass, otherwise server will return an error and the +insertion process will fail. And vice versa.
+After successfully inserting 3 entities into Milvus, we can Flush data from memory to disk so that we can
+retrieve them. Milvus also performs an automatic flush with a fixed interval(configurable, default 1 second),
+see Data Flushing.
You can flush multiple collections at one time, so be aware the parameter is a list.
+>>> client.flush([collection_name])
+Status(code=0, message='OK')
+After insertion, we can get the detail of collection statistics information by get_collection_stats()
Note
+For a better output format, we are using pprint to provide a better format.
>>> from pprint import pprint
+>>> status, stats = client.get_collection_stats(collection_name)
+>>> pprint(stats)
+{'partitions': [{'row_count': 3,
+ 'segments': [{'data_size': 120,
+ 'index_name': 'IDMAP',
+ 'name': '1615279498038473000',
+ 'row_count': 3}],
+ 'tag': '_default'},
+ {'row_count': 0, 'segments': None, 'tag': 'films'}],
+ 'row_count': 3}
+We can also count how many entities are there in the collection.
+>>> client.count_entities(collection_name)
+(Status(code=0, message='Success!'), 3)
+You can get entities by their ids.
+>>> status, films = client.get_entity_by_id(collection_name, [0, 1615279498011637001])
+>>> films
+[[], [0.8309633731842041, 0.7896093726158142, 0.09463301301002502, 0.7827594876289368, 0.5261889100074768, 0.8051634430885315, 0.18777835369110107, 0.28041353821754456]]
+If id exists, an entity will be returned. If id doesn’t exist, [] will be return. For the example above,
+the result films will only have one entity, the other is []. Because vector id are generated by server, so the value of id may differ.
You can get entities by vector similarity. Assuming we have a film_A like below, and we want to get top 2 films
+that are most similar with it.
>>> film_A = [random.random() for _ in range(8)]
+>>> status, results = client.search(collection_name, 2, [film_A])
+Note
+If the collection is index-built, user need to specify search param, and pass parameter params like: client.search(…, params={…}). +You can refer to Index params for more details.
+Note
+If parameter partition_tags is specified, milvus executes search request on these partition instead of whole collection.
+The returned results is a 2-D like structure, 1 for 1 entity querying, 2 for top 2. For more clarity, we obtain
+the film as below. If you want to know how to deal with search result in a better way, you can refer to
+search result in PyMilvus doc.
>>> result = results[0]
+>>> film_1 = result[0]
+>>> film_2 = result[1]
+Then how do we get ids, distances and fields? It’s as below.
+Note
+Because vectors are randomly generated, so the retrieved vector id and distance may differ.
+>>> film_1.id # id
+1615279498011637002
+>>> film_1.distance # distance
+1.0709768533706665
+Finally, let’s move on to deletion in Milvus. +We can delete entities by ids, drop a whole partition, or drop the entire collection.
+You can delete entities by their ids.
+>>> client.delete_entity_by_id(collection_name, [0, 1615279498011637002])
+Status(code=0, message='OK')
+Note
+If one entity corresponding to a specified id doesn’t exist, milvus ignore it and execute next deletion. +In this case, client always return ok status except any exception occurs.
+>>> client.count_entities(collection_name)
+(Status(code=0, message='Success!'), 2)
+You can also drop a partition.
+Danger
+Once you drop a partition, all the data in this partition will be deleted too.
+>>> client.drop_partition(collection_name, "films")
+Status(code=0, message='OK')
+Finally, you can drop an entire collection.
+Danger
+Once you drop a collection, all the data in this collection will be deleted too.
+>>> client.drop_collection(collection_name)
+Status(code=0, message='OK')
+Section author: Yangxuan@milvus
+
+
+
+
+
+
+
+
+
+
+
+# -*- coding: UTF-8 -*-
+
+import collections
+import copy
+import functools
+import logging
+
+from urllib.parse import urlparse
+
+from . import __version__
+from .types import IndexType, MetricType, Status
+from .check import check_pass_param, is_legal_host, is_legal_port
+from .pool import ConnectionPool, SingleConnectionPool, SingletonThreadPool
+from .exceptions import ParamError, DeprecatedError
+
+from ..settings import DefaultConfig as config
+
+LOGGER = logging.getLogger(__name__)
+
+
+def deprecated(func):
+ @functools.wraps(func)
+ def inner(*args, **kwargs):
+ error_str = "Function {} has been deprecated".format(func.__name__)
+ LOGGER.error(error_str)
+ raise DeprecatedError(error_str)
+
+ return inner
+
+
+def check_connect(func):
+ @functools.wraps(func)
+ def inner(self, *args, **kwargs):
+ return func(self, *args, **kwargs)
+
+ return inner
+
+
+def _pool_args(**kwargs):
+ pool_kwargs = dict()
+ for k, v in kwargs.items():
+ if k in ("pool_size", "wait_timeout", "handler", "try_connect", "pre_ping", "max_retry"):
+ pool_kwargs[k] = v
+
+ return pool_kwargs
+
+
+def _set_uri(host, port, uri, handler="GRPC"):
+ default_port = config.GRPC_PORT if handler == "GRPC" else config.HTTP_PORT
+ default_uri = config.GRPC_URI if handler == "GRPC" else config.HTTP_URI
+
+ if host is not None:
+ _port = port if port is not None else default_port
+ _host = host
+ if handler == "HTTP":
+ _proto = "https" if _port == 443 else "http"
+ else:
+ _proto = "tcp"
+ elif port is None:
+ try:
+ _uri = urlparse(uri) if uri else urlparse(default_uri)
+ _host = _uri.hostname
+ _port = _uri.port
+ _proto = _uri.scheme
+ except (AttributeError, ValueError, TypeError) as e:
+ raise ParamError("uri is illegal: {}".format(e))
+ else:
+ raise ParamError("Param is not complete. Please invoke as follow:\n"
+ "\t(host = ${HOST}, port = ${PORT})\n"
+ "\t(uri = ${URI})\n")
+
+ if not is_legal_host(_host) or not is_legal_port(_port):
+ raise ParamError("host {} or port {} is illegal".format(_host, _port))
+
+ return "{}://{}:{}".format(str(_proto), str(_host), str(_port))
+
+
+[docs]class Milvus:
+ def __init__(self, host=None, port=None, handler="GRPC", pool="SingletonThread", **kwargs):
+ """Constructor method
+ """
+ self._name = kwargs.get('name', None)
+ self._uri = None
+ self._status = None
+ self._connected = False
+ self._handler = handler
+
+ #
+ self._conn = None
+
+ _uri = kwargs.get('uri', None)
+ pool_uri = _set_uri(host, port, _uri, self._handler)
+ pool_kwargs = _pool_args(handler=handler, **kwargs)
+
+ if pool == "QueuePool":
+ self._pool = ConnectionPool(pool_uri, **pool_kwargs)
+ elif pool == "SingletonThread":
+ self._pool = SingletonThreadPool(pool_uri, **pool_kwargs)
+ elif pool == "Singleton":
+ self._pool = SingleConnectionPool(pool_uri, **pool_kwargs)
+ else:
+ raise ParamError("Unknown pool value: {}".format(pool))
+
+ # store extra key-words arguments
+ self._kw = kwargs
+ self._hooks = collections.defaultdict()
+
+ def __enter__(self):
+ self._conn = self._pool.fetch()
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self._conn.close()
+ self._conn = None
+
+ def __del__(self):
+ return self.close()
+
+ def _connection(self):
+ return self._pool.fetch()
+
+ @deprecated
+ def set_hook(self, **kwargs):
+ """
+ Deprecated
+ """
+ # TODO: may remove it.
+ if self._stub:
+ self._stub.set_hook(**kwargs)
+ else:
+ self._hooks.update(kwargs)
+
+ @property
+ def name(self):
+ return self._name
+
+ @property
+ def handler(self):
+ return self._handler
+
+ @deprecated
+ def connect(self, host=None, port=None, uri=None, timeout=2):
+ """
+ Deprecated
+ """
+ if self.connected() and self._connected:
+ return Status(message="You have already connected {} !".format(self._uri),
+ code=Status.CONNECT_FAILED)
+
+ if self._stub is None:
+ self._init(host, port, uri, handler=self._handler)
+
+ if self.ping(timeout):
+ self._status = Status(message="Connected")
+ self._connected = True
+ return self._status
+
+ return Status()
+
+ @deprecated
+ def connected(self):
+ """
+ Deprecated
+ """
+ return self._status and self._status.OK()
+
+ @deprecated
+ def disconnect(self):
+ """
+ Deprecated
+ """
+ pass
+
+ def close(self):
+ """
+ Close client instance
+ """
+ self._pool = None
+
+ def client_version(self):
+ """
+ Returns the version of the client.
+
+ :return: Version of the client.
+
+ :rtype: (str)
+ """
+ return __version__
+
+ def server_status(self, timeout=30):
+ """
+ Returns the status of the Milvus server.
+
+ :return:
+ Status: Whether the operation is successful.
+
+ str : Status of the Milvus server.
+
+ :rtype: (Status, str)
+ """
+ return self._cmd("status", timeout)
+
+ def server_version(self, timeout=30):
+ """
+ Returns the version of the Milvus server.
+
+ :return:
+ Status: Whether the operation is successful.
+
+ str : Version of the Milvus server.
+
+ :rtype: (Status, str)
+ """
+
+ return self._cmd("version", timeout)
+
+ @check_connect
+ def _cmd(self, cmd, timeout=30):
+ check_pass_param(cmd=cmd)
+
+ with self._connection() as handler:
+ return handler._cmd(cmd, timeout)
+
+[docs] @check_connect
+ def create_collection(self, param, timeout=30):
+ """
+ Creates a collection.
+
+ :type param: dict
+ :param param: Information needed to create a collection. It contains items:
+
+ * *collection_name* (``str``) -- Collection name.
+ * *dimension* (``int``) -- Dimension of embeddings stored in collection.
+ * *index_file_size* (``int``) -- Segment size. See
+ `Storage Concepts <https://milvus.io/docs/v1.0.0/storage_concept.md>`_.
+ * *metric_type* (``MetricType``) -- Distance Metrics type. Valued form
+ :class:`~milvus.MetricType`. See
+ `Distance Metrics <https://milvus.io/docs/v1.0.0/metric.md>`_.
+
+
+ A demo is as follow:
+
+ .. code-block:: python
+
+ param={'collection_name': 'name',
+ 'dimension': 16,
+ 'index_file_size': 1024 # Optional, default 1024，
+ 'metric_type': MetricType.L2 # Optional, default MetricType.L2
+ }
+
+
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ if not isinstance(param, dict):
+ raise ParamError('Param type incorrect, expect {} but get {} instead'
+ .format(type(dict), type(param)))
+
+ collection_param = copy.deepcopy(param)
+
+ if 'collection_name' not in collection_param:
+ raise ParamError('collection_name is required')
+ collection_name = collection_param["collection_name"]
+ collection_param.pop('collection_name')
+
+ if 'dimension' not in collection_param:
+ raise ParamError('dimension is required')
+ dim = collection_param["dimension"]
+ collection_param.pop("dimension")
+
+ index_file_size = collection_param.get('index_file_size', 1024)
+ collection_param.pop('index_file_size', None)
+
+ metric_type = collection_param.get('metric_type', MetricType.L2)
+ collection_param.pop('metric_type', None)
+
+ check_pass_param(collection_name=collection_name, dimension=dim,
+ index_file_size=index_file_size, metric_type=metric_type)
+
+ with self._connection() as handler:
+ return handler.create_collection(collection_name, dim, index_file_size,
+ metric_type, collection_param, timeout)
+
+[docs] @check_connect
+ def has_collection(self, collection_name, timeout=30):
+ """
+
+ Checks whether a collection exists.
+
+ :param collection_name: Name of the collection to check.
+ :type collection_name: str
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status and the flag indicating if collection exists. Succeed
+ if `Status.OK()` is `True`. If status is not OK, the flag is always `False`.
+ :rtype: Status, bool
+
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.has_collection(collection_name, timeout)
+
+[docs] @check_connect
+ def get_collection_info(self, collection_name, timeout=30):
+ """
+ Returns information of a collection.
+
+ :type collection_name: str
+ :param collection_name: Name of the collection to describe.
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status and collection information. Succeed if `Status.OK()`
+ is `True`. If status is not OK, the returned information is always `None`.
+ :rtype: Status, CollectionSchema
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.describe_collection(collection_name, timeout)
+
+[docs] @check_connect
+ def count_entities(self, collection_name, timeout=30):
+ """
+ Returns the number of vectors in a collection.
+
+ :type collection_name: str
+ :param collection_name: target table name.
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status and row count. Succeed if `Status.OK()` is `True`.
+ If status is not OK, the returned value of is always `None`.
+ :rtype: Status, int
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.count_collection(collection_name, timeout)
+
+[docs] @check_connect
+ def list_collections(self, timeout=30):
+ """
+ Returns collection list.
+
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status and collection name list. Succeed if `Status.OK()` is `True`.
+ If status is not OK, the returned name list is always `[]`.
+ :rtype: Status, list[str]
+ """
+ with self._connection() as handler:
+ return handler.show_collections(timeout)
+
+[docs] @check_connect
+ def get_collection_stats(self, collection_name, timeout=30):
+ """
+
+ Returns collection statistics information.
+
+ :type collection_name: str
+ :param collection_name: target table name.
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status and collection statistics information. Succeed
+ if `Status.OK()` is `True`. If status is not OK, the returned information
+ is always `[]`.
+ :rtype: Status, dict
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.show_collection_info(collection_name, timeout)
+
+[docs] @check_connect
+ def load_collection(self, collection_name, partition_tags=None, timeout=None):
+ """
+ Loads a collection for caching.
+
+ :param collection_name: collection to load
+ :type collection_name: str
+ :param partition_tags: partition tag list. `None` indicates to load whole collection,
+ otherwise to load specified partitions.
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.preload_collection(collection_name, partition_tags, timeout)
+
+[docs] @check_connect
+ def release_collection(self, collection_name, partition_tags=None, timeout=None):
+ """
+ Release a collection from memory and cache.
+
+ :param collection_name: collection to release
+ :type collection_name: str
+ :param partition_tags: partition tag list. `None` indicates to release whole collection,
+ otherwise to release specified partitions.
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.release_collection(collection_name, partition_tags, timeout)
+
+ @check_connect
+ def reload_segments(self, collection_name, segment_ids, timeout=None):
+ """
+ Reloads segment DeletedDocs data to cache. This API is not recommended for users.
+
+ :param collection_name: Name of the collection being deleted
+ :type collection_name: str
+ :param segment_ids: Segment IDs.
+ :type segment_ids: list[str]
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.reload_segments(collection_name, segment_ids, timeout)
+
+[docs] @check_connect
+ def drop_collection(self, collection_name, timeout=30):
+ """
+ Deletes a collection by name.
+
+ :param collection_name: Name of the collection being deleted
+ :type collection_name: str
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.drop_collection(collection_name, timeout)
+
+[docs] @check_connect
+ def insert(self, collection_name, records, ids=None, partition_tag=None,
+ params=None, timeout=None, **kwargs):
+ """
+ Insert vectors to a collection.
+
+ :type collection_name: str
+ :param collection_name: Name of the collection to insert vectors to.
+ :param ids: ID list. `None` indicates ID is generated by server system. Note that if the
+ first time when insert() is invoked ids is not passed into this method, each
+ of the rest time when inset() is invoked ids is not permitted to pass,
+ otherwise server will return an error and the insertion process will fail.
+ And vice versa.
+ :type ids: list[int]
+ :param records: List of vectors to insert.
+ :type records: list[list[float]]
+ :param partition_tag: Tag of a partition.
+ :type partition_tag: str or None. If partition_tag is None, vectors will be inserted to the
+ default partition `_default`.
+ :param params: Insert param. Reserved.
+ :type params: dict
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+ :param kwargs:
+ * *_async* (``bool``) --
+ Indicate if invoke asynchronously. When value is true, method returns a InsertFuture
+ object; otherwise, method returns results from server.
+ * *_callback* (``function``) --
+ The callback function which is invoked after server response successfully. It only
+ takes effect when _async is set to True.
+
+ :return: The operation status and IDs of inserted entities. Succeed if `Status.OK()`
+ is `True`. If status is not OK, the returned IDs is always `[]`.
+ :rtype: Status, list[int]
+ """
+ if kwargs.get("insert_param", None) is not None:
+ with self._connection() as handler:
+ return handler.insert(None, None, timeout=timeout, **kwargs)
+
+ check_pass_param(collection_name=collection_name, records=records)
+ _ = partition_tag is not None and check_pass_param(partition_tag=partition_tag)
+ if ids is not None:
+ check_pass_param(ids=ids)
+ if len(records) != len(ids):
+ raise ParamError("length of vectors do not match that of ids")
+
+ params = params or dict()
+ if not isinstance(params, dict):
+ raise ParamError("Params must be a dictionary type")
+ with self._connection() as handler:
+ return handler.insert(collection_name, records, ids, partition_tag,
+ params, timeout, **kwargs)
+
+[docs] def get_entity_by_id(self, collection_name, ids, timeout=None, partition_tag=None):
+ """
+ Returns raw vectors according to ids.
+
+ :param collection_name: Name of the collection
+ :type collection_name: str
+
+ :param ids: list of vector id
+ :type ids: list
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :param partition_tag: The partition tag of entity
+ :type partition_tag: str
+
+ :return: The operation status and entities. Succeed if `Status.OK()` is `True`.
+ If status is not OK, the returned entities is always `[]`.
+ :rtype: Status, list[list[float]]
+ """
+ check_pass_param(collection_name=collection_name, ids=ids)
+ _ = partition_tag is None or check_pass_param(partition_tag=partition_tag)
+
+ with self._connection() as handler:
+ return handler.get_vectors_by_ids(collection_name, ids, timeout=timeout,
+ partition_tag=partition_tag)
+
+[docs] @check_connect
+ def list_id_in_segment(self, collection_name, segment_name, timeout=None):
+ """
+ Get IDs of entity stored in the specified segment.
+
+ :param collection_name: Collection the segment belongs to.
+ :type collection_name: str
+ :param segment_name: Segment name.
+ :type segment_name: str
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status and entity IDs. Succeed if `Status.OK()` is `True`.
+ If status is not OK, the returned IDs is always `[]`.
+ :rtype: Status, list[int]
+ """
+ check_pass_param(collection_name=collection_name)
+ check_pass_param(collection_name=segment_name)
+ with self._connection() as handler:
+ return handler.get_vector_ids(collection_name, segment_name, timeout)
+
+[docs] @check_connect
+ def create_index(self, collection_name, index_type=None, params=None, timeout=None, **kwargs):
+ """
+ Creates index for a collection.
+
+ :param collection_name: Collection used to create index.
+ :type collection_name: str
+ :param index_type: index params. See `index params <param.html>`_ for supported indexes.
+ :type index_type: IndexType
+ :param params: Index param. See `index params <param.html>`_ for detailed index param of
+ supported indexes.
+ :type params: dict
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+ :param kwargs:
+ * *_async* (``bool``) --
+ Indicate if invoke asynchronously. When value is true, method returns a IndexFuture
+ object; otherwise, method returns results from server.
+ * *_callback* (``function``) --
+ The callback function which is invoked after server response successfully. It only
+ takes effect when _async is set to True.
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ _index_type = IndexType.FLAT if index_type is None else index_type
+ check_pass_param(collection_name=collection_name, index_type=_index_type)
+
+ params = params or dict()
+ if not isinstance(params, dict):
+ raise ParamError("Params must be a dictionary type")
+ with self._connection() as handler:
+ return handler.create_index(collection_name, _index_type, params, timeout, **kwargs)
+
+[docs] @check_connect
+ def get_index_info(self, collection_name, timeout=30):
+ """
+ Show index information of a collection.
+
+ :type collection_name: str
+ :param collection_name: table name been queried
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status and index info. Succeed if `Status.OK()` is `True`.
+ If status is not OK, the returned index info is always `None`.
+ :rtype: Status, IndexParam
+
+ """
+ check_pass_param(collection_name=collection_name)
+
+ with self._connection() as handler:
+ return handler.describe_index(collection_name, timeout)
+
+[docs] @check_connect
+ def drop_index(self, collection_name, timeout=30):
+ """
+ Removes an index.
+
+ :param collection_name: target collection name.
+ :type collection_name: str
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ check_pass_param(collection_name=collection_name)
+
+ with self._connection() as handler:
+ return handler.drop_index(collection_name, timeout)
+
+[docs] @check_connect
+ def create_partition(self, collection_name, partition_tag, timeout=30):
+ """
+ create a partition for a collection.
+
+ :param collection_name: Name of the collection.
+ :type collection_name: str
+ :param partition_tag: Name of the partition tag.
+ :type partition_tag: str
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+
+ """
+ check_pass_param(collection_name=collection_name, partition_tag=partition_tag)
+ with self._connection() as handler:
+ return handler.create_partition(collection_name, partition_tag, timeout)
+
+[docs] @check_connect
+ def has_partition(self, collection_name, partition_tag, timeout=30):
+ """
+ Check if specified partition exists.
+
+ :param collection_name: target table name.
+ :type collection_name: str
+ :param partition_tag: partition tag.
+ :type partition_tag: str
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :returns: The operation status and a flag indicating if partition exists. Succeed
+ if `Status.OK()` is `True`. If status is not ok, the flag is always `False`.
+ :rtype: Status, bool
+
+ """
+ check_pass_param(collection_name=collection_name, partition_tag=partition_tag)
+ with self._connection() as handler:
+ return handler.has_partition(collection_name, partition_tag, timeout)
+
+[docs] @check_connect
+ def list_partitions(self, collection_name, timeout=30):
+ """
+ Show all partitions in a collection.
+
+ :param collection_name: target table name.
+ :type collection_name: str
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :returns: The operation status and partition list. Succeed if `Status.OK()` is `True`.
+ If status is not OK, returned partition list is `[]`.
+ :rtype: Status, list[PartitionParam]
+
+ """
+ check_pass_param(collection_name=collection_name)
+
+ with self._connection() as handler:
+ return handler.show_partitions(collection_name, timeout)
+
+[docs] @check_connect
+ def drop_partition(self, collection_name, partition_tag, timeout=30):
+ """
+ Deletes a partition in a collection.
+
+ :param collection_name: Collection name.
+ :type collection_name: str
+ :param partition_tag: Partition name.
+ :type partition_tag: str
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+
+ """
+ check_pass_param(collection_name=collection_name, partition_tag=partition_tag)
+ with self._connection() as handler:
+ return handler.drop_partition(collection_name, partition_tag, timeout)
+
+[docs] @check_connect
+ def search(self, collection_name, top_k, query_records, partition_tags=None,
+ params=None, timeout=None, **kwargs):
+ """
+ Search vectors in a collection.
+
+ :param collection_name: Name of the collection.
+ :type collection_name: str
+ :param top_k: number of vectors which is most similar with query vectors
+ :type top_k: int
+ :param query_records: vectors to query
+ :type query_records: list[list[float32]]
+ :param partition_tags: tags to search. `None` indicates to search in whole collection.
+ :type partition_tags: list
+ :param params: Search params. The params is related to index type the collection is built.
+ See `index params <param.html>`_ for more detailed information.
+ :type params: dict
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+ :param kwargs:
+ * *_async* (``bool``) --
+ Indicate if invoke asynchronously. When value is true, method returns a SearchFuture
+ object; otherwise, method returns results from server.
+ * *_callback* (``function``) --
+ The callback function which is invoked after server response successfully. It only
+ takes effect when _async is set to True.
+
+ :returns: The operation status and search result. See <a>here</a> to find how to handle
+ search result. Succeed if `Status.OK()` is `True`. If status is not OK,
+ results is always `None`.
+ :rtype: Status, TopKQueryResult
+ """
+ check_pass_param(collection_name=collection_name, topk=top_k, records=query_records)
+ if partition_tags is not None:
+ check_pass_param(partition_tag_array=partition_tags)
+
+ params = dict() if params is None else params
+ if not isinstance(params, dict):
+ raise ParamError("Params must be a dictionary type")
+ with self._connection() as handler:
+ return handler.search(collection_name, top_k, query_records,
+ partition_tags, params, timeout, **kwargs)
+
+ @check_connect
+ def search_in_segment(self, collection_name, file_ids, query_records, top_k,
+ params=None, timeout=None, **kwargs):
+ """
+ Searches for vectors in specific segments of a collection.
+ This API is not recommended for users.
+
+ The Milvus server stores vector data into multiple files. Searching for vectors in specific
+ files is a method used in Mishards. Obtain more detail about Mishards, see
+ `Mishards <https://github.com/milvus-io/milvus/tree/master/shards>`_.
+
+ :param collection_name: table name been queried
+ :type collection_name: str
+ :param file_ids: Specified files id array
+ :type file_ids: list[str] or list[int]
+ :param query_records: all vectors going to be queried
+ :type query_records: list[list[float]]
+ :param top_k: how many similar vectors will be searched
+ :type top_k: int
+ :param params: Search params. The params is related to index type the collection is built.
+ See <a></a> for more detailed information.
+ :type params: dict
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+ :param kwargs:
+ * *_async* (``bool``) --
+ Indicate if invoke asynchronously. When value is true, method returns a SearchFuture
+ object; otherwise, method returns results from server.
+ * *_callback* (``function``) --
+ The callback function which is invoked after server response successfully. It only
+ takes effect when _async is set to True.
+
+ :returns: The operation status and search result. See <a>here</a> to find how to handle
+ search result. Succeed if `Status.OK()` is `True`. If status is not OK, results
+ is always `None`.
+ :rtype: Status, TopKQueryResult
+ """
+ check_pass_param(collection_name=collection_name, topk=top_k,
+ records=query_records, ids=file_ids)
+
+ params = dict() if params is None else params
+ if not isinstance(params, dict):
+ raise ParamError("Params must be a dictionary type")
+ with self._connection() as handler:
+ return handler.search_in_files(collection_name, file_ids,
+ query_records, top_k, params, timeout, **kwargs)
+
+[docs] @check_connect
+ def delete_entity_by_id(self, collection_name, id_array, timeout=None, partition_tag=None):
+ """
+ Deletes vectors in a collection by vector ID.
+
+ :param collection_name: Name of the collection.
+ :type collection_name: str
+ :param id_array: list of vector id
+ :type id_array: list[int]
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+ :param partition_tag: The partition tag of entity
+ :type partition_tag: str
+
+ :returns: The operation status. If the specified ID doesn't exist, Milvus server skip it
+ and try to delete next entities, which is regard as one successful operation.
+ Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ check_pass_param(collection_name=collection_name, ids=id_array)
+ _ = partition_tag is None or check_pass_param(partition_tag=partition_tag)
+ with self._connection() as handler:
+ return handler.delete_by_id(collection_name, id_array, timeout, partition_tag)
+
+[docs] @check_connect
+ def flush(self, collection_name_array=None, timeout=None, **kwargs):
+ """
+ Flushes vector data in one collection or multiple collections to disk.
+
+ :type collection_name_array: list
+ :param collection_name_array: Name of one or multiple collections to flush.
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+ :param kwargs:
+ * *_async* (``bool``) --
+ Indicate if invoke asynchronously. When value is true, method returns a FlushFuture
+ object; otherwise, method returns results from server.
+ * *_callback* (``function``) --
+ The callback function which is invoked after server response successfully. It only
+ takes effect when _async is set to True.
+
+ :returns: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+
+ if collection_name_array in (None, []):
+ with self._connection() as handler:
+ return handler.flush([], timeout)
+
+ if not isinstance(collection_name_array, list):
+ raise ParamError("Collection name array must be type of list")
+
+ if len(collection_name_array) <= 0:
+ raise ParamError("Collection name array is not allowed to be empty")
+
+ for name in collection_name_array:
+ check_pass_param(collection_name=name)
+ with self._connection() as handler:
+ return handler.flush(collection_name_array, timeout, **kwargs)
+
+[docs] @check_connect
+ def compact(self, collection_name, timeout=None, **kwargs):
+ """
+ Compacts segments in a collection. This function is recommended after deleting vectors.
+
+ :type collection_name: str
+ :param collection_name: Name of the collections to compact.
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+ :param kwargs:
+ * *_async* (``bool``) --
+ Indicate if invoke asynchronously. When value is true, method returns a CompactFuture
+ object; otherwise, method returns results from server.
+ * *_callback* (``function``) --
+ The callback function which is invoked after server response successfully. It only
+ takes effect when _async is set to True.
+
+ :returns: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.compact(collection_name, timeout, **kwargs)
+
+ def get_config(self, parent_key, child_key):
+ """
+ Gets Milvus configurations.
+
+ """
+ cmd = "get_config {}.{}".format(parent_key, child_key)
+
+ return self._cmd(cmd)
+
+ def set_config(self, parent_key, child_key, value):
+ """
+ Sets Milvus configurations.
+
+ """
+ cmd = "set_config {}.{} {}".format(parent_key, child_key, value)
+
+ return self._cmd(cmd)
+
+
+
+
+
+
+from enum import IntEnum
+
+
+class Status:
+ """
+ :attribute code: int (optional) default as ok
+
+ :attribute message: str (optional) current status message
+ """
+
+ SUCCESS = 0
+ UNEXPECTED_ERROR = 1
+ CONNECT_FAILED = 2
+ PERMISSION_DENIED = 3
+ COLLECTION_NOT_EXISTS = 4
+ ILLEGAL_ARGUMENT = 5
+ ILLEGAL_RANGE = 6
+ ILLEGAL_DIMENSION = 7
+ ILLEGAL_INDEX_TYPE = 8
+ ILLEGAL_COLLECTION_NAME = 9
+ ILLEGAL_TOPK = 10
+ ILLEGAL_ROWRECORD = 11
+ ILLEGAL_VECTOR_ID = 12
+ ILLEGAL_SEARCH_RESULT = 13
+ FILE_NOT_FOUND = 14
+ META_FAILED = 15
+ CACHE_FAILED = 16
+ CANNOT_CREATE_FOLDER = 17
+ CANNOT_CREATE_FILE = 18
+ CANNOT_DELETE_FOLDER = 19
+ CANNOT_DELETE_FILE = 20
+ BUILD_INDEX_ERROR = 21
+ ILLEGAL_NLIST = 22
+ ILLEGAL_METRIC_TYPE = 23
+ OUT_OF_MEMORY = 24
+
+ def __init__(self, code=SUCCESS, message="Success"):
+ self.code = code
+ self.message = message
+
+ def __repr__(self):
+ attr_list = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.items()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(attr_list))
+
+ def __eq__(self, other):
+ """
+ Make Status comparable with self by code
+ """
+ if isinstance(other, int):
+ return self.code == other
+
+ return isinstance(other, self.__class__) and self.code == other.code
+
+ def __ne__(self, other):
+ return self != other
+
+ def OK(self):
+ return self.code == Status.SUCCESS
+
+
+[docs]class IndexType(IntEnum):
+ """Index type enum.
+ """
+
+ #: Invalid index type.
+ INVALID = 0
+
+ #: FLAT index. See `FLAT <https://milvus.io/docs/v1.0.0/index.md#FLAT>`_.
+ FLAT = 1
+
+ #: IVF(Inverted File) FLAT index.
+ #: See `IVF_FLAT <https://milvus.io/docs/v1.0.0/index.md#IVF_FLAT>`_.
+ IVF_FLAT = 2
+
+ #: IVF SQ8 index. See `IVF_SQ8 <https://milvus.io/docs/v1.0.0/index.md#IVF_SQ8>`_.
+ IVF_SQ8 = 3
+
+ #: RNSG(Refined NSG) index. See `RNSG <https://milvus.io/docs/v1.0.0/index.md#RNSG>`_.
+ RNSG = 4
+
+ #: IVF SQ8 Hybrid index. See `IVF_SQ8H <https://milvus.io/docs/v1.0.0/index.md#IVF_SQ8H>`_.
+ IVF_SQ8H = 5
+
+ #: IVF PQ index. See `IVF_PQ <https://milvus.io/docs/v1.0.0/index.md#IVF_PQ>`_.
+ IVF_PQ = 6
+
+ #: HNSW index. See `HNSW <https://milvus.io/docs/v1.0.0/index.md#HNSW>`_.
+ HNSW = 11
+
+ #: ANNOY index. See `ANNOY <https://milvus.io/docs/v1.0.0/index.md#ANNOY>`_.
+ ANNOY = 12
+
+ #: Alternative name for `IVF_FLAT`. Reserved for compatibility.
+ IVFLAT = IVF_FLAT
+ #: Alternative name for `IVF_SQ8H`. Reserved for compatibility.
+ IVF_SQ8_H = IVF_SQ8H
+
+ def __repr__(self):
+ return "<{}: {}>".format(self.__class__.__name__, self._name_)
+
+ def __str__(self):
+ return self._name_
+
+
+[docs]class MetricType(IntEnum):
+ """Metric type enum.
+ """
+
+ #: Invalid metric type.
+ INVALID = 0
+
+ #: Euclidean distance. A metric for float vectors.
+ #: See `Euclidean distance <https://milvus.io/docs/v1.0.0/metric.md#Euclidean-distance-L2>`_.
+ L2 = 1
+
+ #: Inner product. A metric for float vectors.
+ #: See `Inner Product <https://milvus.io/docs/v1.0.0/metric.md#Inner-product-IP>`_.
+ IP = 2
+
+ #: Hamming distance. A metric for binary vectors.
+ #: See `Hamming distance <https://milvus.io/docs/v1.0.0/metric.md#Hamming-distance>`_.
+ HAMMING = 3
+
+ #: Jaccard distance. A metric for binary vectors.
+ #: See `Jaccard distance <https://milvus.io/docs/v1.0.0/metric.md#Jaccard-distance>`_.
+ JACCARD = 4
+
+ #: Tanimoto distance. A metric for binary vectors.
+ #: See `Tanimoto distance <https://milvus.io/docs/v1.0.0/metric.md#Tanimoto-distance>`_.
+ TANIMOTO = 5
+
+ #: Superstructure. A metric for binary vectors,
+ #: only support :attr:`~milvus.IndexType.FLAT` index.
+ #: See `Superstructure <https://milvus.io/docs/v1.0.0/metric.md#Superstructure>`_.
+ SUBSTRUCTURE = 6
+
+ #: Substructure. A metric for binary vectors, only support :attr:`~milvus.IndexType.FLAT` index.
+ #: See `Substructure <https://milvus.io/docs/v1.0.0/metric.md#Substructure>`_.
+ SUPERSTRUCTURE = 7
+
+ def __repr__(self):
+ return "<{}: {}>".format(self.__class__.__name__, self._name_)
+
+ def __str__(self):
+ return self._name_
+
+
+
+
+
+ This documentation is generated using the Sphinx documentation generator. The source files for the documentation are +located in the doc/ directory of the pymilvus distribution. To generate the docs locally run the following command +under directory doc/:
+$ make html
+The documentation should be generated under directory build/html.
+To preview it, you can open index.html in your browser.
+Or run a web server in that directory:
+$ python3 -m http.server
+Then open your browser to http://localhost:8000.
+Section author: Bosszou@milvus
+
+
+
+
+
+ milvus.Milvus(host=None, port=None, handler='GRPC', pool='SingletonThread', **kwargs)[source]¶create_collection(param, timeout=30)[source]¶Creates a collection.
+param (dict) –
Information needed to create a collection. It contains items:
+collection_name (str) – Collection name.
dimension (int) – Dimension of embeddings stored in collection.
index_file_size (int) – Segment size. See
+Storage Concepts.
metric_type (MetricType) – Distance Metrics type. Valued form
+MetricType. See
+Distance Metrics.
A demo is as follow:
+param={'collection_name': 'name',
+ 'dimension': 16,
+ 'index_file_size': 1024 # Optional, default 1024，
+ 'metric_type': MetricType.L2 # Optional, default MetricType.L2
+ }
+timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status. Succeed if Status.OK() is True.
+Status
+has_collection(collection_name, timeout=30)[source]¶Checks whether a collection exists.
+collection_name (str) – Name of the collection to check.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status and the flag indicating if collection exists. Succeed +if Status.OK() is True. If status is not OK, the flag is always False.
+Status, bool
+get_collection_info(collection_name, timeout=30)[source]¶Returns information of a collection.
+collection_name (str) – Name of the collection to describe.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status and collection information. Succeed if Status.OK() +is True. If status is not OK, the returned information is always None.
+Status, CollectionSchema
+count_entities(collection_name, timeout=30)[source]¶Returns the number of vectors in a collection.
+collection_name (str) – target table name.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status and row count. Succeed if Status.OK() is True. +If status is not OK, the returned value of is always None.
+Status, int
+list_collections(timeout=30)[source]¶Returns collection list.
+timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
+The operation status and collection name list. Succeed if Status.OK() is True. +If status is not OK, the returned name list is always [].
+Status, list[str]
+get_collection_stats(collection_name, timeout=30)[source]¶Returns collection statistics information.
+collection_name (str) – target table name.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status and collection statistics information. Succeed +if Status.OK() is True. If status is not OK, the returned information +is always [].
+Status, dict
+load_collection(collection_name, partition_tags=None, timeout=None)[source]¶Loads a collection for caching.
+collection_name (str) – collection to load
partition_tags – partition tag list. None indicates to load whole collection, +otherwise to load specified partitions.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status. Succeed if Status.OK() is True.
+Status
+release_collection(collection_name, partition_tags=None, timeout=None)[source]¶Release a collection from memory and cache.
+collection_name (str) – collection to release
partition_tags – partition tag list. None indicates to release whole collection, +otherwise to release specified partitions.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status. Succeed if Status.OK() is True.
+Status
+drop_collection(collection_name, timeout=30)[source]¶Deletes a collection by name.
+collection_name (str) – Name of the collection being deleted
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status. Succeed if Status.OK() is True.
+Status
+insert(collection_name, records, ids=None, partition_tag=None, params=None, timeout=None, **kwargs)[source]¶Insert vectors to a collection.
+collection_name (str) – Name of the collection to insert vectors to.
ids (list[int]) – ID list. None indicates ID is generated by server system. Note that if the +first time when insert() is invoked ids is not passed into this method, each +of the rest time when inset() is invoked ids is not permitted to pass, +otherwise server will return an error and the insertion process will fail. +And vice versa.
records (list[list[float]]) – List of vectors to insert.
partition_tag (str or None. If partition_tag is None, vectors will be inserted to the +default partition _default.) – Tag of a partition.
params (dict) – Insert param. Reserved.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
kwargs –
_async (bool) –
+Indicate if invoke asynchronously. When value is true, method returns a InsertFuture
+object; otherwise, method returns results from server.
_callback (function) –
+The callback function which is invoked after server response successfully. It only
+takes effect when _async is set to True.
The operation status and IDs of inserted entities. Succeed if Status.OK() +is True. If status is not OK, the returned IDs is always [].
+Status, list[int]
+get_entity_by_id(collection_name, ids, timeout=None, partition_tag=None)[source]¶Returns raw vectors according to ids.
+collection_name (str) – Name of the collection
ids (list) – list of vector id
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
partition_tag (str) – The partition tag of entity
The operation status and entities. Succeed if Status.OK() is True. +If status is not OK, the returned entities is always [].
+Status, list[list[float]]
+list_id_in_segment(collection_name, segment_name, timeout=None)[source]¶Get IDs of entity stored in the specified segment.
+collection_name (str) – Collection the segment belongs to.
segment_name (str) – Segment name.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status and entity IDs. Succeed if Status.OK() is True. +If status is not OK, the returned IDs is always [].
+Status, list[int]
+create_index(collection_name, index_type=None, params=None, timeout=None, **kwargs)[source]¶Creates index for a collection.
+collection_name (str) – Collection used to create index.
index_type (IndexType) – index params. See index params for supported indexes.
params (dict) –
Index param. See index params for detailed index param of +supported indexes.
+timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
kwargs –
_async (bool) –
+Indicate if invoke asynchronously. When value is true, method returns a IndexFuture
+object; otherwise, method returns results from server.
_callback (function) –
+The callback function which is invoked after server response successfully. It only
+takes effect when _async is set to True.
The operation status. Succeed if Status.OK() is True.
+Status
+get_index_info(collection_name, timeout=30)[source]¶Show index information of a collection.
+collection_name (str) – table name been queried
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status and index info. Succeed if Status.OK() is True. +If status is not OK, the returned index info is always None.
+Status, IndexParam
+drop_index(collection_name, timeout=30)[source]¶Removes an index.
+collection_name (str) – target collection name.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status. Succeed if Status.OK() is True.
+Status
+create_partition(collection_name, partition_tag, timeout=30)[source]¶create a partition for a collection.
+collection_name (str) – Name of the collection.
partition_tag (str) – Name of the partition tag.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status. Succeed if Status.OK() is True.
+Status
+has_partition(collection_name, partition_tag, timeout=30)[source]¶Check if specified partition exists.
+collection_name (str) – target table name.
partition_tag (str) – partition tag.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status and a flag indicating if partition exists. Succeed +if Status.OK() is True. If status is not ok, the flag is always False.
+Status, bool
+list_partitions(collection_name, timeout=30)[source]¶Show all partitions in a collection.
+collection_name (str) – target table name.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status and partition list. Succeed if Status.OK() is True. +If status is not OK, returned partition list is [].
+Status, list[PartitionParam]
+drop_partition(collection_name, partition_tag, timeout=30)[source]¶Deletes a partition in a collection.
+collection_name (str) – Collection name.
partition_tag (str) – Partition name.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status. Succeed if Status.OK() is True.
+Status
+search(collection_name, top_k, query_records, partition_tags=None, params=None, timeout=None, **kwargs)[source]¶Search vectors in a collection.
+collection_name (str) – Name of the collection.
top_k (int) – number of vectors which is most similar with query vectors
query_records (list[list[float32]]) – vectors to query
partition_tags (list) – tags to search. None indicates to search in whole collection.
params (dict) –
Search params. The params is related to index type the collection is built. +See index params for more detailed information.
+timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
kwargs –
_async (bool) –
+Indicate if invoke asynchronously. When value is true, method returns a SearchFuture
+object; otherwise, method returns results from server.
_callback (function) –
+The callback function which is invoked after server response successfully. It only
+takes effect when _async is set to True.
The operation status and search result. See <a>here</a> to find how to handle +search result. Succeed if Status.OK() is True. If status is not OK, +results is always None.
+Status, TopKQueryResult
+delete_entity_by_id(collection_name, id_array, timeout=None, partition_tag=None)[source]¶Deletes vectors in a collection by vector ID.
+collection_name (str) – Name of the collection.
id_array (list[int]) – list of vector id
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
partition_tag (str) – The partition tag of entity
The operation status. If the specified ID doesn’t exist, Milvus server skip it +and try to delete next entities, which is regard as one successful operation. +Succeed if Status.OK() is True.
+Status
+flush(collection_name_array=None, timeout=None, **kwargs)[source]¶Flushes vector data in one collection or multiple collections to disk.
+collection_name_array (list) – Name of one or multiple collections to flush.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
kwargs –
_async (bool) –
+Indicate if invoke asynchronously. When value is true, method returns a FlushFuture
+object; otherwise, method returns results from server.
_callback (function) –
+The callback function which is invoked after server response successfully. It only
+takes effect when _async is set to True.
The operation status. Succeed if Status.OK() is True.
+Status
+compact(collection_name, timeout=None, **kwargs)[source]¶Compacts segments in a collection. This function is recommended after deleting vectors.
+collection_name (str) – Name of the collections to compact.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
kwargs –
_async (bool) –
+Indicate if invoke asynchronously. When value is true, method returns a CompactFuture
+object; otherwise, method returns results from server.
_callback (function) –
+The callback function which is invoked after server response successfully. It only
+takes effect when _async is set to True.
The operation status. Succeed if Status.OK() is True.
+Status
+milvus.MetricType[source]¶Metric type enum.
+INVALID = 0¶Invalid metric type.
+L2 = 1¶Euclidean distance. A metric for float vectors. +See Euclidean distance.
+IP = 2¶Inner product. A metric for float vectors. +See Inner Product.
+HAMMING = 3¶Hamming distance. A metric for binary vectors. +See Hamming distance.
+JACCARD = 4¶Jaccard distance. A metric for binary vectors. +See Jaccard distance.
+TANIMOTO = 5¶Tanimoto distance. A metric for binary vectors. +See Tanimoto distance.
+SUBSTRUCTURE = 6¶Superstructure. A metric for binary vectors,
+only support FLAT index.
+See Superstructure.
SUPERSTRUCTURE = 7¶Substructure. A metric for binary vectors, only support FLAT index.
+See Substructure.
+
+
+
+
+ v1.0.2(Developing)
+v1.0.1
+Remove unused hybrid APIs
Section author: Bosszou@milvus
+
+
+
+
+
+ Contributing is warmly welcomed. You can contribute to PyMilvus project by opening issues and submitting pull +requests on PyMilvus Github page.
+To request a new feature, report a bug or ask a question, it’s recommended for you to open an issue.
+You can tell us why you need it and we will decide whether to implement it soon. +If we think it’s a good improvement, we will make it a feature request and start to work on it. It’s +also welcomed for you to open an issue with your PR as a solution.
+You need to tell us as much information as possible, better start with our +bug report template. +With information, we can reproduce the bug easily and solve it later.
+It’s welcomed to ask any questions about PyMilvus and Milvus, we are pleased to communicate with you.
+If you have improvements to PyMilvus, please submit pull requests(PR) to master, see workflow below.
+PR for codes, you need to tell us why we need it, mentioning an existing issue would be better.
+PR for docs, you also need to tell us why we need it.
+Your PRs will be reviewed and checked, merged into our project if approved.
+This is a brief instruction of Github workflow for beginners.
+Fork the PyMilvus repository on Github.
Clone your fork to your local machine with git clone git@github.com:<your_user_name>/pymilvus.git.
Create a new branch with git checkout -b my_working_branch.
Make your changes, commit, then push to your forked repository.
Visit Github and make you PR.
If you already have an existing local repository, always update it before you start to make changes like below:
+$ git remote add upstream git@github.com:milvus-io/pymilvus.git
+$ git checkout master
+$ git pull upstream master
+$ git checkout -b my_working_branch
+1. Update CHANGELOG.md
+If any improvement or feature being added, you are recommended to open a new issue(if not exist) then +record your change in file CHANGELOG.md. The format is: +- #{GitHub issue number} - {Brief description for your change}
+2. Add unit tests for your codes
+To run unit test in github action, you need make sure the last commit message of PR starts with “[ci]”. +If you want to run unit test locally, under root folder of Pymilvus project run pytest –ip=${IP} –port=${PORT}.
+3. Pass pylint check
+In the root directory, run pylint --rcfile=pylint.conf milvus/client to make sure the rate is 10.
4. For documentations
+You need to enter the doc directory and run make html, please refer to
+About this documentations.
Section author: Yangxuan@milvus
+
+
+
+
+
+ Make sure to set the environment variable GRPC_ENABLE_FORK_SUPPORT=1.
+For reference, see this post.
Try installing PyMilvus in a Conda environment.
+Section author: Yangxuan@milvus
+
+
+
+
+
+ | + |
| + | + |
| + | + |
| + | + |
| + | + |
| + | + |
| + | + |
| + |
| + | + |
| + |
|
+
| + | + |
| + | + |
| + |
+
+
+
+
+ PyMilvus is a python SDK for Milvus and is a recommended way to work with Milvus. This documentation covers +every thing you need to know about PyMilvus.
+Instructions on how to install PyMilvus.
+A quick start to use PyMilvus.
+The complete API documentation.
+Index and relevant parameters.
+How to deal with search results.
+Changes in the latest PyMilvus.
+Method of contribution, bug shooting and contribution guide.
+Some questions that come up often.
+How this documentation is generated.
+Section author: Bosszou@milvus, +Godchen@milvus, +Yangxuan@milvus
+
+
+
+
+
+ PyMilvus is in the Python Package Index.
+PyMilvus only support python3(>= 3.6), usually, it’s ok to install PyMilvus like below.
+$ python3 -m pip install pymilvus
+It’s recommended to use PyMilvus in a virtual environment, using virtual environment allows you to avoid
+installing Python packages globally which could break system tools or other projects.
+We use virtualenv as an example to demonstrate how to install and using PyMilvus in a virtual environment.
+See virtualenv for more information about why and how.
$ python3 -m pip install virtualenv
+$ virtualenv venv
+$ source venv/bin/activate
+(venv) $ pip install pymilvus
+If you want to exit the virtualenv venv, you can use deactivate.
(venv) $ deactivate
+$
+Here we assume you are already in a virtual environment.
+Suitable PyMilvus version depends on Milvus version you are using. See install pymilvus for recommended pymilvus version.
+If you want to install a specific version of PyMilvus:
+(venv) $ pip install pymilvus==1.1.0
+If you want to upgrade PyMilvus into the latest version published:
+(venv) $ pip install --upgrade pymilvus
+This will install the latest PyMilvus into your virtual environment.
+(venv) $ pip install git+https://github.com/milvus-io/pymilvus.git
+Your installation is correct if the following command in the Python shell doesn’t raise an exception.
+(venv) $ python -c "from milvus import Milvus, DataType"
+Section author: Yangxuan@milvus
+
+
+
+
+
+ Milvus support to create index to accelerate vector approximate search.
+To learn how to create an index by python client, see method create_index() and +index example .
+For more detailed information about indexes, please refer to Milvus documentation index chapter.
+To learn how to choose an appropriate index for your application scenarios, please read How to Select an Index in Milvus.
+To learn how to choose an appropriate index for a metric, see Distance Metrics.
+If FLAT index is used, the vectors are stored in an array of float/binary data without any compression. during +searching vectors, all indexed vectors are decoded sequentially and compared to the query vectors.
+FLAT index provides 100% query recall rate. Compared to other indexes, it is the most efficient indexing method +when the number of queries is small.
+The inserted and index-inbuilt vectors and index-dropped vectors are regard as built with FLAT.
building parameters: +N/A
# FLAT
+client.create_index(collection_name, IndexType.FLAT)
+search parameters: +N/A
# FLAT
+client.search(collection_name,
+ 1,
+ query_vectors
+)
+IVF (Inverted File) is an index type based on quantization. It divides the points in space into nlist
+units by clustering method. During searching vectors, it compares the distances between the target vector
+and the center of all the units, and then select the nprobe nearest unit. Then, it compares all the vectors
+in these selected cells to get the final result.
IVF_FLAT is the most basic IVF index, and the encoded data stored in each unit is consistent with the original data.
+building parameters:
+nlist: Number of cluster units.
+# IVF_FLAT
+client.create_index(collection_name, IndexType.IVF_FLAT, {
+ "nlist": 100 # int. 1~65536
+})
+search parameters:
+nprobe: Number of inverted file cell to probe.
+# IVF_FLAT
+client.search(collection_name,
+ 1,
+ query_vectors,
+ params={
+ "nprobe": 8 # int. 1~nlist(cpu), 1~min[2048, nlist](gpu)
+ }
+)
+PQ (Product Quantization) uniformly decomposes the original high-dimensional vector space into
+Cartesian products of m low-dimensional vector spaces, and then quantizes the decomposed low-dimensional
+vector spaces. In the end, each vector is stored in m × nbits bits. Instead of calculating the distances
+between the target vector and the center of all the units, product quantization enables the calculation of
+distances between the target vector, and the clustering center of each low-dimensional space and greatly reduces
+the time complexity and space complexity of the algorithm.
IVF_PQ performs IVF index clustering, and then quantizes the product of vectors. Its index file is even +smaller than IVF_SQ8, but it also causes a loss of accuracy during searching.
+building parameters:
+nlist: Number of cluster units.
+m: Number of factors of product quantization. CPU-only Milvus: m ≡ dim (mod m); GPU-enabled Milvus: m ∈ {1, 2, 3, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 64, 96}, and (dim / m) ∈ {1, 2, 3, 4, 6, 8, 10, 12, 16, 20, 24, 28, 32}. (m x 1024) ≥ MaxSharedMemPerBlock of your graphics card.
nbits: Number of bits in which each low-dimensional vector is stored.
+# IVF_PQ
+client.create_index(collection_name,
+ IndexType.IVF_PQ,
+ {
+ "nlist": 100, # int. 1~65536
+ "m": 8 # int. 1~16. 8 by default
+ }
+)
+search parameters:
+nprobe: Number of inverted file cell to probe.
+# IVF_PQ
+client.search(collection_name,
+ 1,
+ query_vectors,
+ params={
+ "nprobe": 8 # int. 1~nlist(cpu), 1~min[2048, nlist](gpu)
+ }
+)
+IVF_SQ8 does scalar quantization for each vector placed in the unit based on IVF. Scalar quantization +converts each dimension of the original vector from a 4-byte floating-point number to a 1-byte unsigned integer, +so the IVF_SQ8 index file occupies much less space than the IVF_FLAT index file. +However, scalar quantization results in a loss of accuracy during searching vectors.
+building parameters:
+nlist: Number of cluster units.
+# IVF_SQ8
+client.create_index(collection_name,
+ IndexType.IVF_SQ8,
+ {
+ "nlist": 100 # int. 1~65536
+ }
+)
+search parameters:
+nprobe: Number of inverted file cell to probe.
+# IVF_SQ8
+client.search(collection_name,
+ 1,
+ query_vectors,
+ params={
+ "nprobe": 8 # int. 1~nlist(cpu), 1~min[2048, nlist](gpu)
+ }
+)
+Optimized version of IVF_SQ8 that requires both CPU and GPU to work. Unlike IVF_SQ8, IVF_SQ8_H uses a GPU-based +coarse quantizer, which greatly reduces time to quantize.
+IVF_SQ8H is an IVF_SQ8 index that optimizes query execution.
+The query method is as follows:
+If nq ≥ gpu_search_threshold, GPU handles the entire query task.
If nq < gpu_search_threshold, GPU handles the task of retrieving the nprobe nearest unit in the IVF
+index file, and CPU handles the rest.
building parameters:
+nlist: Number of cluster units.
+# IVF_SQ8_H
+client.create_index(collection_name,
+ IndexType.IVF_SQ8_H,
+ {
+ "nlist": 100 # int. 1~65536
+ }
+)
+search parameters:
+nprobe: Number of inverted file cell to probe.
+# IVF_SQ8_H
+client.search(collection_name,
+ 1,
+ query_vectors,
+ params={
+ "nprobe": 8 # int. 1~nlist(cpu), 1~min[2048, nlist](gpu)
+ }
+)
+ANNOY (Approximate Nearest Neighbors Oh Yeah) is an index that uses a hyperplane to divide a +high-dimensional space into multiple subspaces, and then stores them in a tree structure.
+When searching for vectors, ANNOY follows the tree structure to find subspaces closer to the target vector,
+and then compares all the vectors in these subspaces (The number of vectors being compared should not be
+less than search_k) to obtain the final result. Obviously, when the target vector is close to the edge of
+a certain subspace, sometimes it is necessary to greatly increase the number of searched subspaces to obtain
+a high recall rate. Therefore, ANNOY uses n_trees different methods to divide the whole space, and searches
+all the dividing methods simultaneously to reduce the probability that the target vector is always at the edge of the subspace.
building parameters:
+n_trees: The number of methods of space division.
+# ANNOY
+client.create_index(collection_name,
+ IndexType.ANNOY,
+ {
+ "n_trees": 8 # int. 1~1024
+ }
+)
+search parameters:
+search_k: The number of nodes to search. -1 means 5% of the whole data.
+# ANNOY
+client.search(collection_name,
+ 1,
+ query_vectors,
+ params={
+ "search_k": -1 # int. {-1} U [top_k, n*n_trees], n represents vectors count.
+ }
+)
+HNSW (Hierarchical Navigable Small World Graph) is a graph-based indexing algorithm. It builds a +multi-layer navigation structure for an image according to certain rules. In this structure, the upper +layers are more sparse and the distances between nodes are farther; the lower layers are denser and +he distances between nodes are closer. The search starts from the uppermost layer, finds the node closest +to the target in this layer, and then enters the next layer to begin another search. After multiple iterations, +it can quickly approach the target position.
+In order to improve performance, HNSW limits the maximum degree of nodes on each layer of the graph to M.
+In addition, you can use efConstruction (when building index) or ef (when searching targets) to specify a search range.
building parameters:
+M: Maximum degree of the node.
+efConstruction: Take the effect in stage of index construction.
+# HNSW
+client.create_index(collection_name,
+ IndexType.HNSW,
+ {
+ "M": 16, # int. 4~64
+ "efConstruction": 40 # int. 8~512
+ }
+)
+search parameters:
+ef: Take the effect in stage of search scope, should be larger than top_k.
# HNSW
+client.search(collection_name,
+ 1,
+ query_vectors,
+ params={
+ "ef": 64 # int. top_k~32768
+ }
+)
+RNSG (Refined Navigating Spreading-out Graph) is a graph-based indexing algorithm. It sets the center
+position of the whole image as a navigation point, and then uses a specific edge selection strategy to control
+the out-degree of each point (less than or equal to out_degree). Therefore, it can reduce memory usage and
+quickly locate the target position nearby during searching vectors.
The graph construction process of NSG is as follows:
+Find knng nearest neighbors for each point.
Iterate at least search_length times based on knng nearest neighbor nodes to select candidate_pool_size possible nearest neighbor nodes.
Construct the out-edge of each point in the selected candidate_pool_size nodes according to the edge selection strategy.
The query process is similar to the graph building process. It starts from the navigation point and iterates at least search_length times to get the final result.
building parameters:
+search_length: Number of query iterations.
+out_degree: Maximum out-degree of the node.
+candidate_pool_size: Candidate pool size of the node.
+knng: Number of nearest neighbors
+# RNSG
+client.create_index(collection_name,
+ IndexType.RNSG,
+ {
+ "search_length": 60, # int. 10~300
+ "out_degree": 30, # int. 5~300
+ "candidate_pool_size": 300, # int. 50~1000
+ "knng": 50 # int. 5~300
+ }
+)
+search parameters:
+++search_length: Number of query iterations
+
# RNSG
+client.search(collection_name,
+ 1,
+ query_vectors,
+ params={
+ "search_length": 100 # int. 10~300
+ }
+)
+Section author: Godchen@milvus
+
+
+
+
+
+ The invocation of search() is like this:
+>>> results = client.search('demo', query_vectors, topk)
+The result object can be used as a 2-D array. results[i] (0 <= i < len(results)) represents topk results of i-th query +vector, and results[i][j] (0 <= j < len( results[i] )) represents j-th result of i-th query vector. To get result id and distance, +you can invoke like this:
+>>> id = results[i][j].id
+>>> distance = results[i][j].distance
+The results object can be iterated, so you can traverse the results with two-level loop:
+>>> for raw_result in results:
+... for result in raw_result:
+... id = result.id # result id
+... distance = result.distance
+Meanwhile, the results object provide attributes to separately access result id array id_array and distance array distance_array, +so you can traverse the results like this:
+>>> for ids, distances in zip(results.id_array, results.distance_array):
+... for id_, dis_ in zip(ids, distances):
+... print(f"id = {id_}, distance = {dis_}")
+Section author: Bosszou@milvus
+
+
+
+
+
+
+
+
+
+
+ This is a basic introduction to Milvus by PyMilvus.
+For a runnable python script, +checkout example.py on PyMilvus Github, +or hello milvus on Milvus official website. It’s a good recommended +start to get started with Milvus and PyMilvus as well.
+Note
+Here we use float vectors as example vector field data, if you want to learn example about binary vectors, see +binary vector example.
+Before we start, there are some prerequisites.
+Make sure that:
+You have a running Milvus instance.
PyMilvus is correctly installed.
First of all, we need to import PyMilvus.
+>>> from milvus import Milvus, DataType, MetricType
+Then, we can make connection with Milvus server. +By default Milvus runs on localhost in port 19530, so you can use default value to connect to Milvus.
+>>> host = '127.0.0.1'
+>>> port = '19530'
+>>> client = Milvus(host, port)
+After connecting, we can communicate with Milvus in the following ways. If you are confused about the +terminology, see Milvus Terminology for explanations.
+Now let’s create a new collection. Before we start, we can list all the collections already exist. For a brand +new Milvus running instance, the result should be empty.
+>>> client.list_collections()
+(Status(code=0, message='Show collections successfully!'), [])
+To create collection, we need to provide collection parameters.
+collection_param consists of 4 components, they are collection_name, dimension, index_file_size
+and metric_type.
The name of collection should be a unique string to collections already exist.
+For a float vector, dimension should be equal to the length of a vector; for a binary vector, dimension should +be equal to bit size of a vector.
+Milvus controls the size of data segment according to the index_file_size, you can refer to +Storage Concepts for more information about segments and index_file_size.
+Milvus compute distance between two vectors, you can refer to Distance Metrics +for more information.
+Now we can create a collection:
+>>> collection_name = 'demo_film_tutorial'
+>>> collection_param = {
+... "collection_name": collection_name,
+... "dimension": 8,
+... "index_file_size": 2048,
+... "metric_type": MetricType.L2
+... }
+>>> client.create_collection(collection_param)
+Status(code=0, message='Create collection successfully!')
+Then you can list collections and ‘demo_film_tutorial’ will be in the result.
+>>> client.list_collections()
+(Status(code=0, message='Show collections successfully!'), ['demo_film_tutorial'])
+You can also get info of the collection.
+>>> status, info = client.get_collection_info(collection_name)
+>>> info
+CollectionSchema(collection_name='demo_film_tutorial', dimension=8, index_file_size=2048, metric_type=<MetricType: L2>)
+The attributes of collection can be extracted from info.
+>>> info.collection_name
+'demo_film_tutorial'
+>>> info.dimension
+8
+>>> info.index_file_size
+2048
+>>> info.metric_type
+<MetricType: L2>
+This tutorial is a basic intro tutorial, building index won’t be covered by this tutorial. +If you want to go further into Milvus with indexes, it’s recommended to check our +index examples.
+If you’re already known about indexes from index examples, and you want a full lists of params supported
+by PyMilvus, you check out Index
+chapter of the PyMilvus documentation.
Further more, if you want to get a thorough view of indexes, check our official website for +Vector Index.
+If you don’t create a partition, there will be a default one called “_default”, all the entities will be
+inserted into the “_default” partition. You can check it by list_partitions()
>>> client.list_partitions(collection_name)
+(Status(code=0, message='Success'), [(collection_name='demo_film_tutorial', tag='_default')])
+You can provide a partition tag to create a new partition.
+>>> client.create_partition(collection_name, "films")
+Status(code=0, message='OK')
+>>> client.list_partitions(collection_name)
+(Status(code=0, message='Success'), [(collection_name='demo_film_tutorial', tag='_default'), (collection_name='demo_film_tutorial', tag='films')])
+An entity is a group of fields that corresponds to real world objects. In current version, Milvus only contains a vector field. +Here is an example of 3 entities structured in list of list.
+>>> import random
+>>> entities = [[random.random() for _ in range(8)] for _ in range(3)]
+>>>> status, ids = client.insert(collection_name, entities)
+If the entities inserted successfully, ids we provided will be returned.
>>> ids
+[1615279498011637000, 1615279498011637001, 1615279498011637002]
+Or you can also provide entity ids
+>>> entity_ids = [0, 1, 2]
+>>> status, ids = client.insert(collection_name, entities, entity_ids)
+Warning
+If the first time when insert() is invoked ids is not passed into this method, each of the rest time +when inset() is invoked ids is not permitted to pass, otherwise server will return an error and the +insertion process will fail. And vice versa.
+After successfully inserting 3 entities into Milvus, we can Flush data from memory to disk so that we can
+retrieve them. Milvus also performs an automatic flush with a fixed interval(configurable, default 1 second),
+see Data Flushing.
You can flush multiple collections at one time, so be aware the parameter is a list.
+>>> client.flush([collection_name])
+Status(code=0, message='OK')
+After insertion, we can get the detail of collection statistics information by get_collection_stats()
Note
+For a better output format, we are using pprint to provide a better format.
>>> from pprint import pprint
+>>> status, stats = client.get_collection_stats(collection_name)
+>>> pprint(stats)
+{'partitions': [{'row_count': 3,
+ 'segments': [{'data_size': 120,
+ 'index_name': 'IDMAP',
+ 'name': '1615279498038473000',
+ 'row_count': 3}],
+ 'tag': '_default'},
+ {'row_count': 0, 'segments': None, 'tag': 'films'}],
+ 'row_count': 3}
+We can also count how many entities are there in the collection.
+>>> client.count_entities(collection_name)
+(Status(code=0, message='Success!'), 3)
+You can get entities by their ids.
+>>> status, films = client.get_entity_by_id(collection_name, [0, 1615279498011637001])
+>>> films
+[[], [0.8309633731842041, 0.7896093726158142, 0.09463301301002502, 0.7827594876289368, 0.5261889100074768, 0.8051634430885315, 0.18777835369110107, 0.28041353821754456]]
+If id exists, an entity will be returned. If id doesn’t exist, [] will be return. For the example above,
+the result films will only have one entity, the other is []. Because vector id are generated by server, so the value of id may differ.
You can get entities by vector similarity. Assuming we have a film_A like below, and we want to get top 2 films
+that are most similar with it.
>>> film_A = [random.random() for _ in range(8)]
+>>> status, results = client.search(collection_name, 2, [film_A])
+Note
+If the collection is index-built, user need to specify search param, and pass parameter params like: client.search(…, params={…}). +You can refer to Index params for more details.
+Note
+If parameter partition_tags is specified, milvus executes search request on these partition instead of whole collection.
+The returned results is a 2-D like structure, 1 for 1 entity querying, 2 for top 2. For more clarity, we obtain
+the film as below. If you want to know how to deal with search result in a better way, you can refer to
+search result in PyMilvus doc.
>>> result = results[0]
+>>> film_1 = result[0]
+>>> film_2 = result[1]
+Then how do we get ids, distances and fields? It’s as below.
+Note
+Because vectors are randomly generated, so the retrieved vector id and distance may differ.
+>>> film_1.id # id
+1615279498011637002
+>>> film_1.distance # distance
+1.0709768533706665
+Finally, let’s move on to deletion in Milvus. +We can delete entities by ids, drop a whole partition, or drop the entire collection.
+You can delete entities by their ids.
+>>> client.delete_entity_by_id(collection_name, [0, 1615279498011637002])
+Status(code=0, message='OK')
+Note
+If one entity corresponding to a specified id doesn’t exist, milvus ignore it and execute next deletion. +In this case, client always return ok status except any exception occurs.
+>>> client.count_entities(collection_name)
+(Status(code=0, message='Success!'), 2)
+You can also drop a partition.
+Danger
+Once you drop a partition, all the data in this partition will be deleted too.
+>>> client.drop_partition(collection_name, "films")
+Status(code=0, message='OK')
+Finally, you can drop an entire collection.
+Danger
+Once you drop a collection, all the data in this collection will be deleted too.
+>>> client.drop_collection(collection_name)
+Status(code=0, message='OK')
+Section author: Yangxuan@milvus
+
+
+
+
+
+
+
+
+
+
+
+# -*- coding: UTF-8 -*-
+
+import collections
+import copy
+import functools
+import logging
+
+from urllib.parse import urlparse
+
+from . import __version__
+from .types import IndexType, MetricType, Status
+from .check import check_pass_param, is_legal_host, is_legal_port
+from .pool import ConnectionPool, SingleConnectionPool, SingletonThreadPool
+from .exceptions import ParamError, DeprecatedError
+
+from ..settings import DefaultConfig as config
+
+LOGGER = logging.getLogger(__name__)
+
+
+def deprecated(func):
+ @functools.wraps(func)
+ def inner(*args, **kwargs):
+ error_str = "Function {} has been deprecated".format(func.__name__)
+ LOGGER.error(error_str)
+ raise DeprecatedError(error_str)
+
+ return inner
+
+
+def check_connect(func):
+ @functools.wraps(func)
+ def inner(self, *args, **kwargs):
+ return func(self, *args, **kwargs)
+
+ return inner
+
+
+def _pool_args(**kwargs):
+ pool_kwargs = dict()
+ for k, v in kwargs.items():
+ if k in ("pool_size", "wait_timeout", "handler", "try_connect", "pre_ping", "max_retry"):
+ pool_kwargs[k] = v
+
+ return pool_kwargs
+
+
+def _set_uri(host, port, uri, handler="GRPC"):
+ default_port = config.GRPC_PORT if handler == "GRPC" else config.HTTP_PORT
+ default_uri = config.GRPC_URI if handler == "GRPC" else config.HTTP_URI
+
+ if host is not None:
+ _port = port if port is not None else default_port
+ _host = host
+ if handler == "HTTP":
+ _proto = "https" if _port == 443 else "http"
+ else:
+ _proto = "tcp"
+ elif port is None:
+ try:
+ _uri = urlparse(uri) if uri else urlparse(default_uri)
+ _host = _uri.hostname
+ _port = _uri.port
+ _proto = _uri.scheme
+ except (AttributeError, ValueError, TypeError) as e:
+ raise ParamError("uri is illegal: {}".format(e))
+ else:
+ raise ParamError("Param is not complete. Please invoke as follow:\n"
+ "\t(host = ${HOST}, port = ${PORT})\n"
+ "\t(uri = ${URI})\n")
+
+ if not is_legal_host(_host) or not is_legal_port(_port):
+ raise ParamError("host {} or port {} is illegal".format(_host, _port))
+
+ return "{}://{}:{}".format(str(_proto), str(_host), str(_port))
+
+
+[docs]class Milvus:
+ def __init__(self, host=None, port=None, handler="GRPC", pool="SingletonThread", **kwargs):
+ """Constructor method
+ """
+ self._name = kwargs.get('name', None)
+ self._uri = None
+ self._status = None
+ self._connected = False
+ self._handler = handler
+
+ #
+ self._conn = None
+
+ _uri = kwargs.get('uri', None)
+ pool_uri = _set_uri(host, port, _uri, self._handler)
+ pool_kwargs = _pool_args(handler=handler, **kwargs)
+
+ if pool == "QueuePool":
+ self._pool = ConnectionPool(pool_uri, **pool_kwargs)
+ elif pool == "SingletonThread":
+ self._pool = SingletonThreadPool(pool_uri, **pool_kwargs)
+ elif pool == "Singleton":
+ self._pool = SingleConnectionPool(pool_uri, **pool_kwargs)
+ else:
+ raise ParamError("Unknown pool value: {}".format(pool))
+
+ # store extra key-words arguments
+ self._kw = kwargs
+ self._hooks = collections.defaultdict()
+
+ def __enter__(self):
+ self._conn = self._pool.fetch()
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self._conn.close()
+ self._conn = None
+
+ def __del__(self):
+ return self.close()
+
+ def _connection(self):
+ return self._pool.fetch()
+
+ @deprecated
+ def set_hook(self, **kwargs):
+ """
+ Deprecated
+ """
+ # TODO: may remove it.
+ if self._stub:
+ self._stub.set_hook(**kwargs)
+ else:
+ self._hooks.update(kwargs)
+
+ @property
+ def name(self):
+ return self._name
+
+ @property
+ def handler(self):
+ return self._handler
+
+ @deprecated
+ def connect(self, host=None, port=None, uri=None, timeout=2):
+ """
+ Deprecated
+ """
+ if self.connected() and self._connected:
+ return Status(message="You have already connected {} !".format(self._uri),
+ code=Status.CONNECT_FAILED)
+
+ if self._stub is None:
+ self._init(host, port, uri, handler=self._handler)
+
+ if self.ping(timeout):
+ self._status = Status(message="Connected")
+ self._connected = True
+ return self._status
+
+ return Status()
+
+ @deprecated
+ def connected(self):
+ """
+ Deprecated
+ """
+ return self._status and self._status.OK()
+
+ @deprecated
+ def disconnect(self):
+ """
+ Deprecated
+ """
+ pass
+
+ def close(self):
+ """
+ Close client instance
+ """
+ self._pool = None
+
+ def client_version(self):
+ """
+ Returns the version of the client.
+
+ :return: Version of the client.
+
+ :rtype: (str)
+ """
+ return __version__
+
+ def server_status(self, timeout=30):
+ """
+ Returns the status of the Milvus server.
+
+ :return:
+ Status: Whether the operation is successful.
+
+ str : Status of the Milvus server.
+
+ :rtype: (Status, str)
+ """
+ return self._cmd("status", timeout)
+
+ def server_version(self, timeout=30):
+ """
+ Returns the version of the Milvus server.
+
+ :return:
+ Status: Whether the operation is successful.
+
+ str : Version of the Milvus server.
+
+ :rtype: (Status, str)
+ """
+
+ return self._cmd("version", timeout)
+
+ @check_connect
+ def _cmd(self, cmd, timeout=30):
+ check_pass_param(cmd=cmd)
+
+ with self._connection() as handler:
+ return handler._cmd(cmd, timeout)
+
+[docs] @check_connect
+ def create_collection(self, param, timeout=30):
+ """
+ Creates a collection.
+
+ :type param: dict
+ :param param: Information needed to create a collection. It contains items:
+
+ * *collection_name* (``str``) -- Collection name.
+ * *dimension* (``int``) -- Dimension of embeddings stored in collection.
+ * *index_file_size* (``int``) -- Segment size. See
+ `Storage Concepts <https://milvus.io/docs/v1.0.0/storage_concept.md>`_.
+ * *metric_type* (``MetricType``) -- Distance Metrics type. Valued form
+ :class:`~milvus.MetricType`. See
+ `Distance Metrics <https://milvus.io/docs/v1.0.0/metric.md>`_.
+
+
+ A demo is as follow:
+
+ .. code-block:: python
+
+ param={'collection_name': 'name',
+ 'dimension': 16,
+ 'index_file_size': 1024 # Optional, default 1024，
+ 'metric_type': MetricType.L2 # Optional, default MetricType.L2
+ }
+
+
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ if not isinstance(param, dict):
+ raise ParamError('Param type incorrect, expect {} but get {} instead'
+ .format(type(dict), type(param)))
+
+ collection_param = copy.deepcopy(param)
+
+ if 'collection_name' not in collection_param:
+ raise ParamError('collection_name is required')
+ collection_name = collection_param["collection_name"]
+ collection_param.pop('collection_name')
+
+ if 'dimension' not in collection_param:
+ raise ParamError('dimension is required')
+ dim = collection_param["dimension"]
+ collection_param.pop("dimension")
+
+ index_file_size = collection_param.get('index_file_size', 1024)
+ collection_param.pop('index_file_size', None)
+
+ metric_type = collection_param.get('metric_type', MetricType.L2)
+ collection_param.pop('metric_type', None)
+
+ check_pass_param(collection_name=collection_name, dimension=dim,
+ index_file_size=index_file_size, metric_type=metric_type)
+
+ with self._connection() as handler:
+ return handler.create_collection(collection_name, dim, index_file_size,
+ metric_type, collection_param, timeout)
+
+[docs] @check_connect
+ def has_collection(self, collection_name, timeout=30):
+ """
+
+ Checks whether a collection exists.
+
+ :param collection_name: Name of the collection to check.
+ :type collection_name: str
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status and the flag indicating if collection exists. Succeed
+ if `Status.OK()` is `True`. If status is not OK, the flag is always `False`.
+ :rtype: Status, bool
+
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.has_collection(collection_name, timeout)
+
+[docs] @check_connect
+ def get_collection_info(self, collection_name, timeout=30):
+ """
+ Returns information of a collection.
+
+ :type collection_name: str
+ :param collection_name: Name of the collection to describe.
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status and collection information. Succeed if `Status.OK()`
+ is `True`. If status is not OK, the returned information is always `None`.
+ :rtype: Status, CollectionSchema
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.describe_collection(collection_name, timeout)
+
+[docs] @check_connect
+ def count_entities(self, collection_name, timeout=30):
+ """
+ Returns the number of vectors in a collection.
+
+ :type collection_name: str
+ :param collection_name: target table name.
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status and row count. Succeed if `Status.OK()` is `True`.
+ If status is not OK, the returned value of is always `None`.
+ :rtype: Status, int
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.count_collection(collection_name, timeout)
+
+[docs] @check_connect
+ def list_collections(self, timeout=30):
+ """
+ Returns collection list.
+
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status and collection name list. Succeed if `Status.OK()` is `True`.
+ If status is not OK, the returned name list is always `[]`.
+ :rtype: Status, list[str]
+ """
+ with self._connection() as handler:
+ return handler.show_collections(timeout)
+
+[docs] @check_connect
+ def get_collection_stats(self, collection_name, timeout=30):
+ """
+
+ Returns collection statistics information.
+
+ :type collection_name: str
+ :param collection_name: target table name.
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status and collection statistics information. Succeed
+ if `Status.OK()` is `True`. If status is not OK, the returned information
+ is always `[]`.
+ :rtype: Status, dict
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.show_collection_info(collection_name, timeout)
+
+[docs] @check_connect
+ def load_collection(self, collection_name, partition_tags=None, timeout=None):
+ """
+ Loads a collection for caching.
+
+ :param collection_name: collection to load
+ :type collection_name: str
+ :param partition_tags: partition tag list. `None` indicates to load whole collection,
+ otherwise to load specified partitions.
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.preload_collection(collection_name, partition_tags, timeout)
+
+[docs] @check_connect
+ def release_collection(self, collection_name, partition_tags=None, timeout=None):
+ """
+ Release a collection from memory and cache.
+
+ :param collection_name: collection to release
+ :type collection_name: str
+ :param partition_tags: partition tag list. `None` indicates to release whole collection,
+ otherwise to release specified partitions.
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.release_collection(collection_name, partition_tags, timeout)
+
+ @check_connect
+ def reload_segments(self, collection_name, segment_ids, timeout=None):
+ """
+ Reloads segment DeletedDocs data to cache. This API is not recommended for users.
+
+ :param collection_name: Name of the collection being deleted
+ :type collection_name: str
+ :param segment_ids: Segment IDs.
+ :type segment_ids: list[str]
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.reload_segments(collection_name, segment_ids, timeout)
+
+[docs] @check_connect
+ def drop_collection(self, collection_name, timeout=30):
+ """
+ Deletes a collection by name.
+
+ :param collection_name: Name of the collection being deleted
+ :type collection_name: str
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.drop_collection(collection_name, timeout)
+
+[docs] @check_connect
+ def insert(self, collection_name, records, ids=None, partition_tag=None,
+ params=None, timeout=None, **kwargs):
+ """
+ Insert vectors to a collection.
+
+ :type collection_name: str
+ :param collection_name: Name of the collection to insert vectors to.
+ :param ids: ID list. `None` indicates ID is generated by server system. Note that if the
+ first time when insert() is invoked ids is not passed into this method, each
+ of the rest time when inset() is invoked ids is not permitted to pass,
+ otherwise server will return an error and the insertion process will fail.
+ And vice versa.
+ :type ids: list[int]
+ :param records: List of vectors to insert.
+ :type records: list[list[float]]
+ :param partition_tag: Tag of a partition.
+ :type partition_tag: str or None. If partition_tag is None, vectors will be inserted to the
+ default partition `_default`.
+ :param params: Insert param. Reserved.
+ :type params: dict
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+ :param kwargs:
+ * *_async* (``bool``) --
+ Indicate if invoke asynchronously. When value is true, method returns a InsertFuture
+ object; otherwise, method returns results from server.
+ * *_callback* (``function``) --
+ The callback function which is invoked after server response successfully. It only
+ takes effect when _async is set to True.
+
+ :return: The operation status and IDs of inserted entities. Succeed if `Status.OK()`
+ is `True`. If status is not OK, the returned IDs is always `[]`.
+ :rtype: Status, list[int]
+ """
+ if kwargs.get("insert_param", None) is not None:
+ with self._connection() as handler:
+ return handler.insert(None, None, timeout=timeout, **kwargs)
+
+ check_pass_param(collection_name=collection_name, records=records)
+ _ = partition_tag is not None and check_pass_param(partition_tag=partition_tag)
+ if ids is not None:
+ check_pass_param(ids=ids)
+ if len(records) != len(ids):
+ raise ParamError("length of vectors do not match that of ids")
+
+ params = params or dict()
+ if not isinstance(params, dict):
+ raise ParamError("Params must be a dictionary type")
+ with self._connection() as handler:
+ return handler.insert(collection_name, records, ids, partition_tag,
+ params, timeout, **kwargs)
+
+[docs] def get_entity_by_id(self, collection_name, ids, timeout=None, partition_tag=None):
+ """
+ Returns raw vectors according to ids.
+
+ :param collection_name: Name of the collection
+ :type collection_name: str
+
+ :param ids: list of vector id
+ :type ids: list
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :param partition_tag: The partition tag of entity
+ :type partition_tag: str
+
+ :return: The operation status and entities. Succeed if `Status.OK()` is `True`.
+ If status is not OK, the returned entities is always `[]`.
+ :rtype: Status, list[list[float]]
+ """
+ check_pass_param(collection_name=collection_name, ids=ids)
+ _ = partition_tag is None or check_pass_param(partition_tag=partition_tag)
+
+ with self._connection() as handler:
+ return handler.get_vectors_by_ids(collection_name, ids, timeout=timeout,
+ partition_tag=partition_tag)
+
+[docs] @check_connect
+ def list_id_in_segment(self, collection_name, segment_name, timeout=None):
+ """
+ Get IDs of entity stored in the specified segment.
+
+ :param collection_name: Collection the segment belongs to.
+ :type collection_name: str
+ :param segment_name: Segment name.
+ :type segment_name: str
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status and entity IDs. Succeed if `Status.OK()` is `True`.
+ If status is not OK, the returned IDs is always `[]`.
+ :rtype: Status, list[int]
+ """
+ check_pass_param(collection_name=collection_name)
+ check_pass_param(collection_name=segment_name)
+ with self._connection() as handler:
+ return handler.get_vector_ids(collection_name, segment_name, timeout)
+
+[docs] @check_connect
+ def create_index(self, collection_name, index_type=None, params=None, timeout=None, **kwargs):
+ """
+ Creates index for a collection.
+
+ :param collection_name: Collection used to create index.
+ :type collection_name: str
+ :param index_type: index params. See `index params <param.html>`_ for supported indexes.
+ :type index_type: IndexType
+ :param params: Index param. See `index params <param.html>`_ for detailed index param of
+ supported indexes.
+ :type params: dict
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+ :param kwargs:
+ * *_async* (``bool``) --
+ Indicate if invoke asynchronously. When value is true, method returns a IndexFuture
+ object; otherwise, method returns results from server.
+ * *_callback* (``function``) --
+ The callback function which is invoked after server response successfully. It only
+ takes effect when _async is set to True.
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ _index_type = IndexType.FLAT if index_type is None else index_type
+ check_pass_param(collection_name=collection_name, index_type=_index_type)
+
+ params = params or dict()
+ if not isinstance(params, dict):
+ raise ParamError("Params must be a dictionary type")
+ with self._connection() as handler:
+ return handler.create_index(collection_name, _index_type, params, timeout, **kwargs)
+
+[docs] @check_connect
+ def get_index_info(self, collection_name, timeout=30):
+ """
+ Show index information of a collection.
+
+ :type collection_name: str
+ :param collection_name: table name been queried
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status and index info. Succeed if `Status.OK()` is `True`.
+ If status is not OK, the returned index info is always `None`.
+ :rtype: Status, IndexParam
+
+ """
+ check_pass_param(collection_name=collection_name)
+
+ with self._connection() as handler:
+ return handler.describe_index(collection_name, timeout)
+
+[docs] @check_connect
+ def drop_index(self, collection_name, timeout=30):
+ """
+ Removes an index.
+
+ :param collection_name: target collection name.
+ :type collection_name: str
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ check_pass_param(collection_name=collection_name)
+
+ with self._connection() as handler:
+ return handler.drop_index(collection_name, timeout)
+
+[docs] @check_connect
+ def create_partition(self, collection_name, partition_tag, timeout=30):
+ """
+ create a partition for a collection.
+
+ :param collection_name: Name of the collection.
+ :type collection_name: str
+ :param partition_tag: Name of the partition tag.
+ :type partition_tag: str
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+
+ """
+ check_pass_param(collection_name=collection_name, partition_tag=partition_tag)
+ with self._connection() as handler:
+ return handler.create_partition(collection_name, partition_tag, timeout)
+
+[docs] @check_connect
+ def has_partition(self, collection_name, partition_tag, timeout=30):
+ """
+ Check if specified partition exists.
+
+ :param collection_name: target table name.
+ :type collection_name: str
+ :param partition_tag: partition tag.
+ :type partition_tag: str
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :returns: The operation status and a flag indicating if partition exists. Succeed
+ if `Status.OK()` is `True`. If status is not ok, the flag is always `False`.
+ :rtype: Status, bool
+
+ """
+ check_pass_param(collection_name=collection_name, partition_tag=partition_tag)
+ with self._connection() as handler:
+ return handler.has_partition(collection_name, partition_tag, timeout)
+
+[docs] @check_connect
+ def list_partitions(self, collection_name, timeout=30):
+ """
+ Show all partitions in a collection.
+
+ :param collection_name: target table name.
+ :type collection_name: str
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :returns: The operation status and partition list. Succeed if `Status.OK()` is `True`.
+ If status is not OK, returned partition list is `[]`.
+ :rtype: Status, list[PartitionParam]
+
+ """
+ check_pass_param(collection_name=collection_name)
+
+ with self._connection() as handler:
+ return handler.show_partitions(collection_name, timeout)
+
+[docs] @check_connect
+ def drop_partition(self, collection_name, partition_tag, timeout=30):
+ """
+ Deletes a partition in a collection.
+
+ :param collection_name: Collection name.
+ :type collection_name: str
+ :param partition_tag: Partition name.
+ :type partition_tag: str
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+
+ :return: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+
+ """
+ check_pass_param(collection_name=collection_name, partition_tag=partition_tag)
+ with self._connection() as handler:
+ return handler.drop_partition(collection_name, partition_tag, timeout)
+
+[docs] @check_connect
+ def search(self, collection_name, top_k, query_records, partition_tags=None,
+ params=None, timeout=None, **kwargs):
+ """
+ Search vectors in a collection.
+
+ :param collection_name: Name of the collection.
+ :type collection_name: str
+ :param top_k: number of vectors which is most similar with query vectors
+ :type top_k: int
+ :param query_records: vectors to query
+ :type query_records: list[list[float32]]
+ :param partition_tags: tags to search. `None` indicates to search in whole collection.
+ :type partition_tags: list
+ :param params: Search params. The params is related to index type the collection is built.
+ See `index params <param.html>`_ for more detailed information.
+ :type params: dict
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+ :param kwargs:
+ * *_async* (``bool``) --
+ Indicate if invoke asynchronously. When value is true, method returns a SearchFuture
+ object; otherwise, method returns results from server.
+ * *_callback* (``function``) --
+ The callback function which is invoked after server response successfully. It only
+ takes effect when _async is set to True.
+
+ :returns: The operation status and search result. See <a>here</a> to find how to handle
+ search result. Succeed if `Status.OK()` is `True`. If status is not OK,
+ results is always `None`.
+ :rtype: Status, TopKQueryResult
+ """
+ check_pass_param(collection_name=collection_name, topk=top_k, records=query_records)
+ if partition_tags is not None:
+ check_pass_param(partition_tag_array=partition_tags)
+
+ params = dict() if params is None else params
+ if not isinstance(params, dict):
+ raise ParamError("Params must be a dictionary type")
+ with self._connection() as handler:
+ return handler.search(collection_name, top_k, query_records,
+ partition_tags, params, timeout, **kwargs)
+
+ @check_connect
+ def search_in_segment(self, collection_name, file_ids, query_records, top_k,
+ params=None, timeout=None, **kwargs):
+ """
+ Searches for vectors in specific segments of a collection.
+ This API is not recommended for users.
+
+ The Milvus server stores vector data into multiple files. Searching for vectors in specific
+ files is a method used in Mishards. Obtain more detail about Mishards, see
+ `Mishards <https://github.com/milvus-io/milvus/tree/master/shards>`_.
+
+ :param collection_name: table name been queried
+ :type collection_name: str
+ :param file_ids: Specified files id array
+ :type file_ids: list[str] or list[int]
+ :param query_records: all vectors going to be queried
+ :type query_records: list[list[float]]
+ :param top_k: how many similar vectors will be searched
+ :type top_k: int
+ :param params: Search params. The params is related to index type the collection is built.
+ See <a></a> for more detailed information.
+ :type params: dict
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+ :param kwargs:
+ * *_async* (``bool``) --
+ Indicate if invoke asynchronously. When value is true, method returns a SearchFuture
+ object; otherwise, method returns results from server.
+ * *_callback* (``function``) --
+ The callback function which is invoked after server response successfully. It only
+ takes effect when _async is set to True.
+
+ :returns: The operation status and search result. See <a>here</a> to find how to handle
+ search result. Succeed if `Status.OK()` is `True`. If status is not OK, results
+ is always `None`.
+ :rtype: Status, TopKQueryResult
+ """
+ check_pass_param(collection_name=collection_name, topk=top_k,
+ records=query_records, ids=file_ids)
+
+ params = dict() if params is None else params
+ if not isinstance(params, dict):
+ raise ParamError("Params must be a dictionary type")
+ with self._connection() as handler:
+ return handler.search_in_files(collection_name, file_ids,
+ query_records, top_k, params, timeout, **kwargs)
+
+[docs] @check_connect
+ def delete_entity_by_id(self, collection_name, id_array, timeout=None, partition_tag=None):
+ """
+ Deletes vectors in a collection by vector ID.
+
+ :param collection_name: Name of the collection.
+ :type collection_name: str
+ :param id_array: list of vector id
+ :type id_array: list[int]
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+ :param partition_tag: The partition tag of entity
+ :type partition_tag: str
+
+ :returns: The operation status. If the specified ID doesn't exist, Milvus server skip it
+ and try to delete next entities, which is regard as one successful operation.
+ Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ check_pass_param(collection_name=collection_name, ids=id_array)
+ _ = partition_tag is None or check_pass_param(partition_tag=partition_tag)
+ with self._connection() as handler:
+ return handler.delete_by_id(collection_name, id_array, timeout, partition_tag)
+
+[docs] @check_connect
+ def flush(self, collection_name_array=None, timeout=None, **kwargs):
+ """
+ Flushes vector data in one collection or multiple collections to disk.
+
+ :type collection_name_array: list
+ :param collection_name_array: Name of one or multiple collections to flush.
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+ :param kwargs:
+ * *_async* (``bool``) --
+ Indicate if invoke asynchronously. When value is true, method returns a FlushFuture
+ object; otherwise, method returns results from server.
+ * *_callback* (``function``) --
+ The callback function which is invoked after server response successfully. It only
+ takes effect when _async is set to True.
+
+ :returns: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+
+ if collection_name_array in (None, []):
+ with self._connection() as handler:
+ return handler.flush([], timeout)
+
+ if not isinstance(collection_name_array, list):
+ raise ParamError("Collection name array must be type of list")
+
+ if len(collection_name_array) <= 0:
+ raise ParamError("Collection name array is not allowed to be empty")
+
+ for name in collection_name_array:
+ check_pass_param(collection_name=name)
+ with self._connection() as handler:
+ return handler.flush(collection_name_array, timeout, **kwargs)
+
+[docs] @check_connect
+ def compact(self, collection_name, timeout=None, **kwargs):
+ """
+ Compacts segments in a collection. This function is recommended after deleting vectors.
+
+ :type collection_name: str
+ :param collection_name: Name of the collections to compact.
+ :param timeout: An optional duration of time in seconds to allow for the RPC. When timeout
+ is set to None, client waits until server responses or error occurs.
+ :type timeout: float
+ :param kwargs:
+ * *_async* (``bool``) --
+ Indicate if invoke asynchronously. When value is true, method returns a CompactFuture
+ object; otherwise, method returns results from server.
+ * *_callback* (``function``) --
+ The callback function which is invoked after server response successfully. It only
+ takes effect when _async is set to True.
+
+ :returns: The operation status. Succeed if `Status.OK()` is `True`.
+ :rtype: Status
+ """
+ check_pass_param(collection_name=collection_name)
+ with self._connection() as handler:
+ return handler.compact(collection_name, timeout, **kwargs)
+
+ def get_config(self, parent_key, child_key):
+ """
+ Gets Milvus configurations.
+
+ """
+ cmd = "get_config {}.{}".format(parent_key, child_key)
+
+ return self._cmd(cmd)
+
+ def set_config(self, parent_key, child_key, value):
+ """
+ Sets Milvus configurations.
+
+ """
+ cmd = "set_config {}.{} {}".format(parent_key, child_key, value)
+
+ return self._cmd(cmd)
+
+
+
+
+
+
+from enum import IntEnum
+
+
+class Status:
+ """
+ :attribute code: int (optional) default as ok
+
+ :attribute message: str (optional) current status message
+ """
+
+ SUCCESS = 0
+ UNEXPECTED_ERROR = 1
+ CONNECT_FAILED = 2
+ PERMISSION_DENIED = 3
+ COLLECTION_NOT_EXISTS = 4
+ ILLEGAL_ARGUMENT = 5
+ ILLEGAL_RANGE = 6
+ ILLEGAL_DIMENSION = 7
+ ILLEGAL_INDEX_TYPE = 8
+ ILLEGAL_COLLECTION_NAME = 9
+ ILLEGAL_TOPK = 10
+ ILLEGAL_ROWRECORD = 11
+ ILLEGAL_VECTOR_ID = 12
+ ILLEGAL_SEARCH_RESULT = 13
+ FILE_NOT_FOUND = 14
+ META_FAILED = 15
+ CACHE_FAILED = 16
+ CANNOT_CREATE_FOLDER = 17
+ CANNOT_CREATE_FILE = 18
+ CANNOT_DELETE_FOLDER = 19
+ CANNOT_DELETE_FILE = 20
+ BUILD_INDEX_ERROR = 21
+ ILLEGAL_NLIST = 22
+ ILLEGAL_METRIC_TYPE = 23
+ OUT_OF_MEMORY = 24
+
+ def __init__(self, code=SUCCESS, message="Success"):
+ self.code = code
+ self.message = message
+
+ def __repr__(self):
+ attr_list = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.items()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(attr_list))
+
+ def __eq__(self, other):
+ """
+ Make Status comparable with self by code
+ """
+ if isinstance(other, int):
+ return self.code == other
+
+ return isinstance(other, self.__class__) and self.code == other.code
+
+ def __ne__(self, other):
+ return self != other
+
+ def OK(self):
+ return self.code == Status.SUCCESS
+
+
+[docs]class IndexType(IntEnum):
+ """Index type enum.
+ """
+
+ #: Invalid index type.
+ INVALID = 0
+
+ #: FLAT index. See `FLAT <https://milvus.io/docs/v1.0.0/index.md#FLAT>`_.
+ FLAT = 1
+
+ #: IVF(Inverted File) FLAT index.
+ #: See `IVF_FLAT <https://milvus.io/docs/v1.0.0/index.md#IVF_FLAT>`_.
+ IVF_FLAT = 2
+
+ #: IVF SQ8 index. See `IVF_SQ8 <https://milvus.io/docs/v1.0.0/index.md#IVF_SQ8>`_.
+ IVF_SQ8 = 3
+
+ #: RNSG(Refined NSG) index. See `RNSG <https://milvus.io/docs/v1.0.0/index.md#RNSG>`_.
+ RNSG = 4
+
+ #: IVF SQ8 Hybrid index. See `IVF_SQ8H <https://milvus.io/docs/v1.0.0/index.md#IVF_SQ8H>`_.
+ IVF_SQ8H = 5
+
+ #: IVF PQ index. See `IVF_PQ <https://milvus.io/docs/v1.0.0/index.md#IVF_PQ>`_.
+ IVF_PQ = 6
+
+ #: HNSW index. See `HNSW <https://milvus.io/docs/v1.0.0/index.md#HNSW>`_.
+ HNSW = 11
+
+ #: ANNOY index. See `ANNOY <https://milvus.io/docs/v1.0.0/index.md#ANNOY>`_.
+ ANNOY = 12
+
+ #: Alternative name for `IVF_FLAT`. Reserved for compatibility.
+ IVFLAT = IVF_FLAT
+ #: Alternative name for `IVF_SQ8H`. Reserved for compatibility.
+ IVF_SQ8_H = IVF_SQ8H
+
+ def __repr__(self):
+ return "<{}: {}>".format(self.__class__.__name__, self._name_)
+
+ def __str__(self):
+ return self._name_
+
+
+[docs]class MetricType(IntEnum):
+ """Metric type enum.
+ """
+
+ #: Invalid metric type.
+ INVALID = 0
+
+ #: Euclidean distance. A metric for float vectors.
+ #: See `Euclidean distance <https://milvus.io/docs/v1.0.0/metric.md#Euclidean-distance-L2>`_.
+ L2 = 1
+
+ #: Inner product. A metric for float vectors.
+ #: See `Inner Product <https://milvus.io/docs/v1.0.0/metric.md#Inner-product-IP>`_.
+ IP = 2
+
+ #: Hamming distance. A metric for binary vectors.
+ #: See `Hamming distance <https://milvus.io/docs/v1.0.0/metric.md#Hamming-distance>`_.
+ HAMMING = 3
+
+ #: Jaccard distance. A metric for binary vectors.
+ #: See `Jaccard distance <https://milvus.io/docs/v1.0.0/metric.md#Jaccard-distance>`_.
+ JACCARD = 4
+
+ #: Tanimoto distance. A metric for binary vectors.
+ #: See `Tanimoto distance <https://milvus.io/docs/v1.0.0/metric.md#Tanimoto-distance>`_.
+ TANIMOTO = 5
+
+ #: Superstructure. A metric for binary vectors,
+ #: only support :attr:`~milvus.IndexType.FLAT` index.
+ #: See `Superstructure <https://milvus.io/docs/v1.0.0/metric.md#Superstructure>`_.
+ SUBSTRUCTURE = 6
+
+ #: Substructure. A metric for binary vectors, only support :attr:`~milvus.IndexType.FLAT` index.
+ #: See `Substructure <https://milvus.io/docs/v1.0.0/metric.md#Substructure>`_.
+ SUPERSTRUCTURE = 7
+
+ def __repr__(self):
+ return "<{}: {}>".format(self.__class__.__name__, self._name_)
+
+ def __str__(self):
+ return self._name_
+
+
+
+
+
+ This documentation is generated using the Sphinx documentation generator. The source files for the documentation are +located in the doc/ directory of the pymilvus distribution. To generate the docs locally run the following command +under directory doc/:
+$ make html
+The documentation should be generated under directory build/html.
+To preview it, you can open index.html in your browser.
+Or run a web server in that directory:
+$ python3 -m http.server
+Then open your browser to http://localhost:8000.
+Section author: Bosszou@milvus
+
+
+
+
+
+ milvus.Milvus(host=None, port=None, handler='GRPC', pool='SingletonThread', **kwargs)[source]¶create_collection(param, timeout=30)[source]¶Creates a collection.
+param (dict) –
Information needed to create a collection. It contains items:
+collection_name (str) – Collection name.
dimension (int) – Dimension of embeddings stored in collection.
index_file_size (int) – Segment size. See
+Storage Concepts.
metric_type (MetricType) – Distance Metrics type. Valued form
+MetricType. See
+Distance Metrics.
A demo is as follow:
+param={'collection_name': 'name',
+ 'dimension': 16,
+ 'index_file_size': 1024 # Optional, default 1024，
+ 'metric_type': MetricType.L2 # Optional, default MetricType.L2
+ }
+timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status. Succeed if Status.OK() is True.
+Status
+has_collection(collection_name, timeout=30)[source]¶Checks whether a collection exists.
+collection_name (str) – Name of the collection to check.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status and the flag indicating if collection exists. Succeed +if Status.OK() is True. If status is not OK, the flag is always False.
+Status, bool
+get_collection_info(collection_name, timeout=30)[source]¶Returns information of a collection.
+collection_name (str) – Name of the collection to describe.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status and collection information. Succeed if Status.OK() +is True. If status is not OK, the returned information is always None.
+Status, CollectionSchema
+count_entities(collection_name, timeout=30)[source]¶Returns the number of vectors in a collection.
+collection_name (str) – target table name.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status and row count. Succeed if Status.OK() is True. +If status is not OK, the returned value of is always None.
+Status, int
+list_collections(timeout=30)[source]¶Returns collection list.
+timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
+The operation status and collection name list. Succeed if Status.OK() is True. +If status is not OK, the returned name list is always [].
+Status, list[str]
+get_collection_stats(collection_name, timeout=30)[source]¶Returns collection statistics information.
+collection_name (str) – target table name.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status and collection statistics information. Succeed +if Status.OK() is True. If status is not OK, the returned information +is always [].
+Status, dict
+load_collection(collection_name, partition_tags=None, timeout=None)[source]¶Loads a collection for caching.
+collection_name (str) – collection to load
partition_tags – partition tag list. None indicates to load whole collection, +otherwise to load specified partitions.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status. Succeed if Status.OK() is True.
+Status
+release_collection(collection_name, partition_tags=None, timeout=None)[source]¶Release a collection from memory and cache.
+collection_name (str) – collection to release
partition_tags – partition tag list. None indicates to release whole collection, +otherwise to release specified partitions.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status. Succeed if Status.OK() is True.
+Status
+drop_collection(collection_name, timeout=30)[source]¶Deletes a collection by name.
+collection_name (str) – Name of the collection being deleted
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status. Succeed if Status.OK() is True.
+Status
+insert(collection_name, records, ids=None, partition_tag=None, params=None, timeout=None, **kwargs)[source]¶Insert vectors to a collection.
+collection_name (str) – Name of the collection to insert vectors to.
ids (list[int]) – ID list. None indicates ID is generated by server system. Note that if the +first time when insert() is invoked ids is not passed into this method, each +of the rest time when inset() is invoked ids is not permitted to pass, +otherwise server will return an error and the insertion process will fail. +And vice versa.
records (list[list[float]]) – List of vectors to insert.
partition_tag (str or None. If partition_tag is None, vectors will be inserted to the +default partition _default.) – Tag of a partition.
params (dict) – Insert param. Reserved.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
kwargs –
_async (bool) –
+Indicate if invoke asynchronously. When value is true, method returns a InsertFuture
+object; otherwise, method returns results from server.
_callback (function) –
+The callback function which is invoked after server response successfully. It only
+takes effect when _async is set to True.
The operation status and IDs of inserted entities. Succeed if Status.OK() +is True. If status is not OK, the returned IDs is always [].
+Status, list[int]
+get_entity_by_id(collection_name, ids, timeout=None, partition_tag=None)[source]¶Returns raw vectors according to ids.
+collection_name (str) – Name of the collection
ids (list) – list of vector id
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
partition_tag (str) – The partition tag of entity
The operation status and entities. Succeed if Status.OK() is True. +If status is not OK, the returned entities is always [].
+Status, list[list[float]]
+list_id_in_segment(collection_name, segment_name, timeout=None)[source]¶Get IDs of entity stored in the specified segment.
+collection_name (str) – Collection the segment belongs to.
segment_name (str) – Segment name.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status and entity IDs. Succeed if Status.OK() is True. +If status is not OK, the returned IDs is always [].
+Status, list[int]
+create_index(collection_name, index_type=None, params=None, timeout=None, **kwargs)[source]¶Creates index for a collection.
+collection_name (str) – Collection used to create index.
index_type (IndexType) – index params. See index params for supported indexes.
params (dict) –
Index param. See index params for detailed index param of +supported indexes.
+timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
kwargs –
_async (bool) –
+Indicate if invoke asynchronously. When value is true, method returns a IndexFuture
+object; otherwise, method returns results from server.
_callback (function) –
+The callback function which is invoked after server response successfully. It only
+takes effect when _async is set to True.
The operation status. Succeed if Status.OK() is True.
+Status
+get_index_info(collection_name, timeout=30)[source]¶Show index information of a collection.
+collection_name (str) – table name been queried
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status and index info. Succeed if Status.OK() is True. +If status is not OK, the returned index info is always None.
+Status, IndexParam
+drop_index(collection_name, timeout=30)[source]¶Removes an index.
+collection_name (str) – target collection name.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status. Succeed if Status.OK() is True.
+Status
+create_partition(collection_name, partition_tag, timeout=30)[source]¶create a partition for a collection.
+collection_name (str) – Name of the collection.
partition_tag (str) – Name of the partition tag.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status. Succeed if Status.OK() is True.
+Status
+has_partition(collection_name, partition_tag, timeout=30)[source]¶Check if specified partition exists.
+collection_name (str) – target table name.
partition_tag (str) – partition tag.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status and a flag indicating if partition exists. Succeed +if Status.OK() is True. If status is not ok, the flag is always False.
+Status, bool
+list_partitions(collection_name, timeout=30)[source]¶Show all partitions in a collection.
+collection_name (str) – target table name.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status and partition list. Succeed if Status.OK() is True. +If status is not OK, returned partition list is [].
+Status, list[PartitionParam]
+drop_partition(collection_name, partition_tag, timeout=30)[source]¶Deletes a partition in a collection.
+collection_name (str) – Collection name.
partition_tag (str) – Partition name.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
The operation status. Succeed if Status.OK() is True.
+Status
+search(collection_name, top_k, query_records, partition_tags=None, params=None, timeout=None, **kwargs)[source]¶Search vectors in a collection.
+collection_name (str) – Name of the collection.
top_k (int) – number of vectors which is most similar with query vectors
query_records (list[list[float32]]) – vectors to query
partition_tags (list) – tags to search. None indicates to search in whole collection.
params (dict) –
Search params. The params is related to index type the collection is built. +See index params for more detailed information.
+timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
kwargs –
_async (bool) –
+Indicate if invoke asynchronously. When value is true, method returns a SearchFuture
+object; otherwise, method returns results from server.
_callback (function) –
+The callback function which is invoked after server response successfully. It only
+takes effect when _async is set to True.
The operation status and search result. See <a>here</a> to find how to handle +search result. Succeed if Status.OK() is True. If status is not OK, +results is always None.
+Status, TopKQueryResult
+delete_entity_by_id(collection_name, id_array, timeout=None, partition_tag=None)[source]¶Deletes vectors in a collection by vector ID.
+collection_name (str) – Name of the collection.
id_array (list[int]) – list of vector id
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
partition_tag (str) – The partition tag of entity
The operation status. If the specified ID doesn’t exist, Milvus server skip it +and try to delete next entities, which is regard as one successful operation. +Succeed if Status.OK() is True.
+Status
+flush(collection_name_array=None, timeout=None, **kwargs)[source]¶Flushes vector data in one collection or multiple collections to disk.
+collection_name_array (list) – Name of one or multiple collections to flush.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
kwargs –
_async (bool) –
+Indicate if invoke asynchronously. When value is true, method returns a FlushFuture
+object; otherwise, method returns results from server.
_callback (function) –
+The callback function which is invoked after server response successfully. It only
+takes effect when _async is set to True.
The operation status. Succeed if Status.OK() is True.
+Status
+compact(collection_name, timeout=None, **kwargs)[source]¶Compacts segments in a collection. This function is recommended after deleting vectors.
+collection_name (str) – Name of the collections to compact.
timeout (float) – An optional duration of time in seconds to allow for the RPC. When timeout +is set to None, client waits until server responses or error occurs.
kwargs –
_async (bool) –
+Indicate if invoke asynchronously. When value is true, method returns a CompactFuture
+object; otherwise, method returns results from server.
_callback (function) –
+The callback function which is invoked after server response successfully. It only
+takes effect when _async is set to True.
The operation status. Succeed if Status.OK() is True.
+Status
+milvus.MetricType[source]¶Metric type enum.
+INVALID = 0¶Invalid metric type.
+L2 = 1¶Euclidean distance. A metric for float vectors. +See Euclidean distance.
+IP = 2¶Inner product. A metric for float vectors. +See Inner Product.
+HAMMING = 3¶Hamming distance. A metric for binary vectors. +See Hamming distance.
+JACCARD = 4¶Jaccard distance. A metric for binary vectors. +See Jaccard distance.
+TANIMOTO = 5¶Tanimoto distance. A metric for binary vectors. +See Tanimoto distance.
+SUBSTRUCTURE = 6¶Superstructure. A metric for binary vectors,
+only support FLAT index.
+See Superstructure.
SUPERSTRUCTURE = 7¶Substructure. A metric for binary vectors, only support FLAT index.
+See Substructure.
+
+
+
+
+ v1.0.2(Developing)
+v1.0.1
+Remove unused hybrid APIs
Section author: Bosszou@milvus
+
+
+
+
+
+ Contributing is warmly welcomed. You can contribute to PyMilvus project by opening issues and submitting pull +requests on PyMilvus Github page.
+To request a new feature, report a bug or ask a question, it’s recommended for you to open an issue.
+You can tell us why you need it and we will decide whether to implement it soon. +If we think it’s a good improvement, we will make it a feature request and start to work on it. It’s +also welcomed for you to open an issue with your PR as a solution.
+You need to tell us as much information as possible, better start with our +bug report template. +With information, we can reproduce the bug easily and solve it later.
+It’s welcomed to ask any questions about PyMilvus and Milvus, we are pleased to communicate with you.
+If you have improvements to PyMilvus, please submit pull requests(PR) to master, see workflow below.
+PR for codes, you need to tell us why we need it, mentioning an existing issue would be better.
+PR for docs, you also need to tell us why we need it.
+Your PRs will be reviewed and checked, merged into our project if approved.
+This is a brief instruction of Github workflow for beginners.
+Fork the PyMilvus repository on Github.
Clone your fork to your local machine with git clone git@github.com:<your_user_name>/pymilvus.git.
Create a new branch with git checkout -b my_working_branch.
Make your changes, commit, then push to your forked repository.
Visit Github and make you PR.
If you already have an existing local repository, always update it before you start to make changes like below:
+$ git remote add upstream git@github.com:milvus-io/pymilvus.git
+$ git checkout master
+$ git pull upstream master
+$ git checkout -b my_working_branch
+1. Update CHANGELOG.md
+If any improvement or feature being added, you are recommended to open a new issue(if not exist) then +record your change in file CHANGELOG.md. The format is: +- #{GitHub issue number} - {Brief description for your change}
+2. Add unit tests for your codes
+To run unit test in github action, you need make sure the last commit message of PR starts with “[ci]”. +If you want to run unit test locally, under root folder of Pymilvus project run pytest –ip=${IP} –port=${PORT}.
+3. Pass pylint check
+In the root directory, run pylint --rcfile=pylint.conf milvus/client to make sure the rate is 10.
4. For documentations
+You need to enter the doc directory and run make html, please refer to
+About this documentations.
Section author: Yangxuan@milvus
+
+
+
+
+
+ Make sure to set the environment variable GRPC_ENABLE_FORK_SUPPORT=1.
+For reference, see this post.
Try installing PyMilvus in a Conda environment.
+Section author: Yangxuan@milvus
+
+
+
+
+
+ | + |
| + | + |
| + | + |
| + | + |
| + | + |
| + | + |
| + | + |
| + |
| + | + |
| + |
|
+
| + | + |
| + | + |
| + |
+
+
+
+
+ PyMilvus is a python SDK for Milvus and is a recommended way to work with Milvus. This documentation covers +every thing you need to know about PyMilvus.
+Instructions on how to install PyMilvus.
+A quick start to use PyMilvus.
+The complete API documentation.
+Index and relevant parameters.
+How to deal with search results.
+Changes in the latest PyMilvus.
+Method of contribution, bug shooting and contribution guide.
+Some questions that come up often.
+How this documentation is generated.
+Section author: Bosszou@milvus, +Godchen@milvus, +Yangxuan@milvus
+
+
+
+
+
+ PyMilvus is in the Python Package Index.
+PyMilvus only support python3(>= 3.6), usually, it’s ok to install PyMilvus like below.
+$ python3 -m pip install pymilvus
+It’s recommended to use PyMilvus in a virtual environment, using virtual environment allows you to avoid
+installing Python packages globally which could break system tools or other projects.
+We use virtualenv as an example to demonstrate how to install and using PyMilvus in a virtual environment.
+See virtualenv for more information about why and how.
$ python3 -m pip install virtualenv
+$ virtualenv venv
+$ source venv/bin/activate
+(venv) $ pip install pymilvus
+If you want to exit the virtualenv venv, you can use deactivate.
(venv) $ deactivate
+$
+Here we assume you are already in a virtual environment.
+Suitable PyMilvus version depends on Milvus version you are using. See install pymilvus for recommended pymilvus version.
+If you want to install a specific version of PyMilvus:
+(venv) $ pip install pymilvus==1.1.0
+If you want to upgrade PyMilvus into the latest version published:
+(venv) $ pip install --upgrade pymilvus
+This will install the latest PyMilvus into your virtual environment.
+(venv) $ pip install git+https://github.com/milvus-io/pymilvus.git
+Your installation is correct if the following command in the Python shell doesn’t raise an exception.
+(venv) $ python -c "from milvus import Milvus, DataType"
+Section author: Yangxuan@milvus
+
+
+
+
+
+ Milvus support to create index to accelerate vector approximate search.
+To learn how to create an index by python client, see method create_index() and +index example .
+For more detailed information about indexes, please refer to Milvus documentation index chapter.
+To learn how to choose an appropriate index for your application scenarios, please read How to Select an Index in Milvus.
+To learn how to choose an appropriate index for a metric, see Distance Metrics.
+If FLAT index is used, the vectors are stored in an array of float/binary data without any compression. during +searching vectors, all indexed vectors are decoded sequentially and compared to the query vectors.
+FLAT index provides 100% query recall rate. Compared to other indexes, it is the most efficient indexing method +when the number of queries is small.
+The inserted and index-inbuilt vectors and index-dropped vectors are regard as built with FLAT.
building parameters: +N/A
# FLAT
+client.create_index(collection_name, IndexType.FLAT)
+search parameters: +N/A
# FLAT
+client.search(collection_name,
+ 1,
+ query_vectors
+)
+IVF (Inverted File) is an index type based on quantization. It divides the points in space into nlist
+units by clustering method. During searching vectors, it compares the distances between the target vector
+and the center of all the units, and then select the nprobe nearest unit. Then, it compares all the vectors
+in these selected cells to get the final result.
IVF_FLAT is the most basic IVF index, and the encoded data stored in each unit is consistent with the original data.
+building parameters:
+nlist: Number of cluster units.
+# IVF_FLAT
+client.create_index(collection_name, IndexType.IVF_FLAT, {
+ "nlist": 100 # int. 1~65536
+})
+search parameters:
+nprobe: Number of inverted file cell to probe.
+# IVF_FLAT
+client.search(collection_name,
+ 1,
+ query_vectors,
+ params={
+ "nprobe": 8 # int. 1~nlist(cpu), 1~min[2048, nlist](gpu)
+ }
+)
+PQ (Product Quantization) uniformly decomposes the original high-dimensional vector space into
+Cartesian products of m low-dimensional vector spaces, and then quantizes the decomposed low-dimensional
+vector spaces. In the end, each vector is stored in m × nbits bits. Instead of calculating the distances
+between the target vector and the center of all the units, product quantization enables the calculation of
+distances between the target vector, and the clustering center of each low-dimensional space and greatly reduces
+the time complexity and space complexity of the algorithm.
IVF_PQ performs IVF index clustering, and then quantizes the product of vectors. Its index file is even +smaller than IVF_SQ8, but it also causes a loss of accuracy during searching.
+building parameters:
+nlist: Number of cluster units.
+m: Number of factors of product quantization. CPU-only Milvus: m ≡ dim (mod m); GPU-enabled Milvus: m ∈ {1, 2, 3, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 64, 96}, and (dim / m) ∈ {1, 2, 3, 4, 6, 8, 10, 12, 16, 20, 24, 28, 32}. (m x 1024) ≥ MaxSharedMemPerBlock of your graphics card.
nbits: Number of bits in which each low-dimensional vector is stored.
+# IVF_PQ
+client.create_index(collection_name,
+ IndexType.IVF_PQ,
+ {
+ "nlist": 100, # int. 1~65536
+ "m": 8 # int. 1~16. 8 by default
+ }
+)
+search parameters:
+nprobe: Number of inverted file cell to probe.
+# IVF_PQ
+client.search(collection_name,
+ 1,
+ query_vectors,
+ params={
+ "nprobe": 8 # int. 1~nlist(cpu), 1~min[2048, nlist](gpu)
+ }
+)
+IVF_SQ8 does scalar quantization for each vector placed in the unit based on IVF. Scalar quantization +converts each dimension of the original vector from a 4-byte floating-point number to a 1-byte unsigned integer, +so the IVF_SQ8 index file occupies much less space than the IVF_FLAT index file. +However, scalar quantization results in a loss of accuracy during searching vectors.
+building parameters:
+nlist: Number of cluster units.
+# IVF_SQ8
+client.create_index(collection_name,
+ IndexType.IVF_SQ8,
+ {
+ "nlist": 100 # int. 1~65536
+ }
+)
+search parameters:
+nprobe: Number of inverted file cell to probe.
+# IVF_SQ8
+client.search(collection_name,
+ 1,
+ query_vectors,
+ params={
+ "nprobe": 8 # int. 1~nlist(cpu), 1~min[2048, nlist](gpu)
+ }
+)
+Optimized version of IVF_SQ8 that requires both CPU and GPU to work. Unlike IVF_SQ8, IVF_SQ8_H uses a GPU-based +coarse quantizer, which greatly reduces time to quantize.
+IVF_SQ8H is an IVF_SQ8 index that optimizes query execution.
+The query method is as follows:
+If nq ≥ gpu_search_threshold, GPU handles the entire query task.
If nq < gpu_search_threshold, GPU handles the task of retrieving the nprobe nearest unit in the IVF
+index file, and CPU handles the rest.
building parameters:
+nlist: Number of cluster units.
+# IVF_SQ8_H
+client.create_index(collection_name,
+ IndexType.IVF_SQ8_H,
+ {
+ "nlist": 100 # int. 1~65536
+ }
+)
+search parameters:
+nprobe: Number of inverted file cell to probe.
+# IVF_SQ8_H
+client.search(collection_name,
+ 1,
+ query_vectors,
+ params={
+ "nprobe": 8 # int. 1~nlist(cpu), 1~min[2048, nlist](gpu)
+ }
+)
+ANNOY (Approximate Nearest Neighbors Oh Yeah) is an index that uses a hyperplane to divide a +high-dimensional space into multiple subspaces, and then stores them in a tree structure.
+When searching for vectors, ANNOY follows the tree structure to find subspaces closer to the target vector,
+and then compares all the vectors in these subspaces (The number of vectors being compared should not be
+less than search_k) to obtain the final result. Obviously, when the target vector is close to the edge of
+a certain subspace, sometimes it is necessary to greatly increase the number of searched subspaces to obtain
+a high recall rate. Therefore, ANNOY uses n_trees different methods to divide the whole space, and searches
+all the dividing methods simultaneously to reduce the probability that the target vector is always at the edge of the subspace.
building parameters:
+n_trees: The number of methods of space division.
+# ANNOY
+client.create_index(collection_name,
+ IndexType.ANNOY,
+ {
+ "n_trees": 8 # int. 1~1024
+ }
+)
+search parameters:
+search_k: The number of nodes to search. -1 means 5% of the whole data.
+# ANNOY
+client.search(collection_name,
+ 1,
+ query_vectors,
+ params={
+ "search_k": -1 # int. {-1} U [top_k, n*n_trees], n represents vectors count.
+ }
+)
+HNSW (Hierarchical Navigable Small World Graph) is a graph-based indexing algorithm. It builds a +multi-layer navigation structure for an image according to certain rules. In this structure, the upper +layers are more sparse and the distances between nodes are farther; the lower layers are denser and +he distances between nodes are closer. The search starts from the uppermost layer, finds the node closest +to the target in this layer, and then enters the next layer to begin another search. After multiple iterations, +it can quickly approach the target position.
+In order to improve performance, HNSW limits the maximum degree of nodes on each layer of the graph to M.
+In addition, you can use efConstruction (when building index) or ef (when searching targets) to specify a search range.
building parameters:
+M: Maximum degree of the node.
+efConstruction: Take the effect in stage of index construction.
+# HNSW
+client.create_index(collection_name,
+ IndexType.HNSW,
+ {
+ "M": 16, # int. 4~64
+ "efConstruction": 40 # int. 8~512
+ }
+)
+search parameters:
+ef: Take the effect in stage of search scope, should be larger than top_k.
# HNSW
+client.search(collection_name,
+ 1,
+ query_vectors,
+ params={
+ "ef": 64 # int. top_k~32768
+ }
+)
+RNSG (Refined Navigating Spreading-out Graph) is a graph-based indexing algorithm. It sets the center
+position of the whole image as a navigation point, and then uses a specific edge selection strategy to control
+the out-degree of each point (less than or equal to out_degree). Therefore, it can reduce memory usage and
+quickly locate the target position nearby during searching vectors.
The graph construction process of NSG is as follows:
+Find knng nearest neighbors for each point.
Iterate at least search_length times based on knng nearest neighbor nodes to select candidate_pool_size possible nearest neighbor nodes.
Construct the out-edge of each point in the selected candidate_pool_size nodes according to the edge selection strategy.
The query process is similar to the graph building process. It starts from the navigation point and iterates at least search_length times to get the final result.
building parameters:
+search_length: Number of query iterations.
+out_degree: Maximum out-degree of the node.
+candidate_pool_size: Candidate pool size of the node.
+knng: Number of nearest neighbors
+# RNSG
+client.create_index(collection_name,
+ IndexType.RNSG,
+ {
+ "search_length": 60, # int. 10~300
+ "out_degree": 30, # int. 5~300
+ "candidate_pool_size": 300, # int. 50~1000
+ "knng": 50 # int. 5~300
+ }
+)
+search parameters:
+++search_length: Number of query iterations
+
# RNSG
+client.search(collection_name,
+ 1,
+ query_vectors,
+ params={
+ "search_length": 100 # int. 10~300
+ }
+)
+Section author: Godchen@milvus
+
+
+
+
+
+ The invocation of search() is like this:
+>>> results = client.search('demo', query_vectors, topk)
+The result object can be used as a 2-D array. results[i] (0 <= i < len(results)) represents topk results of i-th query +vector, and results[i][j] (0 <= j < len( results[i] )) represents j-th result of i-th query vector. To get result id and distance, +you can invoke like this:
+>>> id = results[i][j].id
+>>> distance = results[i][j].distance
+The results object can be iterated, so you can traverse the results with two-level loop:
+>>> for raw_result in results:
+... for result in raw_result:
+... id = result.id # result id
+... distance = result.distance
+Meanwhile, the results object provide attributes to separately access result id array id_array and distance array distance_array, +so you can traverse the results like this:
+>>> for ids, distances in zip(results.id_array, results.distance_array):
+... for id_, dis_ in zip(ids, distances):
+... print(f"id = {id_}, distance = {dis_}")
+Section author: Bosszou@milvus
+
+
+
+
+
+
+
+
+
+
+ This is a basic introduction to Milvus by PyMilvus.
+For a runnable python script, +checkout example.py on PyMilvus Github, +or hello milvus on Milvus official website. It’s a good recommended +start to get started with Milvus and PyMilvus as well.
+Note
+Here we use float vectors as example vector field data, if you want to learn example about binary vectors, see +binary vector example.
+Before we start, there are some prerequisites.
+Make sure that:
+You have a running Milvus instance.
PyMilvus is correctly installed.
First of all, we need to import PyMilvus.
+>>> from milvus import Milvus, DataType, MetricType
+Then, we can make connection with Milvus server. +By default Milvus runs on localhost in port 19530, so you can use default value to connect to Milvus.
+>>> host = '127.0.0.1'
+>>> port = '19530'
+>>> client = Milvus(host, port)
+After connecting, we can communicate with Milvus in the following ways. If you are confused about the +terminology, see Milvus Terminology for explanations.
+Now let’s create a new collection. Before we start, we can list all the collections already exist. For a brand +new Milvus running instance, the result should be empty.
+>>> client.list_collections()
+(Status(code=0, message='Show collections successfully!'), [])
+To create collection, we need to provide collection parameters.
+collection_param consists of 4 components, they are collection_name, dimension, index_file_size
+and metric_type.
The name of collection should be a unique string to collections already exist.
+For a float vector, dimension should be equal to the length of a vector; for a binary vector, dimension should +be equal to bit size of a vector.
+Milvus controls the size of data segment according to the index_file_size, you can refer to +Storage Concepts for more information about segments and index_file_size.
+Milvus compute distance between two vectors, you can refer to Distance Metrics +for more information.
+Now we can create a collection:
+>>> collection_name = 'demo_film_tutorial'
+>>> collection_param = {
+... "collection_name": collection_name,
+... "dimension": 8,
+... "index_file_size": 2048,
+... "metric_type": MetricType.L2
+... }
+>>> client.create_collection(collection_param)
+Status(code=0, message='Create collection successfully!')
+Then you can list collections and ‘demo_film_tutorial’ will be in the result.
+>>> client.list_collections()
+(Status(code=0, message='Show collections successfully!'), ['demo_film_tutorial'])
+You can also get info of the collection.
+>>> status, info = client.get_collection_info(collection_name)
+>>> info
+CollectionSchema(collection_name='demo_film_tutorial', dimension=8, index_file_size=2048, metric_type=<MetricType: L2>)
+The attributes of collection can be extracted from info.
+>>> info.collection_name
+'demo_film_tutorial'
+>>> info.dimension
+8
+>>> info.index_file_size
+2048
+>>> info.metric_type
+<MetricType: L2>
+This tutorial is a basic intro tutorial, building index won’t be covered by this tutorial. +If you want to go further into Milvus with indexes, it’s recommended to check our +index examples.
+If you’re already known about indexes from index examples, and you want a full lists of params supported
+by PyMilvus, you check out Index
+chapter of the PyMilvus documentation.
Further more, if you want to get a thorough view of indexes, check our official website for +Vector Index.
+If you don’t create a partition, there will be a default one called “_default”, all the entities will be
+inserted into the “_default” partition. You can check it by list_partitions()
>>> client.list_partitions(collection_name)
+(Status(code=0, message='Success'), [(collection_name='demo_film_tutorial', tag='_default')])
+You can provide a partition tag to create a new partition.
+>>> client.create_partition(collection_name, "films")
+Status(code=0, message='OK')
+>>> client.list_partitions(collection_name)
+(Status(code=0, message='Success'), [(collection_name='demo_film_tutorial', tag='_default'), (collection_name='demo_film_tutorial', tag='films')])
+An entity is a group of fields that corresponds to real world objects. In current version, Milvus only contains a vector field. +Here is an example of 3 entities structured in list of list.
+>>> import random
+>>> entities = [[random.random() for _ in range(8)] for _ in range(3)]
+>>>> status, ids = client.insert(collection_name, entities)
+If the entities inserted successfully, ids we provided will be returned.
>>> ids
+[1615279498011637000, 1615279498011637001, 1615279498011637002]
+Or you can also provide entity ids
+>>> entity_ids = [0, 1, 2]
+>>> status, ids = client.insert(collection_name, entities, entity_ids)
+Warning
+If the first time when insert() is invoked ids is not passed into this method, each of the rest time +when inset() is invoked ids is not permitted to pass, otherwise server will return an error and the +insertion process will fail. And vice versa.
+After successfully inserting 3 entities into Milvus, we can Flush data from memory to disk so that we can
+retrieve them. Milvus also performs an automatic flush with a fixed interval(configurable, default 1 second),
+see Data Flushing.
You can flush multiple collections at one time, so be aware the parameter is a list.
+>>> client.flush([collection_name])
+Status(code=0, message='OK')
+After insertion, we can get the detail of collection statistics information by get_collection_stats()
Note
+For a better output format, we are using pprint to provide a better format.
>>> from pprint import pprint
+>>> status, stats = client.get_collection_stats(collection_name)
+>>> pprint(stats)
+{'partitions': [{'row_count': 3,
+ 'segments': [{'data_size': 120,
+ 'index_name': 'IDMAP',
+ 'name': '1615279498038473000',
+ 'row_count': 3}],
+ 'tag': '_default'},
+ {'row_count': 0, 'segments': None, 'tag': 'films'}],
+ 'row_count': 3}
+We can also count how many entities are there in the collection.
+>>> client.count_entities(collection_name)
+(Status(code=0, message='Success!'), 3)
+You can get entities by their ids.
+>>> status, films = client.get_entity_by_id(collection_name, [0, 1615279498011637001])
+>>> films
+[[], [0.8309633731842041, 0.7896093726158142, 0.09463301301002502, 0.7827594876289368, 0.5261889100074768, 0.8051634430885315, 0.18777835369110107, 0.28041353821754456]]
+If id exists, an entity will be returned. If id doesn’t exist, [] will be return. For the example above,
+the result films will only have one entity, the other is []. Because vector id are generated by server, so the value of id may differ.
You can get entities by vector similarity. Assuming we have a film_A like below, and we want to get top 2 films
+that are most similar with it.
>>> film_A = [random.random() for _ in range(8)]
+>>> status, results = client.search(collection_name, 2, [film_A])
+Note
+If the collection is index-built, user need to specify search param, and pass parameter params like: client.search(…, params={…}). +You can refer to Index params for more details.
+Note
+If parameter partition_tags is specified, milvus executes search request on these partition instead of whole collection.
+The returned results is a 2-D like structure, 1 for 1 entity querying, 2 for top 2. For more clarity, we obtain
+the film as below. If you want to know how to deal with search result in a better way, you can refer to
+search result in PyMilvus doc.
>>> result = results[0]
+>>> film_1 = result[0]
+>>> film_2 = result[1]
+Then how do we get ids, distances and fields? It’s as below.
+Note
+Because vectors are randomly generated, so the retrieved vector id and distance may differ.
+>>> film_1.id # id
+1615279498011637002
+>>> film_1.distance # distance
+1.0709768533706665
+Finally, let’s move on to deletion in Milvus. +We can delete entities by ids, drop a whole partition, or drop the entire collection.
+You can delete entities by their ids.
+>>> client.delete_entity_by_id(collection_name, [0, 1615279498011637002])
+Status(code=0, message='OK')
+Note
+If one entity corresponding to a specified id doesn’t exist, milvus ignore it and execute next deletion. +In this case, client always return ok status except any exception occurs.
+>>> client.count_entities(collection_name)
+(Status(code=0, message='Success!'), 2)
+You can also drop a partition.
+Danger
+Once you drop a partition, all the data in this partition will be deleted too.
+>>> client.drop_partition(collection_name, "films")
+Status(code=0, message='OK')
+Finally, you can drop an entire collection.
+Danger
+Once you drop a collection, all the data in this collection will be deleted too.
+>>> client.drop_collection(collection_name)
+Status(code=0, message='OK')
+Section author: Yangxuan@milvus
+
+
+
+
+
+ | \n", + " | movie_index | \n", + "text | \n", + "label_int | \n", + "label | \n", + "
|---|---|---|---|---|
| 0 | \n", + "26813 | \n", + "Fot the most part, this movie feels like a \"ma... | \n", + "0 | \n", + "Negative | \n", + "
| 1 | \n", + "26581 | \n", + "Are you kidding me? The music was SO LOUD in t... | \n", + "0 | \n", + "Negative | \n", + "
| \n", + " | movie_index | \n", + "text | \n", + "chunk | \n", + "vector | \n", + "label_int | \n", + "label | \n", + "
|---|---|---|---|---|---|---|
| 0 | \n", + "26813 | \n", + "Fot the most part, this movie feels like a \"ma... | \n", + "Fot the most part, this movie feels like a \"ma... | \n", + "[-0.022307869, -0.038372956, -0.005369567, -0.... | \n", + "0 | \n", + "Negative | \n", + "
| 1 | \n", + "26813 | \n", + "Fot the most part, this movie feels like a \"ma... | \n", + "more than some can handle just for that reason... | \n", + "[0.02946616, -0.024044147, -0.011064137, -0.03... | \n", + "0 | \n", + "Negative | \n", + "
| 2 | \n", + "26581 | \n", + "Are you kidding me? The music was SO LOUD in t... | \n", + "Are you kidding me? The music was SO LOUD in t... | \n", + "[-0.016822321, -0.030674767, -0.041740056, 0.0... | \n", + "0 | \n", + "Negative | \n", + "
| 3 | \n", + "26581 | \n", + "Are you kidding me? The music was SO LOUD in t... | \n", + "And what does a Kansas teen know about shoppin... | \n", + "[0.035922922, -0.06197654, 0.008055181, -0.025... | \n", + "0 | \n", + "Negative | \n", + "
| 4 | \n", + "40633 | \n", + "First of all, I don't understand why some peop... | \n", + "First of all, I don't understand why some peop... | \n", + "[-0.0035528215, -0.042889904, -0.04559665, 0.0... | \n", + "1 | \n", + "Positive | \n", + "
| \n", + " | movie_index | \n", + "text | \n", + "chunk | \n", + "vector | \n", + "label_int | \n", + "label | \n", + "
|---|---|---|---|---|---|---|
| 0 | \n", + "26813 | \n", + "Fot the most part, this movie feels like a \"ma... | \n", + "Fot the most part, this movie feels like a \"ma... | \n", + "[-0.022307869, -0.038372956, -0.005369567, -0.... | \n", + "0 | \n", + "Negative | \n", + "
| 1 | \n", + "26813 | \n", + "Fot the most part, this movie feels like a \"ma... | \n", + "more than some can handle just for that reason... | \n", + "[0.02946616, -0.024044147, -0.011064137, -0.03... | \n", + "0 | \n", + "Negative | \n", + "
| 2 | \n", + "26581 | \n", + "Are you kidding me? The music was SO LOUD in t... | \n", + "Are you kidding me? The music was SO LOUD in t... | \n", + "[-0.016822321, -0.030674767, -0.041740056, 0.0... | \n", + "0 | \n", + "Negative | \n", + "
| 3 | \n", + "26581 | \n", + "Are you kidding me? The music was SO LOUD in t... | \n", + "And what does a Kansas teen know about shoppin... | \n", + "[0.035922922, -0.06197654, 0.008055181, -0.025... | \n", + "0 | \n", + "Negative | \n", + "
| 4 | \n", + "40633 | \n", + "First of all, I don't understand why some peop... | \n", + "First of all, I don't understand why some peop... | \n", + "[-0.0035528215, -0.042889904, -0.04559665, 0.0... | \n", + "1 | \n", + "Positive | \n", + "