add integration tests for lookups using metadata filtering

.gitignore

@@ -55,3 +55,5 @@ docs/_build/
 # PyBuilder
 target/
 image_match/web/static/tmp/
+
+*.jpg

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:2.7
+FROM python:3.5
 
 RUN apt-get update && apt-get install -y libblas-dev liblapack-dev gfortran
 

docker-compose.yml

@@ -1,22 +1,43 @@
 version: '2'
 
 services:
+
   image-match:
-    build: .
+    build: 
+      context: .
+    image: ascribe/image-match
+    container_name: image-match
     volumes:
       - .:/usr/src/app/
-    links:
-      - elasticsearch
+    depends_on:
+     - "elasticsearch"
+
   elasticsearch:
-    image: elasticsearch
+    image: docker.elastic.co/elasticsearch/elasticsearch:5.2.2
+    container_name: elasticsearch
+    environment:
+      - xpack.security.enabled=false
+      - xpack.monitoring.enabled=false
+      - http.host=0.0.0.0
+      - transport.host=127.0.0.1
+      - cluster.name=docker-cluster
+      - bootstrap.memory_lock=true
+      - ES_JAVA_OPTS=-Xms512m -Xmx512m
+    ports:
+      - "9200:9200"
+
   bdocs:
-    build: .
+    build: 
+      context: .
+    image: ascribe/image-match
+    container_name: bdocs
     volumes:
       - .:/usr/src/app/
     working_dir: /usr/src/app/docs
     command: make html
+
   vdocs:
-    image: nginx
+    image: nginx:alpine
     ports:
       - '40080:80'
     volumes:

image_match/elasticsearch_driver.py

@@ -62,7 +62,7 @@ def search_single_record(self, rec, pre_filter=None):
         }
 
         if pre_filter is not None:
-            body['query']['filter'] = pre_filter
+            body['query']['bool']['filter'] = pre_filter
 
         res = self.es.search(index=self.index,
                               doc_type=self.doc_type,

pytest.ini

@@ -1,3 +1,4 @@
 [pytest]
 testpaths = tests
+; python_functions = test_lookup_with_filter_by_metadata
 norecursedirs = .* *.egg *.egg-info env* devenv* docs

tests/test_elasticsearch_driver.py

@@ -14,6 +14,25 @@
 urllib.request.urlretrieve(test_img_url2, 'test2.jpg')
 
 INDEX_NAME = 'test_environment_{}'.format(hashlib.md5(os.urandom(128)).hexdigest()[:12])
+DOC_TYPE = 'image'
+MAPPINGS = {
+  "mappings": {
+    DOC_TYPE: { 
+      "dynamic": True,
+      "properties": { 
+        "metadata": { 
+            "type": "object",
+            "dynamic": True,
+            "properties": { 
+                "tenant_id": { "type": "keyword" }
+            } 
+        }
+      }
+    }
+  }
+}
+
+
 @pytest.fixture(scope='module', autouse=True)
 def index_name():
     return INDEX_NAME
@@ -22,7 +41,7 @@ def index_name():
 def setup_index(request, index_name):
     es = Elasticsearch()
     try:
-        es.indices.create(index=index_name)
+        es.indices.create(index=index_name, body=MAPPINGS)
     except RequestError as e:
         if e.error == u'index_already_exists_exception':
             es.indices.delete(index_name)
@@ -52,7 +71,7 @@ def es():
 
 @pytest.fixture
 def ses(es, index_name):
-    return SignatureES(es=es, index=index_name)
+    return SignatureES(es=es, index=index_name, doc_type=DOC_TYPE)
 
 def test_elasticsearch_running(es):
     i = 0
@@ -156,6 +175,29 @@ def test_add_image_with_metadata(ses):
     assert 'id' in r[0]
 
 
+def test_lookup_with_filter_by_metadata(ses):
+    metadata = dict(
+            tenant_id='foo'
+    )
+    ses.add_image('test1.jpg', metadata=metadata, refresh_after=True)
+
+    metadata2 = dict(
+            tenant_id='bar-2'
+    )
+    ses.add_image('test2.jpg', metadata=metadata2, refresh_after=True)
+
+    r = ses.search_image('test1.jpg', pre_filter={"term": {"metadata.tenant_id": "foo"}})
+    assert len(r) == 1
+    assert r[0]['metadata'] == metadata
+
+    r = ses.search_image('test1.jpg', pre_filter={"term": {"metadata.tenant_id": "bar-2"}})
+    assert len(r) == 1
+    assert r[0]['metadata'] == metadata2
+
+    r = ses.search_image('test1.jpg', pre_filter={"term": {"metadata.tenant_id": "bar-3"}})
+    assert len(r) == 0
+
+
 def test_all_orientations(ses):
     im = Image.open('test1.jpg')
     im.rotate(90, expand=True).save('rotated_test1.jpg')

tests/test_elasticsearch_driver_metadata_as_nested.py

@@ -0,0 +1,134 @@
+import pytest
+import urllib.request
+import os
+import hashlib
+from elasticsearch import Elasticsearch, ConnectionError, RequestError, NotFoundError
+from time import sleep
+
+from image_match.elasticsearch_driver import SignatureES
+from PIL import Image
+
+test_img_url1 = 'https://camo.githubusercontent.com/810bdde0a88bc3f8ce70c5d85d8537c37f707abe/68747470733a2f2f75706c6f61642e77696b696d656469612e6f72672f77696b6970656469612f636f6d6d6f6e732f7468756d622f652f65632f4d6f6e615f4c6973612c5f62795f4c656f6e6172646f5f64615f56696e63692c5f66726f6d5f4332524d465f7265746f75636865642e6a70672f36383770782d4d6f6e615f4c6973612c5f62795f4c656f6e6172646f5f64615f56696e63692c5f66726f6d5f4332524d465f7265746f75636865642e6a7067'
+test_img_url2 = 'https://camo.githubusercontent.com/826e23bc3eca041110a5af467671b012606aa406/68747470733a2f2f63322e737461746963666c69636b722e636f6d2f382f373135382f363831343434343939315f303864383264653537655f7a2e6a7067'
+urllib.request.urlretrieve(test_img_url1, 'test1.jpg')
+urllib.request.urlretrieve(test_img_url2, 'test2.jpg')
+
+INDEX_NAME = 'test_environment_{}'.format(hashlib.md5(os.urandom(128)).hexdigest()[:12])
+DOC_TYPE = 'image'
+MAPPINGS = {
+  "mappings": {
+    DOC_TYPE: { 
+      "dynamic": True,
+      "properties": { 
+        "metadata": { 
+            "type": "nested",
+            "dynamic": True,
+            "properties": { 
+                "tenant_id": { "type": "keyword" },
+                "project_id": { "type": "keyword" }
+            } 
+        }
+      }
+    }
+  }
+}
+
+
+@pytest.fixture(scope='module', autouse=True)
+def index_name():
+    return INDEX_NAME
+
+@pytest.fixture(scope='function', autouse=True)
+def setup_index(request, index_name):
+    es = Elasticsearch()
+    try:
+        es.indices.create(index=index_name, body=MAPPINGS)
+    except RequestError as e:
+        if e.error == u'index_already_exists_exception':
+            es.indices.delete(index_name)
+        else:
+            raise
+
+    def fin():
+        try:
+            es.indices.delete(index_name)
+        except NotFoundError:
+            pass
+
+    request.addfinalizer(fin)
+
+@pytest.fixture(scope='function', autouse=True)
+def cleanup_index(request, es, index_name):
+    def fin():
+        try:
+            es.indices.delete(index_name)
+        except NotFoundError:
+            pass
+    request.addfinalizer(fin)
+
+@pytest.fixture
+def es():
+    return Elasticsearch()
+
+@pytest.fixture
+def ses(es, index_name):
+    return SignatureES(es=es, index=index_name, doc_type=DOC_TYPE)
+
+def test_elasticsearch_running(es):
+    i = 0
+    while i < 5:
+        try:
+            es.ping()
+            assert True
+            return
+        except ConnectionError:
+            i += 1
+            sleep(2)
+
+    pytest.fail('Elasticsearch not running (failed to connect after {} tries)'
+                .format(str(i)))
+
+
+def test_lookup_with_filter_by_metadata(ses):
+
+    ses.add_image('test1.jpg', metadata=_metadata('foo', 'project-x'), refresh_after=True)
+    ses.add_image('test2.jpg', metadata=_metadata('foo', 'project-x'), refresh_after=True)
+    ses.add_image('test3.jpg', img='test1.jpg', metadata=_metadata('foo', 'project-y'), refresh_after=True)
+
+    ses.add_image('test2.jpg', metadata=_metadata('bar', 'project-x'), refresh_after=True)
+
+    r = ses.search_image('test1.jpg', pre_filter=_nested_filter('foo', 'project-x'))
+    assert len(r) == 2
+
+    r = ses.search_image('test1.jpg', pre_filter=_nested_filter('foo', 'project-z'))
+    assert len(r) == 0  
+
+    r = ses.search_image('test1.jpg', pre_filter=_nested_filter('bar', 'project-x'))
+    assert len(r) == 1
+
+    r = ses.search_image('test1.jpg', pre_filter=_nested_filter('bar-2', 'project-x'))
+    assert len(r) == 0
+
+    r = ses.search_image('test1.jpg', pre_filter=_nested_filter('bar', 'project-z'))
+    assert len(r) == 0    
+
+def _metadata(tenant_id, project_id):
+    return dict(
+            tenant_id=tenant_id,
+            project_id=project_id
+    )
+
+def _nested_filter(tenant_id, project_id):
+    return {
+        "nested" : {
+            "path" : "metadata",
+            "query" : {
+                "bool" : {
+                    "must" : [
+                        {"term": {"metadata.tenant_id": tenant_id}},
+                        {"term": {"metadata.project_id": project_id}}
+                    ]
+                }
+             }            
+        }
+    }