From 85d5701955d65d2dcfe704b4f2639dfa3958a729 Mon Sep 17 00:00:00 2001 From: James Jory Date: Fri, 10 Jun 2022 07:26:29 -0700 Subject: [PATCH] OpenSearch changes --- .../base/_template.yaml | 6 +- .../base/opensearch.yaml | 14 ++--- .../event-engine/base-workshop.yaml | 2 +- aws/cloudformation-templates/template.yaml | 6 +- .../opensearch-pre-index.py | 3 +- .../opensearch-pre-index/requirements.txt | 2 +- src/docker-compose.yml | 2 +- src/search/openapi/spec.yaml | 3 +- src/search/src/search-service/app.py | 2 +- workshop/0-StartHere/Search.ipynb | 58 ++++++++++--------- 10 files changed, 48 insertions(+), 50 deletions(-) diff --git a/aws/cloudformation-templates/base/_template.yaml b/aws/cloudformation-templates/base/_template.yaml index c123c11c6..a04bccf53 100644 --- a/aws/cloudformation-templates/base/_template.yaml +++ b/aws/cloudformation-templates/base/_template.yaml @@ -222,10 +222,10 @@ Resources: # Create role here rather than in OpenSearch nested template so role has time to become consistent OpenSearchServiceLinkedRole: Condition: CreateOpenSearchServiceLinkedRole - Type: 'AWS::IAM::ServiceLinkedRole' + Type: AWS::IAM::ServiceLinkedRole Properties: - AWSServiceName: es.amazonaws.com - Description: "Role for ES to access resources in VPC" + AWSServiceName: opensearchservice.amazonaws.com + Description: "Role for OpenSearch to access resources in VPC" OpenSearchRoleWaitHandle: Condition: CreateOpenSearchServiceLinkedRole diff --git a/aws/cloudformation-templates/base/opensearch.yaml b/aws/cloudformation-templates/base/opensearch.yaml index 348606862..58ba53ac5 100644 --- a/aws/cloudformation-templates/base/opensearch.yaml +++ b/aws/cloudformation-templates/base/opensearch.yaml @@ -21,20 +21,18 @@ Parameters: Resources: OpenSearchDomain: - Type: AWS::Elasticsearch::Domain + Type: AWS::OpenSearchService::Domain Properties: - ElasticsearchVersion: '7.4' - ElasticsearchClusterConfig: + EngineVersion: OpenSearch_1.2 + ClusterConfig: InstanceCount: 1 - InstanceType: t3.small.elasticsearch + InstanceType: t3.small.search ZoneAwarenessEnabled: false EBSOptions: EBSEnabled: true Iops: 0 VolumeSize: 10 - VolumeType: standard - SnapshotOptions: - AutomatedSnapshotStartHour: 0 + VolumeType: gp2 AccessPolicies: Version: 2012-10-17 Statement: @@ -58,7 +56,7 @@ Resources: SecurityGroup: Type: AWS::EC2::SecurityGroup Properties: - GroupDescription: Elasticsearch Security Group + GroupDescription: OpenSearch Security Group VpcId: !Ref VpcId SecurityGroupIngress: - FromPort: 443 diff --git a/aws/cloudformation-templates/event-engine/base-workshop.yaml b/aws/cloudformation-templates/event-engine/base-workshop.yaml index fb53b4e7a..8dadd9023 100644 --- a/aws/cloudformation-templates/event-engine/base-workshop.yaml +++ b/aws/cloudformation-templates/event-engine/base-workshop.yaml @@ -47,7 +47,7 @@ Resources: GitHubBranch: "master" # N/A SO DO NOT CHANGE GitHubToken: "dummy" # N/A SO DO NOT CHANGE GitHubUser: "aws-samples" # N/A SO DO NOT CHANGE - PreIndexElasticsearch: "No" # Change to 'Yes' if you don't want customers to have to do ES workshop + PreIndexOpenSearch: "No" # Change to 'Yes' if you don't want customers to have to do ES workshop PreCreatePersonalizeResources: "No" # Change to 'Yes' if you don't want customers to have to do Personalize workshop (requires 2.5 hours AFTER CFN deployment to complete) PreCreatePinpointWorkshop: "No" # Change to 'Yes' if you don't want customers to have to do Pinpoint workshop PinpointEmailFromAddress: "dummy@dummy.com" # Only change if you change PreCreatePinpointWorkshop to 'Yes' diff --git a/aws/cloudformation-templates/template.yaml b/aws/cloudformation-templates/template.yaml index d795fb851..68ed7a3d9 100644 --- a/aws/cloudformation-templates/template.yaml +++ b/aws/cloudformation-templates/template.yaml @@ -878,9 +878,9 @@ Outputs: AWS_XRAY_DAEMON_ADDRESS=xray:2000 # Search service - ES_SEARCH_DOMAIN_SCHEME=https - ES_SEARCH_DOMAIN_HOST="${Base.Outputs.ElasticsearchDomainEndpoint}" - ES_SEARCH_DOMAIN_PORT=9200 + OPENSEARCH_DOMAIN_SCHEME=https + OPENSEARCH_DOMAIN_HOST="${Base.Outputs.OpenSearchDomainEndpoint}" + OPENSEARCH_DOMAIN_PORT=9200 # Users service AWS_REGION=${AWS::Region} diff --git a/src/aws-lambda/opensearch-pre-index/opensearch-pre-index.py b/src/aws-lambda/opensearch-pre-index/opensearch-pre-index.py index 738befb73..3cdda7b40 100644 --- a/src/aws-lambda/opensearch-pre-index/opensearch-pre-index.py +++ b/src/aws-lambda/opensearch-pre-index/opensearch-pre-index.py @@ -1,12 +1,11 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: MIT-0 -import os import yaml import logging import boto3 from crhelper import CfnResource -from opensearch import OpenSearch +from opensearchpy import OpenSearch logger = logging.getLogger() logger.setLevel(logging.INFO) diff --git a/src/aws-lambda/opensearch-pre-index/requirements.txt b/src/aws-lambda/opensearch-pre-index/requirements.txt index 34bdb6408..9de7fc875 100644 --- a/src/aws-lambda/opensearch-pre-index/requirements.txt +++ b/src/aws-lambda/opensearch-pre-index/requirements.txt @@ -1,3 +1,3 @@ crhelper pyyaml -git+https://github.com/opensearch-project/opensearch-py.git#egg=opensearch \ No newline at end of file +opensearch-py==1.1.0 \ No newline at end of file diff --git a/src/docker-compose.yml b/src/docker-compose.yml index 7d8558e85..9d84bc3e4 100644 --- a/src/docker-compose.yml +++ b/src/docker-compose.yml @@ -81,7 +81,7 @@ services: # OpenSearch required for search service opensearch: - image: opensearchproject/opensearch:1.0.0 + image: opensearchproject/opensearch:1.3.3 container_name: opensearch environment: - plugins.security.disabled=true diff --git a/src/search/openapi/spec.yaml b/src/search/openapi/spec.yaml index e5abb2b58..fb5507e28 100644 --- a/src/search/openapi/spec.yaml +++ b/src/search/openapi/spec.yaml @@ -3,8 +3,7 @@ info: title: Search API version: v1 description: |- - The Search web service provides a RESTful API for retrieving product information based on a search term. The [Web UI](../../web-ui) makes calls to this service when a user performs a search. - Internally, this service makes calls to an [Elasticsearch](https://www.elastic.co/) cluster for search results. When deployed on AWS, [Amazon Elasticsearch Service](https://aws.amazon.com/elasticsearch-service/) is used. When deployed locally, a local Elasticsearch node is used for searches. + The Search web service provides a RESTful API for retrieving product information based on search criteria. OpenSearch is used as the underlying search engine. license: url: https://github.com/aws-samples/retail-demo-store/blob/master/LICENSE diff --git a/src/search/src/search-service/app.py b/src/search/src/search-service/app.py index e02593984..eafe9f9ec 100644 --- a/src/search/src/search-service/app.py +++ b/src/search/src/search-service/app.py @@ -10,7 +10,7 @@ from flask import Flask, jsonify from flask import request from flask_cors import CORS -from opensearch import OpenSearch, NotFoundError +from opensearchpy import OpenSearch, NotFoundError import json import os diff --git a/workshop/0-StartHere/Search.ipynb b/workshop/0-StartHere/Search.ipynb index e46f04540..cce73fe7e 100644 --- a/workshop/0-StartHere/Search.ipynb +++ b/workshop/0-StartHere/Search.ipynb @@ -6,7 +6,7 @@ "source": [ "# Retail Demo Store - Search Workshop\n", "\n", - "Welcome to the Retail Demo Store Search Workshop. In this module we'll be configuring the Retail Demo Store Search service to allow searching for product data via [Amazon OpenSearch Service](https://aws.amazon.com/opensearch-service/) (formerly Amazon Elasticsearch Service). An Amazon OpenSearch domain should already be provisioned for you in your AWS environment.\n", + "Welcome to the Retail Demo Store Search Workshop. In this module we'll be configuring the Retail Demo Store Search service to allow searching for product data via [Amazon OpenSearch Service](https://aws.amazon.com/opensearch-service/) (formerly Amazon Elasticsearch Service). An Amazon OpenSearch domain should already be provisioned for you in your AWS environment as part of the Retail Demo Store deployment.\n", "\n", "Recommended Time: 20 Minutes" ] @@ -24,7 +24,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Import Dependencies and Setup Boto3 Python Clients\n", + "### Import dependencies and setup Boto3 python clients\n", "\n", "Througout this workshop we will need access to some common libraries and clients for connecting to AWS services." ] @@ -92,14 +92,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Create Index and Bulk Index Product Data" + "## Create index and bulk index product data" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Get Products Service Instance\n", + "### Get Products Service instance\n", "\n", "We will be creating a new OpenSearch Index and indexing our product data so that our users can search for products. To do this, first we will be pulling our Product data from [Products Service](https://github.com/aws-samples/retail-demo-store/tree/master/src/products) that is deployed as part of the Retail Demo Store. To connect to the Products Service we will use Service Discovery to discover an instance of the Products Service, and then connect directly to that service instances to access our data." ] @@ -127,7 +127,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "#### Download and Explore the Products Dataset\n", + "#### Download and explore the Products dataset\n", "\n", "Now that we have the IP address of one of our Products Service instances, we can connect to it and fetch our product catalog. To more easily explore our data, we will convert the json response form the Products Service into a Pandas dataframe and print it as a table. " ] @@ -150,9 +150,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Install OpenSearch Python Library\n", + "### Install OpenSearch python library\n", "\n", - "We will use the Python OpenSearch library to connect to our Amazon OpenSearch cluster, create a new index, and then bulk index our product data. First, we need to install the OpenSearch library into our environment." + "We will use the Python OpenSearch library to connect to our Amazon OpenSearch cluster, create a new index, and then bulk index our product data. First, we need to install the OpenSearch library into the local notebook environment. We'll ensure pip is updated as well." ] }, { @@ -169,9 +169,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Find OpenSearch Domain Endpoint\n", + "### Discover OpenSearch domain endpoint\n", "\n", - "Before we can configure the OpenSearch client, we need to determine the endpoint for the OpenSearch domain created in your AWS environment. We will accomplish this by looking for the OpenSearch domain with tag key of `Name` and tag value of `retaildemostore`. This tag was associated with the Amazon OpenSearch domain that was created when the project was deployed to your AWS account." + "Before we can configure the OpenSearch client, we need to determine the endpoint for the OpenSearch domain created in your AWS environment. We will accomplish this by looking for the OpenSearch domain with tag key of `Name` and tag value of `retaildemostore`. This tag was associated with the Amazon OpenSearch domain that was created when the project was deployed to your AWS account using CloudFormation." ] }, { @@ -210,7 +210,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Configure and Create OpenSearch Python Client" + "### Configure and create OpenSearch python client" ] }, { @@ -219,27 +219,29 @@ "metadata": {}, "outputs": [], "source": [ - "from opensearch import OpenSearch\n", + "from opensearchpy import OpenSearch\n", "\n", "SEARCH_HOST = {\n", " 'host' : opensearch_domain_endpoint,\n", " 'port' : 443,\n", " 'scheme' : 'https',\n", "}\n", + "\n", + "client = OpenSearch(hosts = [SEARCH_HOST])\n", + "\n", + "# These variables will be used throughout the rest of the notebook\n", "INDEX_NAME = 'products'\n", "TYPE_NAME = 'product'\n", - "ID_FIELD = 'id'\n", - "\n", - "client = OpenSearch(hosts = [SEARCH_HOST])" + "ID_FIELD = 'id'" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Prepare Product Data for Indexing\n", + "### Prepare Product data for indexing\n", "\n", - "Batch products into chunks to avoid timeouts." + "Batch products into chunks that will be used for batch indexing below." ] }, { @@ -277,7 +279,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Check for and Delete Existing Indexes\n", + "### Check for and delete existing indexes\n", "\n", "If the products index already exists, we'll delete it so everything gets rebuilt from scratch." ] @@ -293,14 +295,14 @@ " res = client.indices.delete(index = INDEX_NAME)\n", " print(\" response: '%s'\" % (res))\n", "else:\n", - " print('Index does not exist. Nothing to do.')" + " print('Index does not exist. Nothing to delete.')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Create Index" + "### Create index" ] }, { @@ -324,7 +326,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Perform Bulk Indexing" + "### Perform bulk indexing" ] }, { @@ -344,9 +346,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Validate Results Through Elasticsearch\n", + "### Validate results through OpenSearch\n", "\n", - "To verify that the products have been successfully indexed, let's perform a wildcard search for `brush*` directly against the Elasticsearch index." + "To verify that the products have been successfully indexed, let's perform a wildcard search for `brush*` directly against the OpenSearch index." ] }, { @@ -363,9 +365,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Validate Results Through Search Service\n", + "## Validate results through Search Service\n", "\n", - "Finally, let's verify that the Retail Demo Store's [Search service](https://github.com/aws-samples/retail-demo-store/tree/master/src/search) can successfully query from the Elasticsearch index as well.\n", + "Finally, let's verify that the Retail Demo Store's [Search service](https://github.com/aws-samples/retail-demo-store/tree/master/src/search) can successfully query the the OpenSearch index as well.\n", "\n", "### Discover Search Service\n", "\n", @@ -413,7 +415,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Finally, let's do the same `brush` search through the Search service. We should get back the same item IDs as the direct Elasticsearch query above." + "Finally, let's do the same `brush` search through the Search service. We should get back the same item IDs as the direct OpenSearch query above." ] }, { @@ -429,16 +431,16 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Workshop Complete\n", + "## Workshop complete\n", "\n", - "**Congratulations!** You have completed the first Retail Demo Store workshop where we indexed the products from the Retail Demo Store's Products microservice in an Elasticsearch domain index. This domain is used by the Retail Demo Store's Search microservice to process search queries from the Web user interface. To see this in action, open the Retail Demo Store's web UI in a new browser tab/window and enter a value in the search field at the top of the page." + "**Congratulations!** You have completed the first Retail Demo Store workshop where we indexed the products from the Retail Demo Store's Products microservice in an OpenSearch domain index. This domain is used by the Retail Demo Store's Search microservice to process search queries from the Web user interface. To see this in action, open the Retail Demo Store's web UI in a new browser tab/window and enter a value in the search field at the top of the page." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Next Step\n", + "### Next step\n", "\n", "Move on to the **[1-Personalization](../1-Personalization/Lab-1-Introduction-and-data-preparation.ipynb)** workshop where we will learn how to train machine learning models using Amazon Personalize to produce personalized product recommendations to users and add the ability to provide personalized reranking of products." ]