Merge branch 'develop' v0.3.1

aws-samples · Aug 23, 2022 · 24c7bd1 · 24c7bd1
2 parents 00b43b6 + c4fae49
commit 24c7bd1
Show file tree

Hide file tree

Showing 7 changed files with 382 additions and 787 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.3.1] - 2022-08-23
+### Fixed
+- Support nested folders in bulk workflow
+
 ## [0.3.0] - 2022-08-12
 ### Added
 - Initial processing for Genesys CTR telephony files. See [Integration with Telephony CTR Files](./README.md#integration-with-telephony-ctr-files)
@@ -64,7 +68,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Initial release
 
-[Unreleased]: https://github.com/aws-samples/amazon-transcribe-post-call-analytics/compare/v0.3.0...develop
+[Unreleased]: https://github.com/aws-samples/amazon-transcribe-post-call-analytics/compare/v0.3.1...develop
+[0.3.1]: https://github.com/aws-samples/amazon-transcribe-post-call-analytics/releases/tag/v0.3.1
 [0.3.0]: https://github.com/aws-samples/amazon-transcribe-post-call-analytics/releases/tag/v0.3.0
 [0.2.5]: https://github.com/aws-samples/amazon-transcribe-post-call-analytics/releases/tag/v0.2.5
 [0.2.4]: https://github.com/aws-samples/amazon-transcribe-post-call-analytics/releases/tag/v0.2.4

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-0.3.0
+0.3.1
diff --git a/pca-main-nokendra.template b/pca-main-nokendra.template
@@ -1,6 +1,6 @@
 AWSTemplateFormatVersion: "2010-09-09"
 
-Description: Amazon Transcribe Post Call Analytics - PCA (v0.3.0) (uksb-1sn29lk73)
+Description: Amazon Transcribe Post Call Analytics - PCA (v0.3.1) (uksb-1sn29lk73)
 
 Parameters:
 

diff --git a/pca-main.template b/pca-main.template
@@ -1,6 +1,6 @@
 AWSTemplateFormatVersion: "2010-09-09"
 
-Description: Amazon Transcribe Post Call Analytics - PCA (v0.3.0) (uksb-1sn29lk73)
+Description: Amazon Transcribe Post Call Analytics - PCA (v0.3.1) (uksb-1sn29lk73)
 
 Parameters:
 

diff --git a/pca-server/src/pca/pca-aws-sf-bulk-files-count.py b/pca-server/src/pca/pca-aws-sf-bulk-files-count.py
@@ -11,7 +11,6 @@
 import copy
 import boto3
 
-
 def lambda_handler(event, context):
 
     # Get our params, looking them up if we haven't got them
@@ -39,9 +38,11 @@ def lambda_handler(event, context):
 
     # Just get a single S3 check on whether or not we have files to go
     s3Client = boto3.client('s3')
-    response = s3Client.list_objects_v2(Bucket=bucket, MaxKeys=dripRate)
+    maxKeys = dripRate + 10 # list a few additional keys to allow for some folder objects that won't be moved
+    response = s3Client.list_objects_v2(Bucket=bucket, MaxKeys=maxKeys)
     if "Contents" in response:
-        filesFound = len(response["Contents"])
+        files = [f for f in response["Contents"] if not f["Key"].endswith("/")][:dripRate] # ignore folder objects
+        filesFound = len(files)
     else:
         filesFound = 0
     sfData["filesToMove"] = filesFound

diff --git a/pca-server/src/pca/pca-aws-sf-bulk-move-files.py b/pca-server/src/pca/pca-aws-sf-bulk-move-files.py
@@ -24,16 +24,19 @@ def lambda_handler(event, context):
 
     # Get as many files from S3 as we can move this time (minimum of queueSpace and dripRate)
     s3Client = boto3.client('s3')
-    response = s3Client.list_objects_v2(Bucket=sourceBucket, MaxKeys=(min(dripRate, queueSpace)))
+    maxKeys = min(dripRate, queueSpace) + 10 # list a few additional keys to allow for some folder objects that won't be moved
+    response = s3Client.list_objects_v2(Bucket=sourceBucket, MaxKeys=maxKeys)
     if "Contents" in response:
         # We now have a list of objects that we can use
         sourcePrefix = "/" + sourceBucket + "/"
         keyPrefix = targetAudioKey
         if keyPrefix != "":
             keyPrefix += "/"
-        for audioFile in response["Contents"]:
+        files = [f for f in response["Contents"] if not f["Key"].endswith("/")][:dripRate] # ignore folder objects
+        for audioFile in files:
             try:
                 # Copy and delete file
+                print(f'Copying: Bucket={targetBucket}, CopySource={(sourcePrefix + audioFile["Key"])}, Key={(keyPrefix + audioFile["Key"])}')
                 copyResponse = s3Client.copy_object(Bucket=targetBucket,
                                                     CopySource=(sourcePrefix + audioFile["Key"]),
                                                     Key=(keyPrefix + audioFile["Key"]))
@@ -58,5 +61,5 @@ def lambda_handler(event, context):
         "dripRate": 50,
         "filesProcessed": 0,
         "queueSpace": 250
-    }
+    }   
     lambda_handler(event, "")