Add durable functions for faster response time (#22)

* Add Durable Functions implementation * Fix docker compose
Hekku2 · Jul 14, 2024 · 16caf56 · 16caf56
1 parent e7a6e03
commit 16caf56
Show file tree

Hide file tree

Showing 12 changed files with 175 additions and 54 deletions.
diff --git a/Create-Settings.ps1 b/Create-Settings.ps1
@@ -45,10 +45,22 @@ Write-Host "Writing Function Core Toole support $funcSettingsFile"
 $localSettings = @{
     IsEncrypted = $false
     Values      = @{
-        AzureWebJobsStorage      = 'UseDevelopmentStorage=true'
-        FUNCTIONS_WORKER_RUNTIME = 'dotnet-isolated'
+        AzureWebJobsStorage                             = 'UseDevelopmentStorage=true'
+        FUNCTIONS_WORKER_RUNTIME                        = 'dotnet-isolated'
+        DiscordConfiguration__Token                     = $settingsJson.DiscordToken
+        DiscordConfiguration__GuildId                   = $settingsJson.DiscordGuildId
+        DiscordConfiguration__ChannelId                 = $settingsJson.DiscordChannelId
+        DiscordConfiguration__PublicKey                 = $settingsJson.DiscordPublicKey
+        ImageAnalysisConfiguration__Endpoint            = $cognitiveServicesEndpoint
+        BlobStorageImageSourceOptions__ConnectionString = 'UseDevelopmentStorage=true'
+        BlobStorageImageSourceOptions__ContainerName    = 'images'
+        BlobStorageImageSourceOptions__FolderPath       = 'root'
+        ImageIndexOptions__ConnectionString             = 'UseDevelopmentStorage=true'
+        ImageIndexOptions__ContainerName                = 'images'
+        ImageIndexOptions__IndexFileName                = 'index.json'
     }
 }
+
 $localSettings | ConvertTo-Json | Out-File -FilePath $funcSettingsFile -Encoding utf8
 
 Write-Host "Writing user-secrets for console tester."

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -9,8 +9,8 @@ services:
     ports:
       - 8080:80
     environment:
-      - ASPNETCORE_ENVIRONMENT=Development
-      - AzureWebJobsStorage=AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;DefaultEndpointsProtocol=https;BlobEndpoint=http://azurite:local.storage.emulator/devstoreaccount1;QueueEndpoint=http://local.storage.emulator:10001/devstoreaccount1;TableEndpoint=http://local.storage.emulator:10002/devstoreaccount1;
+      - WEBSITE_HOSTNAME=localhost:8080
+      - AzureWebJobsStorage=AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;DefaultEndpointsProtocol=https;BlobEndpoint=http://local.storage.emulator:10000/devstoreaccount1;QueueEndpoint=http://local.storage.emulator:10001/devstoreaccount1;TableEndpoint=http://local.storage.emulator:10002/devstoreaccount1;
       - AzureWebJobsSecretStorageType=files
       - DiscordConfiguration__Token=${DISCORD_TOKEN}
       - DiscordConfiguration__GuildId=${DISCORD_GUILDID}
@@ -31,7 +31,7 @@ services:
   local.storage.emulator:
     image: mcr.microsoft.com/azure-storage/azurite:3.30.0
     container_name: local.storage.emulator
-    command: azurite --loose --disableProductStyleUrl --blobHost 0.0.0.0 --blobPort 10000 --queueHost 0.0.0.0 --queuePort 10001 --location /workspace --debug /workspace/debug.log
+    command: azurite --loose --disableProductStyleUrl --blobHost 0.0.0.0 --blobPort 10000 --queueHost 0.0.0.0 --queuePort 10001 --tableHost 0.0.0.0 --tablePort 10002 --location /workspace --debug /workspace/debug.log
     ports:
       - 10000:10000
       - 10001:10001

diff --git a/infra/functions.bicep b/infra/functions.bicep
@@ -33,6 +33,8 @@ param cognitiveServicesAccountResourceGroup string
 var hostingPlanName = 'asp-${baseName}'
 var functionAppName = 'func-${baseName}'
 var storageBlobDataOwnerRoleDefinitionId = 'b7e6dc6d-f1e8-4753-8033-0f276bb0955b'
+var storageQueueDataContributorRoleDefinitionId = '974c5e8b-45b9-4653-ba55-5f855dd0fb88'
+var storageTableDataContributorRoleDefinitionId = '0a9a7e1f-b9d0-4cc4-a60d-0319b160aaa3'
 
 resource applicationInsights 'Microsoft.Insights/components@2020-02-02' existing = {
   name: applicationInsightsName
@@ -219,4 +221,30 @@ resource functionAppFunctionBlobStorageAccess 'Microsoft.Authorization/roleAssig
   }
 }
 
+resource functionAppFunctionQueueStorageAccess 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
+  scope: functionStorageAccount
+  name: guid(identity.id, storageQueueDataContributorRoleDefinitionId, functionStorageAccount.id)
+  properties: {
+    principalId: identity.properties.principalId
+    principalType: 'ServicePrincipal'
+    roleDefinitionId: subscriptionResourceId(
+      'Microsoft.Authorization/roleDefinitions',
+      storageQueueDataContributorRoleDefinitionId
+    )
+  }
+}
+
+resource functionAppFunctionTableStorageAccess 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
+  scope: functionStorageAccount
+  name: guid(identity.id, storageTableDataContributorRoleDefinitionId, functionStorageAccount.id)
+  properties: {
+    principalId: identity.properties.principalId
+    principalType: 'ServicePrincipal'
+    roleDefinitionId: subscriptionResourceId(
+      'Microsoft.Authorization/roleDefinitions',
+      storageTableDataContributorRoleDefinitionId
+    )
+  }
+}
+
 output functionAppResourceId string = functionApp.id
diff --git a/scripts/Get-ImageIndex.ps1 b/scripts/Get-ImageIndex.ps1
@@ -9,19 +9,39 @@
     Settings file that contains environment settings. Defaults to 'developer-settings.json'
 #>
 param(
-    [Parameter()][string]$SettingsFile = 'developer-settings.json'
+    [Parameter()][string]$SettingsFile = 'developer-settings.json',
+
+    [Parameter(ParameterSetName = 'Azure')][switch]
+    $UseAzure,
+
+    [Parameter(ParameterSetName = 'Docker')][switch]
+    $UseDocker
 )
 $ErrorActionPreference = "Stop"
 Set-StrictMode -Version Latest
 
 .$PSScriptRoot/FunctionUtil.ps1
 
 $settingsJson = Get-DeveloperSettings -SettingsFile $SettingsFile
-$appName = "func-$($settingsJson.ResourceGroup)"
 
-$webApp = Get-AzWebApp -ResourceGroupName $settingsJson.ResourceGroup -Name $appName
-$url = Get-FunctionBaseUrl -FunctionApp $webApp
-$code = Get-FunctionCode -FunctionApp $webApp
+if ($UseAzure) {
+    $appName = "func-$($settingsJson.ResourceGroup)"
+
+    $webApp = Get-AzWebApp -ResourceGroupName $settingsJson.ResourceGroup -Name $appName
+    $url = Get-FunctionBaseUrl -FunctionApp $webApp
+    $code = Get-FunctionCode -FunctionApp $webApp
+
+    $functionUrl = "$($url)GetImageIndex?code=$code"
+    Invoke-RestMethod -Method Get -Uri $functionUrl -ContentType 'application/json'
+}
+elseif ($UseDocker) {
+    # This could be read from dev_secrets
+    $url = 'http://localhost:8080/api/'
+    $code = 'mock-secret-for-local-testing'
+    $functionUrl = "$($url)GetImageIndex?code=$code"
+    Invoke-RestMethod -Method Get -Uri $functionUrl -ContentType 'application/json'
+}
+else {
+    Write-Error "Please specify either -UseAzure or -UseDocker"
+}
 
-$functionUrl = "$($url)GetImageIndex?code=$code"
-Invoke-RestMethod -Method Get -Uri $functionUrl -ContentType 'application/json'
diff --git a/src/FunctionApp.Isolated/Dockerfile b/src/FunctionApp.Isolated/Dockerfile
@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/dotnet/sdk:8.0-preview AS installer-env
+FROM mcr.microsoft.com/dotnet/sdk:8.0 AS installer-env
 
 COPY . /src/
 RUN cd /src/FunctionApp.Isolated && \

diff --git a/src/FunctionApp.Isolated/FunctionApp.Isolated.csproj b/src/FunctionApp.Isolated/FunctionApp.Isolated.csproj
@@ -12,9 +12,11 @@
     <PackageReference Include="Azure.Storage.Blobs" Version="12.20.0" />
     <PackageReference Include="Discord.Net.Rest" Version="3.15.2" />
     <PackageReference Include="Microsoft.Azure.Functions.Worker" Version="1.22.0" />
+    <PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.DurableTask" Version="1.1.4" />
     <PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http" Version="3.2.0" />
     <PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Timer" Version="4.3.1" />
-    <PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" Version="1.17.2" />
+    <PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" Version="1.17.4" OutputItemType="Analyzer" />
+    <PackageReference Include="Microsoft.DurableTask.Generators" Version="1.0.0-preview.1" OutputItemType="Analyzer" />
     <PackageReference Include="Microsoft.Extensions.Options.DataAnnotations" Version="8.0.0" />
   </ItemGroup>
 

diff --git a/src/FunctionApp.Isolated/Functions/DiscordWebhookFunction.cs b/src/FunctionApp.Isolated/Functions/DiscordWebhookFunction.cs
@@ -3,8 +3,10 @@
 using DiscordImagePoster.Common.Discord;
 using DiscordImagePoster.Common.RandomizationService;
 using DiscordImagePoster.FunctionApp.Isolated.DiscordDto;
+using DiscordImagePoster.FunctionApp.Isolated.Functions;
 using Microsoft.Azure.Functions.Worker;
 using Microsoft.Azure.Functions.Worker.Http;
+using Microsoft.DurableTask.Client;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Newtonsoft.Json;
@@ -15,21 +17,22 @@ public class DiscordWebhookFunction
 {
     private readonly ILogger<DiscordWebhookFunction> _logger;
     private readonly string _publicKey;
-    private readonly IRandomImagePoster _randomImagePoster;
 
     public DiscordWebhookFunction(
         ILogger<DiscordWebhookFunction> logger,
-        IOptions<DiscordConfiguration> options,
-        IRandomImagePoster randomImagePoster
+        IOptions<DiscordConfiguration> options
     )
     {
         _logger = logger;
         _publicKey = options.Value.PublicKey;
-        _randomImagePoster = randomImagePoster;
     }
 
     [Function("HandleDiscordWebHook")]
-    public async Task<HttpResponseData> HandleWebhook([HttpTrigger(AuthorizationLevel.Function, "post")] HttpRequestData req)
+    public async Task<HttpResponseData> HandleWebhook(
+        [HttpTrigger(AuthorizationLevel.Function, "post")] HttpRequestData req,
+        [DurableClient] DurableTaskClient client,
+        CancellationToken cancellation
+        )
     {
 
         _logger.LogInformation("Handling webhook.");
@@ -47,8 +50,7 @@ public async Task<HttpResponseData> HandleWebhook([HttpTrigger(AuthorizationLeve
         }
 
         var stringBody = await req.ReadAsStringAsync() ?? string.Empty;
-        var client = new DiscordRestClient();
-        var isValid = client.IsValidHttpInteraction(_publicKey, signature, timestamp, stringBody);
+        var isValid = new DiscordRestClient().IsValidHttpInteraction(_publicKey, signature, timestamp, stringBody);
         if (!isValid)
         {
             _logger.LogError("Invalid signature.");
@@ -66,7 +68,7 @@ public async Task<HttpResponseData> HandleWebhook([HttpTrigger(AuthorizationLeve
                 _logger.LogInformation("Handle command");
                 if (response?.Data?.Name == "post-random-image")
                 {
-                    await _randomImagePoster.PostRandomImageAsync();
+                    await client.ScheduleNewOrchestrationInstanceAsync(nameof(ImageSendOrchestration), "", cancellation);
                 }
                 return await CreateCommandResponse(req);
             default:

diff --git a/src/FunctionApp.Isolated/Functions/ImageSendFunction.cs b/src/FunctionApp.Isolated/Functions/ImageSendFunction.cs
@@ -1,7 +1,8 @@
 using System.Net;
-using DiscordImagePoster.Common.RandomizationService;
+using DiscordImagePoster.FunctionApp.Isolated.Functions;
 using Microsoft.Azure.Functions.Worker;
 using Microsoft.Azure.Functions.Worker.Http;
+using Microsoft.DurableTask.Client;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
@@ -11,31 +12,34 @@ public class ImageSendFunction
 {
     private readonly ILogger<ImageSendFunction> _logger;
     private readonly FeatureSettings _featureSettings;
-    private readonly IRandomImagePoster _randomImagePoster;
 
     public ImageSendFunction(
         ILogger<ImageSendFunction> logger,
-        IOptions<FeatureSettings> featureSettings,
-        IRandomImagePoster randomImagePoster
+        IOptions<FeatureSettings> featureSettings
     )
     {
         _logger = logger;
         _featureSettings = featureSettings.Value;
-        _randomImagePoster = randomImagePoster;
     }
 
     [Function("SendImage")]
-    public async Task<HttpResponseData> SendRandomImage([HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req)
+    public async Task<HttpResponseData> SendRandomImage(
+        [HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req,
+        [DurableClient] DurableTaskClient client,
+        CancellationToken cancellation)
     {
         _logger.LogInformation("Sending random image triggered manually.");
 
-        await _randomImagePoster.PostRandomImageAsync();
+        await client.ScheduleNewOrchestrationInstanceAsync(nameof(ImageSendOrchestration), "", cancellation);
 
         return req.CreateResponse(HttpStatusCode.Accepted);
     }
 
     [Function("SendRandomImage")]
-    public async Task TriggerTimerSendRandomImage([TimerTrigger("0 0 */4 * * *")] TimerInfo timer)
+    public async Task TriggerTimerSendRandomImage(
+        [TimerTrigger("0 0 */4 * * *")] TimerInfo timer,
+        [DurableClient] DurableTaskClient client,
+        CancellationToken cancellation)
     {
         _logger.LogDebug("Sending timed random image");
         if (_featureSettings.DisableTimedSending)
@@ -44,7 +48,7 @@ public async Task TriggerTimerSendRandomImage([TimerTrigger("0 0 */4 * * *")] Ti
             return;
         }
 
-        await _randomImagePoster.PostRandomImageAsync();
+        await client.ScheduleNewOrchestrationInstanceAsync(nameof(ImageSendOrchestration), "", cancellation);
 
         if (timer.ScheduleStatus is not null)
         {

diff --git a/src/FunctionApp.Isolated/Functions/ImageSendOrchestration.cs b/src/FunctionApp.Isolated/Functions/ImageSendOrchestration.cs
@@ -0,0 +1,19 @@
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.DurableTask;
+using Microsoft.Extensions.Logging;
+
+namespace DiscordImagePoster.FunctionApp.Isolated.Functions;
+
+public static class ImageSendOrchestration
+{
+
+    [Function(nameof(ImageSendOrchestration))]
+    public static async Task<string> RunAsync([OrchestrationTrigger] TaskOrchestrationContext context, string input)
+    {
+        var logger = context.CreateReplaySafeLogger(nameof(ImageSendOrchestration));
+        logger.LogInformation("Starting image send orchestration.");
+
+        return await context.CallActivityAsync<string>(nameof(SendRandomImageActivity), input);
+
+    }
+}
diff --git a/src/FunctionApp.Isolated/Functions/SendRandomImageActivity.cs b/src/FunctionApp.Isolated/Functions/SendRandomImageActivity.cs
@@ -0,0 +1,23 @@
+using DiscordImagePoster.Common.RandomizationService;
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.DurableTask;
+using Microsoft.Extensions.Logging;
+
+public class SendRandomImageActivity
+{
+    private readonly ILogger<SendRandomImageActivity> _logger;
+    private readonly IRandomImagePoster _randomImagePoster;
+
+    public SendRandomImageActivity(ILogger<SendRandomImageActivity> logger, IRandomImagePoster randomImagePoster)
+    {
+        _logger = logger;
+        _randomImagePoster = randomImagePoster;
+    }
+
+    [Function(nameof(SendRandomImageActivity))]
+    public async Task RunAsync([ActivityTrigger] string context)
+    {
+        _logger.LogInformation("Sending random image triggered by orchestration.");
+        await _randomImagePoster.PostRandomImageAsync();
+    }
+}
diff --git a/src/FunctionApp.Isolated/dev_secrets/host.json b/src/FunctionApp.Isolated/dev_secrets/host.json
@@ -1,14 +1,25 @@
-{
-    "masterKey": {
-        "name": "master",
-        "value": "mock-secret-for-local-testing",
-        "encrypted": false
-    },
-    "functionKeys": [
-        {
-            "name": "default",
-            "value": "mock-secret-for-local-testing",
-            "encrypted": false
-        }
-    ]
+{
+  "masterKey": {
+    "name": "master",
+    "value": "mock-secret-for-local-testing",
+    "encrypted": false
+  },
+  "functionKeys": [
+    {
+      "name": "default",
+      "value": "mock-secret-for-local-testing",
+      "encrypted": false
+    }
+  ],
+  "systemKeys": [
+    {
+      "name": "durabletask_extension",
+      "value": "mock-secret-for-local-testing",
+      "encrypted": false
+    }
+  ],
+  "hostName": "localhost:8080",
+  "instanceId": "0000000000000000000000008156EC52",
+  "source": "runtime",
+  "decryptionKeyId": ""
 }
diff --git a/src/FunctionApp.Isolated/host.json b/src/FunctionApp.Isolated/host.json
@@ -1,12 +1,12 @@
 {
-    "version": "2.0",
-    "logging": {
-        "applicationInsights": {
-            "samplingSettings": {
-                "isEnabled": true,
-                "excludedTypes": "Request"
-            },
-            "enableLiveMetricsFilters": true
-        }
+  "version": "2.0",
+  "logging": {
+    "applicationInsights": {
+      "samplingSettings": {
+        "isEnabled": true,
+        "excludedTypes": "Request"
+      },
+      "enableLiveMetricsFilters": true
     }
-}
+  }
+}