Merge pull request #4 from Hekku2/feat/process-updates

Update documentation and improve infrastructure

.env-sample

@@ -1,3 +1,7 @@
+# This is just a sample file. You can use `Create-Settings.ps1` to generate
+# these files automatically. However, if the script doesn't work, use this
+# as the base .env file for `docker-compose.yml`
+# This file is NOT needed if docker is not used.
 DISCORD_TOKEN=
 DISCORD_GUILDID=
 DISCORD_CHANNELID=

Create-Environment.ps1

@@ -7,9 +7,13 @@
 
     .PARAMETER SettinsFile
     Settings file that contains environment settings. Defaults to 'developer-settings.json'
+
+    .PARAMETER NoDiscord
+    If set, discord sending is disabled. Useful for CI/CD and testing.
 #>
 param(
-    [Parameter()][string]$SettingsFile = 'developer-settings.json'
+    [Parameter()][string]$SettingsFile = 'developer-settings.json',
+    [Parameter()][switch]$NoDiscord
 )
 $ErrorActionPreference = "Stop"
 Set-StrictMode -Version Latest
@@ -20,7 +24,7 @@ Write-Host "Reading settings from file $SettingsFile"
 $settingsJson = Get-Content -Raw -Path $SettingsFile | ConvertFrom-Json
 
 Write-Host 'Checking if there is an existing installation...'
-$webSitePackageLocation = Get-AzWebApp -ResourceGroupName $settingsJson.ResourceGroup -Name "func-$($settingsJson.ApplicationName)" -ErrorAction SilentlyContinue | Get-WebSitePackage
+$webSitePackageLocation = Get-AzWebApp -ResourceGroupName $settingsJson.ResourceGroup -Name "func-$($settingsJson.ApplicationName)" -ErrorAction SilentlyContinue | Get-WebSitePackage -ErrorAction SilentlyContinue
 if ($webSitePackageLocation) {
     Write-Host "Function app already exist with website package, using it..."
 }
@@ -35,6 +39,7 @@ Write-Host 'Deploying template...'
 $parameters = @{
     baseName               = $settingsJson.ApplicationName
     webSitePackageLocation = $webSitePackageLocation
+    disableDiscordSending  = $NoDiscord ? $true : $false
     discordSettings        = @{
         token     = $settingsJson.DiscordToken
         channelId = $settingsJson.DiscordChannelId

Create-Settings.ps1

@@ -0,0 +1,42 @@
+<#
+    .SYNOPSIS
+    This script generates envvironmetn variable files based on developer-settings.json
+
+    .DESCRIPTION
+    Purpose of this is to make starting the development more convenient by generating environment files.
+
+    .PARAMETER SettinsFile
+    Settings file that contains environment settings. Defaults to 'developer-settings.json'
+#>
+param(
+    [Parameter()][string]$SettingsFile = 'developer-settings.json'
+)
+$ErrorActionPreference = "Stop"
+Set-StrictMode -Version Latest
+
+.$PSScriptRoot/scripts/FunctionUtil.ps1
+
+Write-Host "Reading settings from file $SettingsFile"
+$settingsJson = Get-Content -Raw -Path $SettingsFile | ConvertFrom-Json
+
+# Docker compose support
+$dockerEnvFile = "$PSScriptRoot/.env"
+$dockerEnvFileContent = "
+# This file was generated by Create-Settings.ps1. Don't commit this to version control.
+DISCORD_TOKEN=$($settingsJson.DiscordToken)
+DISCORD_GUILDID=$($settingsJson.DiscordGuildId)
+DISCORD_CHANNELID=$($settingsJson.DiscordChannelId)
+"
+Write-Host "Writing $dockerEnvFile"
+$dockerEnvFileContent | Out-File -FilePath $dockerEnvFile -Encoding utf8
+
+# Function Core Tools support
+$funcSettingsFile = "$PSScriptRoot/src/FunctionApp.Isolated/local.settings.json"
+$localSettings = @{
+    IsEncrypted = $false
+    Values      = @{
+        AzureWebJobsStorage      = 'UseDevelopmentStorage=true'
+        FUNCTIONS_WORKER_RUNTIME = 'dotnet-isolated'
+    }
+}
+$localSettings | ConvertTo-Json | Out-File -FilePath $funcSettingsFile -Encoding utf8

README.md

@@ -7,21 +7,41 @@ Currently this is designed to work with one server and one channel.
 
 ## How does it work?
 
-In short, software periodically selects random image from image storage and sends it to discord channel.
+In short, software periodically selects random image from image storage and
+sends it to discord channel.
 
 In depth
  1. Image sending is periodically triggered by Function App TimerTrigger.
- 1. When Image sending is triggered, software selects a random image from index. Selection is not really random, it prefers less posted images. See [RandomizationService](src/Common/RandomizationService/RandomizationService.cs) for implementation details.
-   * If there is no index, software builds an index of images available in source.
- 1. After image is selected, software downloads the image from storage and sends the image to the chosen Discord channel.
-   * If Imageis removed, error is logged and image is not sent to channel.
+ 1. When Image sending is triggered, software selects a random image from
+ index. Selection is not really random, it prefers less posted images. See
+ [RandomizationService](src/Common/RandomizationService/RandomizationService.cs)
+ for implementation details.
+   * If there is no index, software builds an index of images available in
+   source.
+ 1. After image is selected, software downloads the image from storage and
+ sends the image to the chosen Discord channel.
+   * If Image is removed, error is logged and image is not sent to channel.
 
 Indexing
  * Indexing is performed automatically if index doesn't exist.
  * Image index can be regenerated by calling the related function.
  * Currently changes in storage don't trigger image index regeneration.
- * The index contains data of the images and how often those have been posted and other similar metadata.
- * Images can also be ignored. Ignored images are not selected by randomization logic.
+ * The index contains data of the images and how often those have been posted
+ and other similar metadata.
+ * Images can also be ignored. Ignored images are not selected by randomization
+ logic.
+
+## Tools
+
+This section lists tools that are used in developing and deploying this
+software. Some are not strictly
+
+Development
+  * Dotnet 8.0 (or later)
+  * Azure Powershell
+    * Bicep with version that supports importing user types (0.28.1 is
+    enough, earlier may also suffice)
+  * Docker (and docker compose).
 
 ## Deployment and running
 

doc/technical-reasoning.md

@@ -0,0 +1,47 @@
+# Technical reasoning
+
+This document describes reasoning behind different architectural and technical
+decisions related to this project. Purpose of this is help with future
+refactoring because the technical landscape and the available Azure services
+may have changed from the time these decisions where made. This should also
+help choosing new implementations I have forgot to think about some aspect
+or I haven't known about some available tool etc.
+
+## Requirements
+
+This sections lists overall requirements.
+
+ 1. Main function of the software (Image sending) is atomatically activated
+    periodically.
+ 2. Software needs to be able to handle HTTP requests from Discord and from
+    users. These should be relatively rare.
+    * Discord MAY add a response time requirement.
+ 1. Software should prefer Managed Identity authentication.
+ 1. Software should be cheap to run and it shouldn't cost much when it's not
+    active.
+ 1. Secrets should be stored securely.
+ 1. Local development with containers should be supported.
+ 1. Infrastructure creation must be automatic with a single command.
+
+## Technical selection
+
+Azure Function App with dotnet-isolated runtime was chosen.
+
+ * Azure
+   * Can handle all of the requirements, but mainly because I'm most familiar
+   with this cloud service provider.
+ * Azure Function App
+   * Should be cheap to run.
+   * Supports HTTP requests
+   * Supports Managed Identity
+   * Can be activated periodically (`TimerTrigger`)
+   * Supports containers for local development.
+ * Isolated worker model
+   * Chosen instead of In-process model, because In-Process support is ending.
+ * Consumption mode for Function App
+   * Should be the cheapest option for this kind of workload.
+ * App Service plan
+   * Chosen instead of Container Apps, because the development experiment with
+   container app was still quite poor.
+ * Azure Key Vault
+   * Is a secure way to store secrets and integrates well with function app.

infra/functions.bicep

@@ -18,6 +18,9 @@ param location string = resourceGroup().location
 @description('Web site package location. Leave empty if none is found.')
 param webSitePackageLocation string = ''
 
+@description('If true, messages are not sent to Discord. This should only be used when testing.')
+param disableDiscordSending bool = false
+
 var hostingPlanName = 'asp-${baseName}'
 var functionAppName = 'func-${baseName}'
 var storageBlobDataOwnerRoleDefinitionId = 'b7e6dc6d-f1e8-4753-8033-0f276bb0955b'
@@ -86,7 +89,11 @@ resource functionApp 'Microsoft.Web/sites@2021-02-01' = {
       use32BitWorkerProcess: false
       ftpsState: 'FtpsOnly'
       minTlsVersion: '1.2'
-
+      cors: {
+        allowedOrigins: [
+          'https://portal.azure.com'
+        ]
+      }
       appSettings: [
         {
           name: 'AzureWebJobsStorage__accountName'
@@ -112,6 +119,10 @@ resource functionApp 'Microsoft.Web/sites@2021-02-01' = {
           name: 'APPINSIGHTS_INSTRUMENTATIONKEY'
           value: reference(applicationInsights.id, '2015-05-01').InstrumentationKey
         }
+        {
+          name: 'FeatureSettings__DisableDiscordSending'
+          value: '${disableDiscordSending}'
+        }
         {
           name: '${discordSettingsKey}__Token'
           value: discordSettings.token
@@ -125,24 +136,16 @@ resource functionApp 'Microsoft.Web/sites@2021-02-01' = {
           value: '${discordSettings.channelId}'
         }
         {
-          name: '${blobStorageKey}__ConnectionString'
-          value: imageStorageSettings.connectionString
-        }
-        {
-          name: '${blobStorageKey}__ContainerName'
-          value: imageStorageSettings.containerName
+          name: '${blobStorageKey}__BlobContainerUri'
+          value: imageStorageSettings.blobContainerUri
         }
         {
           name: '${blobStorageKey}__FolderPath'
           value: imageStorageSettings.folderPath
         }
         {
-          name: '${imageIndexStorageKey}__ConnectionString'
-          value: imageStorageSettings.connectionString
-        }
-        {
-          name: '${imageIndexStorageKey}__ContainerName'
-          value: 'index'
+          name: '${imageIndexStorageKey}__BlobContainerUri'
+          value: '${functionStorageAccount.properties.primaryEndpoints.blob}index'
         }
       ]
     }
@@ -163,3 +166,4 @@ resource functionAppFunctionBlobStorageAccess 'Microsoft.Authorization/roleAssig
 }
 
 output functionAppPrincipalId string = functionApp.identity.principalId
+output functionAppResourceId string = functionApp.id

infra/main.bicep

@@ -12,6 +12,9 @@ param location string = resourceGroup().location
 @description('Web site package location. Leave empty if none is found.')
 param webSitePackageLocation string = ''
 
+@description('If true, messages are not sent to Discord. This should only be used when testing.')
+param disableDiscordSending bool = false
+
 module appInsights 'app-insights.bicep' = {
   name: 'application-insights'
   params: {
@@ -39,9 +42,19 @@ resource imageStorage 'Microsoft.Storage/storageAccounts@2022-05-01' = {
   }
 }
 
+var imageContainerName = 'images'
+resource blobServices 'Microsoft.Storage/storageAccounts/blobServices@2023-01-01' = {
+  parent: imageStorage
+  name: 'default'
+}
+
+resource container 'Microsoft.Storage/storageAccounts/blobServices/containers@2023-01-01' = {
+  parent: blobServices
+  name: imageContainerName
+}
+
 var imageSettings = {
-  connectionString: 'DefaultEndpointsProtocol=https;AccountName=${imageStorage.name};EndpointSuffix=${az.environment().suffixes.storage};AccountKey=${imageStorage.listKeys().keys[0].value}'
-  containerName: 'images'
+  blobContainerUri: '${imageStorage.properties.primaryEndpoints.blob}${imageContainerName}'
   folderPath: 'root'
 }
 
@@ -54,5 +67,21 @@ module functions 'functions.bicep' = {
     discordSettings: discordSettings
     imageStorageSettings: imageSettings
     webSitePackageLocation: webSitePackageLocation
+    disableDiscordSending: disableDiscordSending
+  }
+}
+
+// TODO Also this assignment could probably be a separate module etc.
+var storageBlobDataReaderRoleDefinitionId = '2a2b9908-6ea1-4ae2-8e65-a410df84e7d1'
+resource functionAppFunctionBlobStorageAccess 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
+  scope: container
+  name: guid(functions.name, storageBlobDataReaderRoleDefinitionId, container.id)
+  properties: {
+    principalId: functions.outputs.functionAppPrincipalId
+    principalType: 'ServicePrincipal'
+    roleDefinitionId: subscriptionResourceId(
+      'Microsoft.Authorization/roleDefinitions',
+      storageBlobDataReaderRoleDefinitionId
+    )
   }
 }

infra/types.bicep

@@ -7,7 +7,6 @@ type DiscordSettings = {
 
 @export()
 type ImageStorageSettings = {
-  connectionString: string
-  containerName: string
+  blobContainerUri: string
   folderPath: string
 }

src/Common/BlobStorageImageService/BlobStorageImageSourceOptions.cs

@@ -1,4 +1,5 @@
 using System.ComponentModel.DataAnnotations;
+using Microsoft.Identity.Client;
 
 namespace DiscordImagePoster.Common.BlobStorageImageService;
 
@@ -16,9 +17,17 @@ public class BlobStorageImageSourceOptions
 
     /// <summary>
     /// The name of the container where the images are stored.
+    /// This is not needed (nor used) if BlobContainerUri is used.
     /// </summary>
-    [Required]
-    public required string ContainerName { get; set; }
+    public required string? ContainerName { get; set; }
+
+    /// <summary>
+    /// The URI to the container where the images are stored.
+    /// https://{account_name}.blob.core.windows.net/{container_name}
+    /// 
+    /// If this is used, the ConnectionString is not needed and managed identity is used
+    /// </summary>
+    public required string? BlobContainerUri { get; set; }
 
     /// <summary>
     /// The path to the folder where the images are stored.

src/Common/IndexService/BlobStorageIndexStorageService.cs

@@ -2,29 +2,32 @@
 using Azure.Storage.Blobs;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
 
 namespace DiscordImagePoster.Common.IndexService;
 
 public class BlobStorageIndexStorageService : IIndexStorageService
 {
-    private ILogger<BlobStorageIndexStorageService> _logger;
+    private readonly ILogger<BlobStorageIndexStorageService> _logger;
+    private readonly ImageIndexOptions _options;
     private readonly BlobContainerClient _blobContainerClient;
 
-    private const string IndexBlobName = "index.json";
-
     public BlobStorageIndexStorageService
     (
         ILogger<BlobStorageIndexStorageService> logger,
+        IOptions<ImageIndexOptions> options,
         [FromKeyedServices(KeyedServiceConstants.ImageIndexBlobContainerClient)] BlobContainerClient blobContainerClient
     )
     {
         _logger = logger;
+        _options = options.Value;
         _blobContainerClient = blobContainerClient;
     }
 
     public async Task<ImageIndex?> GetImageIndexAsync()
     {
-        var blobClient = _blobContainerClient.GetBlobClient(IndexBlobName);
+        _logger.LogTrace("Getting image index from {IndexFileName}", _options.IndexFileName);
+        var blobClient = _blobContainerClient.GetBlobClient(_options.IndexFileName);
         var exists = await blobClient.ExistsAsync();
         if (!exists)
         {
@@ -38,9 +41,10 @@ public BlobStorageIndexStorageService
 
     public async Task UpdateIndexAsync(ImageIndex index)
     {
+        _logger.LogTrace("Updating image index to {IndexFileName}", _options.IndexFileName);
         await _blobContainerClient.CreateIfNotExistsAsync();
         var bytes = JsonSerializer.SerializeToUtf8Bytes(index);
-        var blobClient = _blobContainerClient.GetBlobClient(IndexBlobName);
+        var blobClient = _blobContainerClient.GetBlobClient(_options.IndexFileName);
         await blobClient.UploadAsync(new MemoryStream(bytes), true);
     }
 }