Azure Application Gateway (Standard V1) returns 502 while deleting instances in VM Scale Set

Question

While deleting instance in VM Scale Set, Application Gateway returns 502 for a brief few seconds.

I have made sure that:

• Connection draining on the HTTP setting used is enabled, timeout is set to 60s
• Cookie affinity is disabled
• No NSG blocking requests from Application Gateway

From my understanding, since connection draining is enabled, deregistering backend instances (in my case, VM instances in a VM scale set) should stop receiving new requests and allow currently in-process requests to finish within the draining timeout.

Also, cookie affinity is disabled so that Application Gateway won't route to the same instances every request, thus making sure the requests won't be routed to the deregistering instances.

Is my understanding or something about my configuration here is wrong?

This is the full Bicep template for my Application Gateway:

resource agw 'Microsoft.Network/applicationGateways@2020-11-01' = {
  name: 'agw-1'
  location: 'southeastasia'
  tags: {
    env: 'prod'
  }
  properties: {
    sku: {
      name: 'Standard_Medium'
      tier: 'Standard'
      capacity: 1
    }
    gatewayIPConfigurations: [
      {
        name: 'appGatewayIpConfig'
        properties: {
          subnet: {
            id: '${vnetId}/subnets/publicApplicationGateways'
          }
        }
      }
    ]
    sslCertificates: []
    authenticationCertificates: []
    frontendIPConfigurations: [
      {
        name: 'appGwPublicFrontendIp'
        properties: {
          privateIPAllocationMethod: 'Dynamic'
          publicIPAddress: {
            id: pipId
          }
        }
      }
      {
        name: 'appGwPrivateFrontendIp'
        properties: {
          privateIPAllocationMethod: 'Dynamic'
          subnet: {
            id: '${vnetId}/subnets/publicApplicationGateways'
          }
        }
      }
    ]
    frontendPorts: [
      {
        name: 'port_80'
        properties: {
          port: 80
        }
      }
      {
        name: 'port_443'
        properties: {
          port: 443
        }
      }
    ]
    backendAddressPools: [
      {
        name: 'webchat'
        properties: {
          backendAddresses: []
        }
      }
    ]
    backendHttpSettingsCollection: [
      {
        name: 'http'
        properties: {
          port: 80
          protocol: 'Http'
          cookieBasedAffinity: 'Disabled'
          connectionDraining: {
            enabled: true
            drainTimeoutInSec: 60
          }
          pickHostNameFromBackendAddress: false
          affinityCookieName: 'ApplicationGatewayAffinity'
          requestTimeout: 60
        }
      }
    ]
    httpListeners: [
      {
        name: 'http-webchat'
        properties: {
          frontendIPConfiguration: {
            id: '${agwId}/frontendIPConfigurations/appGwPublicFrontendIp'
          }
          frontendPort: {
            id: '${agwId}/frontendPorts/port_80'
          }
          protocol: 'Http'
          hostName: 'bagaswh.com'
          hostNames: []
          requireServerNameIndication: false
        }
      }
      {
        name: 'https-webchat'
        properties: {
          frontendIPConfiguration: {
            id: '${agwId}/frontendIPConfigurations/appGwPublicFrontendIp'
          }
          frontendPort: {
            id: '${agwId}/frontendPorts/port_443'
          }
          protocol: 'Https'
          sslCertificate: {
            id: '${agwId}/sslCertificates/bagaswh.com'
          }
          hostName: 'bagaswh.com'
          hostNames: []
          requireServerNameIndication: true
        }
      }
    ]
    urlPathMaps: []
    requestRoutingRules: [
      {
        name: 'http-public-webchat'
        properties: {
          ruleType: 'Basic'
          httpListener: {
            id: '${agwId}/httpListeners/http-webchat'
          }
          backendAddressPool: {
            id: '${agwId}/backendAddressPools/webchat'
          }
          backendHttpSettings: {
            id: '${agwId}/backendHttpSettingsCollection/http'
          }
        }
      }
      {
        name: 'https-public-webchat'
        properties: {
          ruleType: 'Basic'
          httpListener: {
            id: '${agwId}/httpListeners/https-webchat'
          }
          backendAddressPool: {
            id: '${agwId}/backendAddressPools/webchat'
          }
          backendHttpSettings: {
            id: '${agwId}/backendHttpSettingsCollection/http'
          }
        }
      }
    ]
    probes: [
      {
        name: 'http-webchat-healthProbe1'
        properties: {
          protocol: 'Http'
          host: 'bagaswh.com'
          path: '/'
          pickHostNameFromBackendHttpSettings: false
          interval: 10
          timeout: 60
          unhealthyThreshold: 3
        }
      }
    ]
    rewriteRuleSets: []
    redirectConfigurations: []
    enableHttp2: false
  }
}