C# HttpListener reuse address?

Question

So I've been searching around the web and wasn't able to find out how to configure an .NET HttpListener instance, so it can re-use the local port number (SocketOption: ReuseAddress); otherwise, if I restart HttpListener on the same port as its last incarnation, I often get the following exception:

System.Net.HttpListenerException (0x80004005): Address already in use
   at System.Net.HttpEndPointManager.GetEPListener(String host, Int32 port, HttpListener listener, Boolean secure)
   at System.Net.HttpEndPointManager.AddPrefixInternal(String p, HttpListener listener)
   at System.Net.HttpEndPointManager.AddListener(HttpListener listener)
   at System.Net.HttpListener.Start()

Any ideas about is it possible to customize the behavior for HttpLisener?

EDIT: the following is the test program running under the dotnet core 2.0 (C# 7.1) environment, and to answer a comment below - no, there is no dispose happening yet when the exception occurs - the exception occurs during HttpListener.Start()

using System;
using System.IO;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;

namespace Test
{
    class Program
    {
        static async Task Main(string[] args)
        {
            var server = new AsyncHttpServer(portNumber: 1234);
            await server.Run();
        }

        class AsyncHttpServer
        {
            readonly HttpListener listener;
            const string RESPONSE_TEMPLATE =
                "{{\"RawUrl\":\"{0}\", \"i\":{1}}}";

            public AsyncHttpServer(int portNumber)
            {
                listener = new HttpListener();
                listener.Prefixes.Add(string.Format("http://+:{0}/", portNumber));
            }

            int i = 0;

            async void HandleRequest(HttpListenerContext context)
            {
                Console.WriteLine($"HandleRequest: {context.Request.Url}, {context.Request.RawUrl}, {context.Request.HttpMethod}");

                await Task.Delay(TimeSpan.FromSeconds(1));

                var response = context.Request.RawUrl == "/" ? string.Format(RESPONSE_TEMPLATE, context.Request.Url.ToString(), ++i) : "";

                using (var sw = new StreamWriter(context.Response.OutputStream))
                {
                    await sw.WriteAsync(response);
                    await sw.FlushAsync();
                }
            }

            public async Task Run()
            {
                try
                {
                    listener.Start();

                    while (true)
                    {
                        var ctx = await listener.GetContextAsync();
                        HandleRequest(ctx);
                    }
                }
                catch (Exception e)
                {
                    Console.WriteLine(e);
                }
            }
        }
    }
}