// SignalR Hub
public class CrawlHub : Hub
{
    public async IAsyncEnumerable<UIMessage> Crawl(string url, [EnumeratorCancellation]CancellationToken cancellationToken)
    {
        Log.Information("Here");
        // Trying to catch this error further up pipeline as
        // can't try catch here due to yield return
        throw new HubException("This error will be sent to the client!");
        // handing off to Crawler which returns back messages (UIMessage objects) every now and again on progress
        await foreach (var uiMessage in Crawler.Crawl(url, cancellationToken))
        {
// Check the cancellation token regularly so that the server will stop
            // producing items if the client disconnects.
            cancellationToken.ThrowIfCancellationRequested()
            // update the stream UI with whatever is happening in static Crawl
            yield return new UIMessage(uiMessage.Message, uiMessage.Hyperlink, uiMessage.NewLine);
        }
    }
}

2019-11-24 08:35:48.636 +00:00 [INF]
2019-11-24 08:35:48.682 +00:00 [INF] Starting up BLC.Website (Program.cs)
2019-11-24 08:35:48.917 +00:00 [INF] Development environment - using developer exception page
2019-11-24 08:35:48.995 +00:00 [INF] Application started. Press Ctrl+C to shut down.
2019-11-24 08:35:48.997 +00:00 [INF] Hosting environment: Development
2019-11-24 08:35:48.998 +00:00 [INF] Content root path: c:\\dev\\test\\BrokenLink\\BLC.Website
2019-11-24 08:35:49.138 +00:00 [INF] HTTP GET / responded 200 in 125.315 ms
2019-11-24 08:35:54.652 +00:00 [INF] HTTP GET /scan?urlToCrawl=davemateer.com responded 200 in 34.0029 ms
2019-11-24 08:35:54.820 +00:00 [INF] HTTP POST /crawlHub/negotiate responded 200 in 11.954 ms
2019-11-24 08:35:54.947 +00:00 [INF] Here

public class Program
{
    public static void Main(string[] args)
    {
        Log.Logger = new LoggerConfiguration()
            //.MinimumLevel.Information() // this is the default
            // Suppress framework log noise eg routing and handling
            // so we'll see warnings and errors from the framework
            .MinimumLevel.Override("Microsoft.AspNetCore", LogEventLevel.Warning)
            .Enrich.FromLogContext()
            .WriteTo.Console()
            .WriteTo.File("log.txt", rollingInterval: RollingInterval.Day)
            .CreateLogger();

        try
        {
            Log.Information("");
            Log.Information("Starting up BLC.Website (Program.cs)");
            CreateHostBuilder(args).Build().Run();
        }
        catch (Exception ex)
        {
            Log.Fatal(ex,"Application start-up failed");
        }
        finally
        {
            Log.CloseAndFlush();
        }
    }

    public static IHostBuilder CreateHostBuilder(string[] args) =>
        Host.CreateDefaultBuilder(args)
            .UseSerilog()
            // configuring logging for SignalR
            .ConfigureLogging(logging =>
            {
                logging.AddFilter("Microsoft.AspNetCore.SignalR", LogLevel.Warning);
                // turn on for connection debugging
                //logging.AddFilter("Microsoft.AspNetCore.Http.Connections", LogLevel.Debug);
            })
            .ConfigureWebHostDefaults(webBuilder =>
            {
                webBuilder.UseStartup<Startup>();
            });
}

// This method gets called by the runtime. Use this method to add services to the container.
public void ConfigureServices(IServiceCollection services, IWebHostEnvironment env)
{
    services.AddRazorPages();
    // send errors to the client

    services.AddSignalR(options =>
    {
        if (env.IsDevelopment())
        {
            options.EnableDetailedErrors = true;
        }
    });
}

public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
{
    if (env.IsDevelopment())
    {
        Log.Information("Development environment - using developer exception page");
        app.UseDeveloperExceptionPage();
    }
    else
    {
        Log.Information("Non Development environment - errors go to /Error");
        app.UseExceptionHandler("/Error");
        // The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts.
        app.UseHsts();
    }

    app.UseHttpsRedirection();
    app.UseStaticFiles();

    // don't want request logging for static files so put it here in the pipeline
    app.UseSerilogRequestLogging();

    app.UseRouting();

    app.UseAuthorization();

    app.UseEndpoints(endpoints =>
    {
        endpoints.MapRazorPages();
        endpoints.MapHub<CrawlHub>("/crawlHub");
    });
}

// SignalR Hub
public class CrawlHub : Hub
{
    // A wrapper iterator so can catch exceptions here which can't be done in
    // a block which does yield return
    public async IAsyncEnumerable<UIMessage> Crawl(string url, [EnumeratorCancellation] CancellationToken cancellationToken)
    {
        var enumerable = CrawlAndGetMessages(url, cancellationToken);
        await using var enumerator = enumerable.GetAsyncEnumerator(cancellationToken);
        for (var more = true; more;)
        {
            // Catch exceptions only on executing/resuming the iterator function
            try
            {
                more = await enumerator.MoveNextAsync();
            }
            catch (Exception ex)
            {
                Log.Fatal("IteratorFunction() threw exception:" + ex);
                throw;
            }
            // yield out this UIMessage this has to be outside a try catch block
            yield return enumerator.Current;
        }
    }

    public async IAsyncEnumerable<UIMessage> CrawlAndGetMessages(string url, [EnumeratorCancellation]CancellationToken cancellationToken)
    {
        // handing off to Crawler which returns back messages (UIMessage objects) every now and again on progress
        await foreach (var uiMessage in Crawler.Crawl(url, cancellationToken))
        {
            // Check the cancellation token regularly so that the server will stop
            // producing items if the client disconnects.
            cancellationToken.ThrowIfCancellationRequested();

            if (uiMessage.Message.Contains("404"))
                // it should be displayed on the error list - this is not a stream
                await Clients.Caller.SendAsync("ReceiveBrokenLinkMessage","404 error on blah", cancellationToken);
            else
                // update the stream UI with whatever is happening in static Crawl
                yield return new UIMessage(uiMessage.Message, uiMessage.Hyperlink, uiMessage.NewLine);
        }
    }
}