Add fetching of suburb boundaries from external API. Some minor updates to align with best practices, some optimisations.

backend/Config/SuburbBoundaryApiConfig.cs

@@ -0,0 +1,6 @@
+namespace backend.Config;
+
+public class SuburbBoundaryApiConfig
+{
+    public required string Url { get; init; }
+}

backend/Extensions/ServiceCollectionExtensions.cs

@@ -27,6 +27,7 @@ public static class ServiceCollectionExtensions
             .AddSingleton<IFuelTypeService, FuelTypeService>()
             .AddSingleton<IStationWithPricesService, StationWithPricesService>()
             .AddSingleton<IPriceTrendsService, PriceTrendsService>()
+            .AddSingleton<ISuburbBoundariesService, SuburbBoundariesService>()
             .AddSingleton<DatabaseInitialiser>()
             .AddSingleton<HangfireInitialiser>()
             .AddLogging(options =>
@@ -61,9 +62,9 @@ public static class ServiceCollectionExtensions
 
     public static void SetupConfiguration(this IServiceCollection services, IConfiguration configuration)
     {
-        //TODO: in prod, get the config values from a secret manager (see TODO.md)
         services
-            .AddSingleton(configuration.GetSection(nameof(NswFuelApiConfig)).Get<NswFuelApiConfig>()!);
+            .AddSingleton(configuration.GetSection(nameof(NswFuelApiConfig)).Get<NswFuelApiConfig>()!)
+            .AddSingleton(configuration.GetSection(nameof(SuburbBoundaryApiConfig)).Get<SuburbBoundaryApiConfig>()!);
     }
 
     public static void SetupDatabase(this IServiceCollection services, IConfiguration configuration)

backend/Models/SuburbBoundariesApi/SuburbBoundary.cs

@@ -0,0 +1,14 @@
+using ServiceStack.DataAnnotations;
+
+namespace backend.Models.SuburbBoundariesApi;
+
+public class SuburbBoundary
+{
+    [PrimaryKey] public Guid Id { get; init; } = Guid.NewGuid();
+
+    public string? Name { get; set; }
+
+    public string? Geometry { get; set; }
+
+    public DateTimeOffset? DtCreate { get; set; }
+}

backend/Services/ISuburbBoundariesService.cs

@@ -0,0 +1,9 @@
+using System.Text.Json;
+
+namespace backend.Services;
+
+public interface ISuburbBoundariesService
+{
+    public Task<bool> FetchFeatures();
+    public Task<bool> InsertFeatures(JsonElement features);
+}

backend/Services/Init/DatabaseInitialiser.cs

@@ -1,4 +1,5 @@
 using backend.Models.NswFuelApi;
+using backend.Models.SuburbBoundariesApi;
 using ServiceStack.Data;
 using ServiceStack.OrmLite;
 
@@ -24,6 +25,7 @@ public class DatabaseInitialiser
         db.CreateTableIfNotExists<BrandType>();
         db.CreateTableIfNotExists<FuelType>();
         db.CreateTableIfNotExists<Station>();
+        db.CreateTableIfNotExists<SuburbBoundary>();
 
         //check if the prices hypertable already exists, otherwise the raw SQL will fail
         if (!db.TableExists<Price>())

backend/Services/Init/HangfireInitialiser.cs

@@ -17,5 +17,9 @@ public class HangfireInitialiser
         RecurringJob.AddOrUpdate<INswFuelApiService>("refresh_lovs", x => x.GetLovsAsync(), "*/10 * * * *");
         //ten minute refresh for prices
         RecurringJob.AddOrUpdate<INswFuelApiService>("refresh_prices", x => x.GetCurrentPricesAsync(), "*/10 * * * *");
+        //daily refresh for suburb boundaries at midnight - do this in Sydney time (safe to assume the app will be run in NSW)
+        RecurringJob.AddOrUpdate<ISuburbBoundariesService>("fetch_suburb_boundaries", x => x.FetchFeatures(),
+            "0 0 * * *",
+            new RecurringJobOptions { TimeZone = TimeZoneInfo.FindSystemTimeZoneById("Australia/Sydney") });
     }
 }

backend/Services/NswFuelApiService.cs

@@ -10,15 +10,15 @@ namespace backend.Services;
 public class NswFuelApiService : INswFuelApiService
 {
     private readonly IDbConnectionFactory _dbConnectionFactory;
-    private readonly HttpClient _httpClient;
+    private readonly IHttpClientFactory _httpClientFactory;
     private readonly ILogger<NswFuelApiService> _logger;
     private readonly NswFuelApiConfig _nswFuelApiConfig;
     private AccessTokenResponse? _cachedToken;
 
-    public NswFuelApiService(HttpClient httpClient, ILogger<NswFuelApiService> logger,
+    public NswFuelApiService(IHttpClientFactory httpClientFactory, ILogger<NswFuelApiService> logger,
         NswFuelApiConfig nswFuelApiConfig, IDbConnectionFactory dbConnectionFactory)
     {
-        _httpClient = httpClient;
+        _httpClientFactory = httpClientFactory;
         _nswFuelApiConfig = nswFuelApiConfig;
         _logger = logger;
         _dbConnectionFactory = dbConnectionFactory;
@@ -26,12 +26,13 @@ public class NswFuelApiService : INswFuelApiService
 
     public async Task<bool> GetLovsAsync()
     {
+        var client = _httpClientFactory.CreateClient();
         var token = await GetAccessTokenAsync();
         var url = $"{_nswFuelApiConfig.BaseUrl}/FuelCheckRefData/v2/fuel/lovs";
         var request = CreateRequestMessage(HttpMethod.Get, url,
             token?.AccessToken);
 
-        var response = await _httpClient.SendAsync(request);
+        var response = await client.SendAsync(request);
         response.EnsureSuccessStatusCode();
 
         var content = await response.Content.ReadAsStreamAsync();
@@ -79,7 +80,7 @@ public class NswFuelApiService : INswFuelApiService
         }
         catch (Exception e)
         {
-            _logger.LogError(e.ToString());
+            _logger.LogError("Error while attempting to get update lovs: {Error}", e);
             transaction.Rollback();
             return false;
         }
@@ -90,11 +91,12 @@ public class NswFuelApiService : INswFuelApiService
 
     public async Task<bool> GetCurrentPricesAsync()
     {
+        var client = _httpClientFactory.CreateClient();
         var token = await GetAccessTokenAsync();
         var url = $"{_nswFuelApiConfig.BaseUrl}/FuelPriceCheck/v2/fuel/prices";
         var request = CreateRequestMessage(HttpMethod.Get, url, token?.AccessToken);
 
-        var response = await _httpClient.SendAsync(request);
+        var response = await client.SendAsync(request);
         response.EnsureSuccessStatusCode();
 
         var content = await response.Content.ReadAsStreamAsync();
@@ -147,7 +149,7 @@ public class NswFuelApiService : INswFuelApiService
             //stage 3: insert only new or updated prices
             if (pricesToAdd.Count > 0)
             {
-                _logger.LogInformation($"Attempting to insert {pricesToAdd.Count} new prices");
+                _logger.LogInformation("Attempting to insert {Count} new prices", pricesToAdd.Count);
                 db.BulkInsert(pricesToAdd);
             }
             else
@@ -159,7 +161,7 @@ public class NswFuelApiService : INswFuelApiService
         }
         catch (Exception e)
         {
-            _logger.LogError(e.ToString());
+            _logger.LogError("Error while attempting to get current prices: {Error}", e);
             transaction.Rollback();
             return false;
         }
@@ -188,12 +190,13 @@ public class NswFuelApiService : INswFuelApiService
 
         _logger.LogInformation("Cached token has expired, getting a new one");
 
+        var client = _httpClientFactory.CreateClient();
         var url = $"{_nswFuelApiConfig.BaseUrl}/oauth/client_credential/accesstoken?grant_type=client_credentials";
         var request = new HttpRequestMessage(HttpMethod.Get,
             url);
         request.Headers.Authorization = new AuthenticationHeaderValue("Basic", _nswFuelApiConfig.AuthorisationHeader);
 
-        var response = await _httpClient.SendAsync(request);
+        var response = await client.SendAsync(request);
         response.EnsureSuccessStatusCode();
         _logger.LogInformation("Successfully retrieved a new access token");
 

backend/Services/SuburbBoundariesService.cs

@@ -0,0 +1,104 @@
+using System.Text.Json;
+using backend.Config;
+using backend.Models.SuburbBoundariesApi;
+using ServiceStack.Data;
+using ServiceStack.OrmLite;
+
+namespace backend.Services;
+
+public class SuburbBoundariesService : ISuburbBoundariesService
+{
+    private readonly ILogger<SuburbBoundariesService> _logger;
+    private readonly IDbConnectionFactory _dbConnectionFactory;
+    private readonly IHttpClientFactory _httpClientFactory;
+    private readonly SuburbBoundaryApiConfig _suburbBoundaryApiConfig;
+
+    public SuburbBoundariesService(IDbConnectionFactory dbConnectionFactory, IHttpClientFactory httpClientFactory,
+        SuburbBoundaryApiConfig suburbBoundaryApiConfig, ILogger<SuburbBoundariesService> logger)
+    {
+        _dbConnectionFactory = dbConnectionFactory;
+        _httpClientFactory = httpClientFactory;
+        _suburbBoundaryApiConfig = suburbBoundaryApiConfig;
+        _logger = logger;
+    }
+
+    public async Task<bool> FetchFeatures()
+    {
+        _logger.LogInformation("Started suburb boundary data fetch job");
+
+        var client = _httpClientFactory.CreateClient();
+        var geoJsonData = await client.GetAsync(_suburbBoundaryApiConfig.Url, HttpCompletionOption.ResponseHeadersRead);
+
+        if (!geoJsonData.IsSuccessStatusCode)
+        {
+            _logger.LogInformation(
+                "Unsuccessful status code received while fetching suburb boundary data from the API: {StatusCode}",
+                geoJsonData.StatusCode);
+            return false;
+        }
+
+        await using var responseStream = await geoJsonData.Content.ReadAsStreamAsync();
+        using var jsonDocument = await JsonDocument.ParseAsync(responseStream);
+
+        var features = jsonDocument.RootElement.GetProperty("features");
+
+        if (features.GetArrayLength() != 0) return await InsertFeatures(features);
+
+        _logger.LogError("Suburb boundary Feature Collection is empty - data not refreshed");
+
+        return false;
+    }
+
+    public async Task<bool> InsertFeatures(JsonElement features)
+    {
+        using var db = _dbConnectionFactory.OpenDbConnection();
+
+        try
+        {
+            const int batchSize = 50;
+            var suburbBoundaries = new List<SuburbBoundary>(batchSize);
+
+            foreach (var feature in features.EnumerateArray())
+            {
+                if (!feature.TryGetProperty("geometry", out var geometry) ||
+                    geometry.GetProperty("type").GetString() != "MultiPolygon") continue;
+
+                suburbBoundaries.Add(new SuburbBoundary
+                {
+                    Name = feature.GetProperty("properties").GetProperty("nsw_loca_2").GetString(),
+                    Geometry = ConvertMultiPolygonToWkt(geometry.GetProperty("coordinates")),
+                    DtCreate = feature.GetProperty("properties").TryGetProperty("dt_create", out var dtCreate)
+                        ? DateTimeOffset.Parse(dtCreate.GetString() ?? throw new InvalidOperationException())
+                        : null
+                });
+
+                if (suburbBoundaries.Count < batchSize) continue;
+                await db.InsertAllAsync(suburbBoundaries);
+                suburbBoundaries.Clear();
+            }
+
+            if (suburbBoundaries.Count > 0)
+            {
+                await db.InsertAllAsync(suburbBoundaries);
+            }
+        }
+        catch (Exception e)
+        {
+            _logger.LogError("Error inserting suburb boundary features: {EMessage}", e.Message);
+            throw;
+        }
+
+        _logger.LogInformation("Successfully inserted suburb boundary data into the database");
+        return true;
+    }
+
+    private static string ConvertMultiPolygonToWkt(JsonElement multiPolygonCoordinates)
+    {
+        var polygons = multiPolygonCoordinates.EnumerateArray()
+            .Select(polygon => string.Join(", ",
+                polygon[0].EnumerateArray().Select(coord => $"{coord[0].GetDouble()} {coord[1].GetDouble()}")))
+            .Select(coordinates => $"(({coordinates}))").ToList();
+
+        return $"MULTIPOLYGON({string.Join(", ", polygons)})";
+    }
+}

backend/appsettings.Development.json

@@ -11,7 +11,10 @@
     "ApiSecret": "BMvWacw15Et8uFGF",
     "AuthorisationHeader": "MU1ZU1JBeDV5dnFIVVpjNlZHdHhpeDZvTUEycWdmUlQ6Qk12V2FjdzE1RXQ4dUZHRg=="
   },
+  "SuburbBoundaryApiConfig": {
+    "Url": "https://data.gov.au/geoserver/nsw-suburb-locality-boundaries-psma-administrative-boundaries/wfs?request=GetFeature&typeName=ckan_91e70237_d9d1_4719_a82f_e71b811154c6&outputFormat=json"
+  },
   "TimescaleDbConfig": {
     "ConnectionString": "User ID=postgres;Password=password;Host=localhost;Port=5432;Database=postgres;"
   }
-}
+}