diff --git a/server/Models/ProxySettings.cs b/server/Models/ProxySettings.cs index 389e9c3..ac80196 100644 --- a/server/Models/ProxySettings.cs +++ b/server/Models/ProxySettings.cs @@ -1,7 +1,7 @@ namespace Server.Models; -public record ProxySettings(string Url, ProxyCredentials? Credentials = null) { - public ProxySettings() : this("") { } +public record ProxySettings(bool UseProxy, string Url, ProxyCredentials? Credentials = null) { + public ProxySettings() : this(false, "") { } } public record ProxyCredentials(string Username, string Password) { diff --git a/server/Services/Implementations/DataManager.cs b/server/Services/Implementations/DataManager.cs index a8e0e25..5bac346 100644 --- a/server/Services/Implementations/DataManager.cs +++ b/server/Services/Implementations/DataManager.cs @@ -1,114 +1,115 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Net; -using System.Net.Http; -using System.Threading.Tasks; -using InfoferScraper; -using InfoferScraper.Models.Station; -using InfoferScraper.Models.Train; -using Microsoft.Extensions.Logging; -using scraper.Models.Itinerary; -using Server.Models; -using Server.Services.Interfaces; -using Server.Utils; - -namespace Server.Services.Implementations { - public class DataManager : IDataManager { - private ILogger Logger { get; } - private IDatabase Database { get; } - - private NodaTime.IDateTimeZoneProvider TzProvider { get; } - private NodaTime.DateTimeZone CfrTimeZone => TzProvider["Europe/Bucharest"]; - - public DataManager(NodaTime.IDateTimeZoneProvider tzProvider, IDatabase database, ILogger logger, ProxySettings? proxySettings) { - this.TzProvider = tzProvider; - this.Database = database; - this.Logger = logger; - - HttpClientHandler httpClientHandler = new (){ - UseProxy = proxySettings != null, - Proxy = proxySettings == null ? null : new WebProxy(proxySettings.Url), - DefaultProxyCredentials = proxySettings?.Credentials == null ? null : new NetworkCredential(proxySettings.Credentials.Username, proxySettings.Credentials.Password), - }; - InfoferScraper.Scrapers.StationScraper stationScraper = new(httpClientHandler); - InfoferScraper.Scrapers.TrainScraper trainScraper = new(httpClientHandler); - InfoferScraper.Scrapers.RouteScraper routeScraper = new(httpClientHandler); - - stationCache = new(async (t) => { - var (stationName, date) = t; - Logger.LogDebug("Fetching station {StationName} for date {Date}", stationName, date); - var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone); - - var station = await stationScraper.Scrape(stationName, zonedDate.ToDateTimeOffset()); - if (station != null) { - _ = Task.Run(async () => { - var watch = Stopwatch.StartNew(); - await Database.OnStationData(station); - var ms = watch.ElapsedMilliseconds; - Logger.LogInformation("OnStationData timing: {StationDataMs} ms", ms); - }); - } - return station; - }, TimeSpan.FromMinutes(1)); - trainCache = new(async (t) => { - var (trainNumber, date) = t; - Logger.LogDebug("Fetching train {TrainNumber} for date {Date}", trainNumber, date); - var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone); - - var train = await trainScraper.Scrape(trainNumber, zonedDate.ToDateTimeOffset()); - if (train != null) { - _ = Task.Run(async () => { - var watch = Stopwatch.StartNew(); - await Database.OnTrainData(train); - var ms = watch.ElapsedMilliseconds; - Logger.LogInformation("OnTrainData timing: {StationDataMs} ms", ms); - }); - } - return train; - }, TimeSpan.FromSeconds(30)); - itinerariesCache = new(async (t) => { - var (from, to, date) = t; - Logger.LogDebug("Fetching itinerary from {From} to {To} for date {Date}", from, to, date); - var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone); - - var itineraries = await routeScraper.Scrape(from, to, zonedDate.ToDateTimeOffset()); - if (itineraries != null) { - _ = Task.Run(async () => { - var watch = Stopwatch.StartNew(); - await Database.OnItineraries(itineraries); - var ms = watch.ElapsedMilliseconds; - Logger.LogInformation("OnItineraries timing: {StationDataMs} ms", ms); - }); - } - - return itineraries; - }, TimeSpan.FromMinutes(1)); - } - - private readonly AsyncCache<(string, DateOnly), IStationScrapeResult?> stationCache; - private readonly AsyncCache<(string, DateOnly), ITrainScrapeResult?> trainCache; - private readonly AsyncCache<(string, string, DateOnly), IReadOnlyList?> itinerariesCache; - - public Task FetchStation(string stationName, DateTimeOffset date) { - var cfrDateTime = new NodaTime.ZonedDateTime(NodaTime.Instant.FromDateTimeOffset(date), CfrTimeZone); - var cfrDate = new DateOnly(cfrDateTime.Year, cfrDateTime.Month, cfrDateTime.Day); - - return stationCache.GetItem((stationName.RoLettersToEn().ToLowerInvariant(), cfrDate)); - } - - public Task FetchTrain(string trainNumber, DateTimeOffset date) { - var cfrDateTime = new NodaTime.ZonedDateTime(NodaTime.Instant.FromDateTimeOffset(date), CfrTimeZone); - var cfrDate = new DateOnly(cfrDateTime.Year, cfrDateTime.Month, cfrDateTime.Day); - - return trainCache.GetItem((trainNumber, cfrDate)); - } - - public async Task?> FetchItineraries(string from, string to, DateTimeOffset? date = null) { - var cfrDateTime = new NodaTime.ZonedDateTime(NodaTime.Instant.FromDateTimeOffset(date ?? DateTimeOffset.Now), CfrTimeZone); - var cfrDate = new DateOnly(cfrDateTime.Year, cfrDateTime.Month, cfrDateTime.Day); - - return await itinerariesCache.GetItem((from, to, cfrDate)); - } - } -} +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using InfoferScraper; +using InfoferScraper.Models.Station; +using InfoferScraper.Models.Train; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using scraper.Models.Itinerary; +using Server.Models; +using Server.Services.Interfaces; +using Server.Utils; + +namespace Server.Services.Implementations { + public class DataManager : IDataManager { + private ILogger Logger { get; } + private IDatabase Database { get; } + + private NodaTime.IDateTimeZoneProvider TzProvider { get; } + private NodaTime.DateTimeZone CfrTimeZone => TzProvider["Europe/Bucharest"]; + + public DataManager(NodaTime.IDateTimeZoneProvider tzProvider, IDatabase database, ILogger logger, IOptions proxySettings) { + this.TzProvider = tzProvider; + this.Database = database; + this.Logger = logger; + + HttpClientHandler httpClientHandler = new() { + UseProxy = proxySettings.Value.UseProxy, + Proxy = proxySettings.Value.UseProxy ? new WebProxy(proxySettings.Value.Url) : null, + DefaultProxyCredentials = string.IsNullOrEmpty(proxySettings.Value.Credentials?.Username) ? null : new NetworkCredential(proxySettings.Value.Credentials.Username, proxySettings.Value.Credentials.Password), + }; + InfoferScraper.Scrapers.StationScraper stationScraper = new(httpClientHandler); + InfoferScraper.Scrapers.TrainScraper trainScraper = new(httpClientHandler); + InfoferScraper.Scrapers.RouteScraper routeScraper = new(httpClientHandler); + + stationCache = new(async (t) => { + var (stationName, date) = t; + Logger.LogDebug("Fetching station {StationName} for date {Date}", stationName, date); + var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone); + + var station = await stationScraper.Scrape(stationName, zonedDate.ToDateTimeOffset()); + if (station != null) { + _ = Task.Run(async () => { + var watch = Stopwatch.StartNew(); + await Database.OnStationData(station); + var ms = watch.ElapsedMilliseconds; + Logger.LogInformation("OnStationData timing: {StationDataMs} ms", ms); + }); + } + return station; + }, TimeSpan.FromMinutes(1)); + trainCache = new(async (t) => { + var (trainNumber, date) = t; + Logger.LogDebug("Fetching train {TrainNumber} for date {Date}", trainNumber, date); + var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone); + + var train = await trainScraper.Scrape(trainNumber, zonedDate.ToDateTimeOffset()); + if (train != null) { + _ = Task.Run(async () => { + var watch = Stopwatch.StartNew(); + await Database.OnTrainData(train); + var ms = watch.ElapsedMilliseconds; + Logger.LogInformation("OnTrainData timing: {StationDataMs} ms", ms); + }); + } + return train; + }, TimeSpan.FromSeconds(30)); + itinerariesCache = new(async (t) => { + var (from, to, date) = t; + Logger.LogDebug("Fetching itinerary from {From} to {To} for date {Date}", from, to, date); + var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone); + + var itineraries = await routeScraper.Scrape(from, to, zonedDate.ToDateTimeOffset()); + if (itineraries != null) { + _ = Task.Run(async () => { + var watch = Stopwatch.StartNew(); + await Database.OnItineraries(itineraries); + var ms = watch.ElapsedMilliseconds; + Logger.LogInformation("OnItineraries timing: {StationDataMs} ms", ms); + }); + } + + return itineraries; + }, TimeSpan.FromMinutes(1)); + } + + private readonly AsyncCache<(string, DateOnly), IStationScrapeResult?> stationCache; + private readonly AsyncCache<(string, DateOnly), ITrainScrapeResult?> trainCache; + private readonly AsyncCache<(string, string, DateOnly), IReadOnlyList?> itinerariesCache; + + public Task FetchStation(string stationName, DateTimeOffset date) { + var cfrDateTime = new NodaTime.ZonedDateTime(NodaTime.Instant.FromDateTimeOffset(date), CfrTimeZone); + var cfrDate = new DateOnly(cfrDateTime.Year, cfrDateTime.Month, cfrDateTime.Day); + + return stationCache.GetItem((stationName.RoLettersToEn().ToLowerInvariant(), cfrDate)); + } + + public Task FetchTrain(string trainNumber, DateTimeOffset date) { + var cfrDateTime = new NodaTime.ZonedDateTime(NodaTime.Instant.FromDateTimeOffset(date), CfrTimeZone); + var cfrDate = new DateOnly(cfrDateTime.Year, cfrDateTime.Month, cfrDateTime.Day); + + return trainCache.GetItem((trainNumber, cfrDate)); + } + + public async Task?> FetchItineraries(string from, string to, DateTimeOffset? date = null) { + var cfrDateTime = new NodaTime.ZonedDateTime(NodaTime.Instant.FromDateTimeOffset(date ?? DateTimeOffset.Now), CfrTimeZone); + var cfrDate = new DateOnly(cfrDateTime.Year, cfrDateTime.Month, cfrDateTime.Day); + + return await itinerariesCache.GetItem((from, to, cfrDate)); + } + } +}