kbruen
/
new-infofer-scraper
1
0
Fork 0
Code Pull Requests Releases Activity
Browse Source

Fix Options pattern

master
Kenneth Bruen 3 months ago
parent
441546fb93
commit
f00b602e71
Signed by: kbruen
GPG Key ID: C1980A470C3EE5B1
2 changed files with 117 additions and 116 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      server/Models/ProxySettings.cs
  2. 229
      server/Services/Implementations/DataManager.cs

4
server/Models/ProxySettings.cs
Unescape View File

@ -1,7 +1,7 @@
namespace Server.Models; namespace Server.Models;
public record ProxySettings(string Url, ProxyCredentials? Credentials = null) { public record ProxySettings(bool UseProxy, string Url, ProxyCredentials? Credentials = null) {
public ProxySettings() : this("") { } public ProxySettings() : this(false, "") { }
} }
public record ProxyCredentials(string Username, string Password) { public record ProxyCredentials(string Username, string Password) {

229
server/Services/Implementations/DataManager.cs
Unescape View File

@ -1,114 +1,115 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using System.Net; using System.Net;
using System.Net.Http; using System.Net.Http;
using System.Threading.Tasks; using System.Threading.Tasks;
using InfoferScraper; using InfoferScraper;
using InfoferScraper.Models.Station; using InfoferScraper.Models.Station;
using InfoferScraper.Models.Train; using InfoferScraper.Models.Train;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using scraper.Models.Itinerary; using Microsoft.Extensions.Options;
using Server.Models; using scraper.Models.Itinerary;
using Server.Services.Interfaces; using Server.Models;
using Server.Utils; using Server.Services.Interfaces;
using Server.Utils;
namespace Server.Services.Implementations {
public class DataManager : IDataManager { namespace Server.Services.Implementations {
private ILogger<DataManager> Logger { get; } public class DataManager : IDataManager {
private IDatabase Database { get; } private ILogger<DataManager> Logger { get; }
private IDatabase Database { get; }
private NodaTime.IDateTimeZoneProvider TzProvider { get; }
private NodaTime.DateTimeZone CfrTimeZone => TzProvider["Europe/Bucharest"]; private NodaTime.IDateTimeZoneProvider TzProvider { get; }
private NodaTime.DateTimeZone CfrTimeZone => TzProvider["Europe/Bucharest"];
public DataManager(NodaTime.IDateTimeZoneProvider tzProvider, IDatabase database, ILogger<DataManager> logger, ProxySettings? proxySettings) {
this.TzProvider = tzProvider; public DataManager(NodaTime.IDateTimeZoneProvider tzProvider, IDatabase database, ILogger<DataManager> logger, IOptions<ProxySettings> proxySettings) {
this.Database = database; this.TzProvider = tzProvider;
this.Logger = logger; this.Database = database;
this.Logger = logger;
HttpClientHandler httpClientHandler = new (){
UseProxy = proxySettings != null, HttpClientHandler httpClientHandler = new() {
Proxy = proxySettings == null ? null : new WebProxy(proxySettings.Url), UseProxy = proxySettings.Value.UseProxy,
DefaultProxyCredentials = proxySettings?.Credentials == null ? null : new NetworkCredential(proxySettings.Credentials.Username, proxySettings.Credentials.Password), Proxy = proxySettings.Value.UseProxy ? new WebProxy(proxySettings.Value.Url) : null,
}; DefaultProxyCredentials = string.IsNullOrEmpty(proxySettings.Value.Credentials?.Username) ? null : new NetworkCredential(proxySettings.Value.Credentials.Username, proxySettings.Value.Credentials.Password),
InfoferScraper.Scrapers.StationScraper stationScraper = new(httpClientHandler); };
InfoferScraper.Scrapers.TrainScraper trainScraper = new(httpClientHandler); InfoferScraper.Scrapers.StationScraper stationScraper = new(httpClientHandler);
InfoferScraper.Scrapers.RouteScraper routeScraper = new(httpClientHandler); InfoferScraper.Scrapers.TrainScraper trainScraper = new(httpClientHandler);
InfoferScraper.Scrapers.RouteScraper routeScraper = new(httpClientHandler);
stationCache = new(async (t) => {
var (stationName, date) = t; stationCache = new(async (t) => {
Logger.LogDebug("Fetching station {StationName} for date {Date}", stationName, date); var (stationName, date) = t;
var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone); Logger.LogDebug("Fetching station {StationName} for date {Date}", stationName, date);
var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone);
var station = await stationScraper.Scrape(stationName, zonedDate.ToDateTimeOffset());
if (station != null) { var station = await stationScraper.Scrape(stationName, zonedDate.ToDateTimeOffset());
_ = Task.Run(async () => { if (station != null) {
var watch = Stopwatch.StartNew(); _ = Task.Run(async () => {
await Database.OnStationData(station); var watch = Stopwatch.StartNew();
var ms = watch.ElapsedMilliseconds; await Database.OnStationData(station);
Logger.LogInformation("OnStationData timing: {StationDataMs} ms", ms); var ms = watch.ElapsedMilliseconds;
}); Logger.LogInformation("OnStationData timing: {StationDataMs} ms", ms);
} });
return station; }
}, TimeSpan.FromMinutes(1)); return station;
trainCache = new(async (t) => { }, TimeSpan.FromMinutes(1));
var (trainNumber, date) = t; trainCache = new(async (t) => {
Logger.LogDebug("Fetching train {TrainNumber} for date {Date}", trainNumber, date); var (trainNumber, date) = t;
var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone); Logger.LogDebug("Fetching train {TrainNumber} for date {Date}", trainNumber, date);
var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone);
var train = await trainScraper.Scrape(trainNumber, zonedDate.ToDateTimeOffset());
if (train != null) { var train = await trainScraper.Scrape(trainNumber, zonedDate.ToDateTimeOffset());
_ = Task.Run(async () => { if (train != null) {
var watch = Stopwatch.StartNew(); _ = Task.Run(async () => {
await Database.OnTrainData(train); var watch = Stopwatch.StartNew();
var ms = watch.ElapsedMilliseconds; await Database.OnTrainData(train);
Logger.LogInformation("OnTrainData timing: {StationDataMs} ms", ms); var ms = watch.ElapsedMilliseconds;
}); Logger.LogInformation("OnTrainData timing: {StationDataMs} ms", ms);
} });
return train; }
}, TimeSpan.FromSeconds(30)); return train;
itinerariesCache = new(async (t) => { }, TimeSpan.FromSeconds(30));
var (from, to, date) = t; itinerariesCache = new(async (t) => {
Logger.LogDebug("Fetching itinerary from {From} to {To} for date {Date}", from, to, date); var (from, to, date) = t;
var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone); Logger.LogDebug("Fetching itinerary from {From} to {To} for date {Date}", from, to, date);
var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone);
var itineraries = await routeScraper.Scrape(from, to, zonedDate.ToDateTimeOffset());
if (itineraries != null) { var itineraries = await routeScraper.Scrape(from, to, zonedDate.ToDateTimeOffset());
_ = Task.Run(async () => { if (itineraries != null) {
var watch = Stopwatch.StartNew(); _ = Task.Run(async () => {
await Database.OnItineraries(itineraries); var watch = Stopwatch.StartNew();
var ms = watch.ElapsedMilliseconds; await Database.OnItineraries(itineraries);
Logger.LogInformation("OnItineraries timing: {StationDataMs} ms", ms); var ms = watch.ElapsedMilliseconds;
}); Logger.LogInformation("OnItineraries timing: {StationDataMs} ms", ms);
} });
}
return itineraries;
}, TimeSpan.FromMinutes(1)); return itineraries;
} }, TimeSpan.FromMinutes(1));
}
private readonly AsyncCache<(string, DateOnly), IStationScrapeResult?> stationCache;
private readonly AsyncCache<(string, DateOnly), ITrainScrapeResult?> trainCache; private readonly AsyncCache<(string, DateOnly), IStationScrapeResult?> stationCache;
private readonly AsyncCache<(string, string, DateOnly), IReadOnlyList<IItinerary>?> itinerariesCache; private readonly AsyncCache<(string, DateOnly), ITrainScrapeResult?> trainCache;
private readonly AsyncCache<(string, string, DateOnly), IReadOnlyList<IItinerary>?> itinerariesCache;
public Task<IStationScrapeResult?> FetchStation(string stationName, DateTimeOffset date) {
var cfrDateTime = new NodaTime.ZonedDateTime(NodaTime.Instant.FromDateTimeOffset(date), CfrTimeZone); public Task<IStationScrapeResult?> FetchStation(string stationName, DateTimeOffset date) {
var cfrDate = new DateOnly(cfrDateTime.Year, cfrDateTime.Month, cfrDateTime.Day); var cfrDateTime = new NodaTime.ZonedDateTime(NodaTime.Instant.FromDateTimeOffset(date), CfrTimeZone);
var cfrDate = new DateOnly(cfrDateTime.Year, cfrDateTime.Month, cfrDateTime.Day);
return stationCache.GetItem((stationName.RoLettersToEn().ToLowerInvariant(), cfrDate));
} return stationCache.GetItem((stationName.RoLettersToEn().ToLowerInvariant(), cfrDate));
}
public Task<ITrainScrapeResult?> FetchTrain(string trainNumber, DateTimeOffset date) {
var cfrDateTime = new NodaTime.ZonedDateTime(NodaTime.Instant.FromDateTimeOffset(date), CfrTimeZone); public Task<ITrainScrapeResult?> FetchTrain(string trainNumber, DateTimeOffset date) {
var cfrDate = new DateOnly(cfrDateTime.Year, cfrDateTime.Month, cfrDateTime.Day); var cfrDateTime = new NodaTime.ZonedDateTime(NodaTime.Instant.FromDateTimeOffset(date), CfrTimeZone);
var cfrDate = new DateOnly(cfrDateTime.Year, cfrDateTime.Month, cfrDateTime.Day);
return trainCache.GetItem((trainNumber, cfrDate));
} return trainCache.GetItem((trainNumber, cfrDate));
}
public async Task<IReadOnlyList<IItinerary>?> FetchItineraries(string from, string to, DateTimeOffset? date = null) {
var cfrDateTime = new NodaTime.ZonedDateTime(NodaTime.Instant.FromDateTimeOffset(date ?? DateTimeOffset.Now), CfrTimeZone); public async Task<IReadOnlyList<IItinerary>?> FetchItineraries(string from, string to, DateTimeOffset? date = null) {
var cfrDate = new DateOnly(cfrDateTime.Year, cfrDateTime.Month, cfrDateTime.Day); var cfrDateTime = new NodaTime.ZonedDateTime(NodaTime.Instant.FromDateTimeOffset(date ?? DateTimeOffset.Now), CfrTimeZone);
var cfrDate = new DateOnly(cfrDateTime.Year, cfrDateTime.Month, cfrDateTime.Day);
return await itinerariesCache.GetItem((from, to, cfrDate));
} return await itinerariesCache.GetItem((from, to, cfrDate));
} }
} }
}

Write Preview
Loading…
Cancel
Save