diff --git a/scraper/src/Scrapers/Route.cs b/scraper/src/Scrapers/Route.cs index ab07a3d..3296b94 100644 --- a/scraper/src/Scrapers/Route.cs +++ b/scraper/src/Scrapers/Route.cs @@ -187,6 +187,8 @@ public static class RouteScraper { foreach (var div in leftSideDivs[2] .QuerySelectorAll(":scope > div") .Where((_, i) => i % 2 != 0)) { + var text = div.Text().WithCollapsedSpaces(); + if (text == "Nu sunt stații intermediare.") continue; train.AddIntermediateStop(div.Text().WithCollapsedSpaces()); } diff --git a/server/Controllers/V3/ItinerariesController.cs b/server/Controllers/V3/ItinerariesController.cs new file mode 100644 index 0000000..162561c --- /dev/null +++ b/server/Controllers/V3/ItinerariesController.cs @@ -0,0 +1,40 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using scraper.Models.Itinerary; +using Server.Services.Interfaces; + +namespace Server.Controllers.V3; + +[ApiController] +[ApiExplorerSettings(GroupName = "v3")] +[Route("/v3/[controller]")] +public class ItinerariesController : Controller { + private IDataManager DataManager { get; } + private IDatabase Database { get; } + + public ItinerariesController(IDataManager dataManager, IDatabase database) { + this.DataManager = dataManager; + this.Database = database; + } + + + [HttpGet("")] + [ProducesResponseType(typeof(IEnumerable), StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status404NotFound)] + public async Task>> FindItineraries( + [FromQuery] string from, + [FromQuery] string to, + [FromQuery] DateTimeOffset? date + ) { + var itineraries = await DataManager.FetchItineraries(from, to, date); + + if (itineraries == null) { + return NotFound(); + } + + return Ok(itineraries); + } +} \ No newline at end of file diff --git a/server/Services/Implementations/DataManager.cs b/server/Services/Implementations/DataManager.cs index ac0d4e7..6733cea 100644 --- a/server/Services/Implementations/DataManager.cs +++ b/server/Services/Implementations/DataManager.cs @@ -8,6 +8,7 @@ using Server.Services.Interfaces; using Server.Utils; using InfoferScraper; using Microsoft.Extensions.Logging; +using scraper.Models.Itinerary; namespace Server.Services.Implementations { public class DataManager : IDataManager { @@ -52,10 +53,27 @@ namespace Server.Services.Implementations { } return train; }, TimeSpan.FromSeconds(30)); + itinerariesCache = new(async (t) => { + var (from, to, date) = t; + var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone); + + var itineraries = await InfoferScraper.Scrapers.RouteScraper.Scrape(from, to, zonedDate.ToDateTimeOffset()); + if (itineraries != null) { + _ = Task.Run(async () => { + var watch = Stopwatch.StartNew(); + await Database.OnItineraries(itineraries); + var ms = watch.ElapsedMilliseconds; + Logger.LogInformation("OnItineraries timing: {StationDataMs} ms", ms); + }); + } + + return itineraries; + }, TimeSpan.FromMinutes(1)); } private readonly AsyncCache<(string, DateOnly), IStationScrapeResult?> stationCache; private readonly AsyncCache<(string, DateOnly), ITrainScrapeResult?> trainCache; + private readonly AsyncCache<(string, string, DateOnly), IReadOnlyList?> itinerariesCache; public Task FetchStation(string stationName, DateTimeOffset date) { var cfrDateTime = new NodaTime.ZonedDateTime(NodaTime.Instant.FromDateTimeOffset(date), CfrTimeZone); @@ -70,5 +88,12 @@ namespace Server.Services.Implementations { return trainCache.GetItem((trainNumber, cfrDate)); } + + public async Task?> FetchItineraries(string from, string to, DateTimeOffset? date = null) { + var cfrDateTime = new NodaTime.ZonedDateTime(NodaTime.Instant.FromDateTimeOffset(date ?? DateTimeOffset.Now), CfrTimeZone); + var cfrDate = new DateOnly(cfrDateTime.Year, cfrDateTime.Month, cfrDateTime.Day); + + return await itinerariesCache.GetItem((from, to, cfrDate)); + } } -} +} \ No newline at end of file diff --git a/server/Services/Implementations/Database.cs b/server/Services/Implementations/Database.cs index ce30355..0756620 100644 --- a/server/Services/Implementations/Database.cs +++ b/server/Services/Implementations/Database.cs @@ -13,6 +13,7 @@ using Microsoft.Extensions.Options; using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; using MongoDB.Driver; +using scraper.Models.Itinerary; using Server.Models.Database; using Server.Utils; @@ -339,6 +340,17 @@ public class Database : Server.Services.Interfaces.IDatabase { await ProcessTrain(train); } } + + public async Task OnItineraries(IReadOnlyList itineraries) { + foreach (var itinerary in itineraries) { + foreach (var train in itinerary.Trains) { + await FoundTrainAtStations( + train.IntermediateStops.Concat(new[] { train.From, train.To }), + train.TrainNumber + ); + } + } + } } public record DbRecord( diff --git a/server/Services/Interfaces/IDataManager.cs b/server/Services/Interfaces/IDataManager.cs index 0a24dae..71ee14c 100644 --- a/server/Services/Interfaces/IDataManager.cs +++ b/server/Services/Interfaces/IDataManager.cs @@ -1,11 +1,14 @@ using System; +using System.Collections.Generic; using System.Threading.Tasks; using InfoferScraper.Models.Train; using InfoferScraper.Models.Station; +using scraper.Models.Itinerary; namespace Server.Services.Interfaces; public interface IDataManager { public Task FetchStation(string stationName, DateTimeOffset date); public Task FetchTrain(string trainNumber, DateTimeOffset date); + public Task?> FetchItineraries(string from, string to, DateTimeOffset? date = null); } diff --git a/server/Services/Interfaces/IDatabase.cs b/server/Services/Interfaces/IDatabase.cs index c6a4901..da6076e 100644 --- a/server/Services/Interfaces/IDatabase.cs +++ b/server/Services/Interfaces/IDatabase.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Threading.Tasks; using InfoferScraper.Models.Train; using InfoferScraper.Models.Station; +using scraper.Models.Itinerary; using Server.Models.Database; namespace Server.Services.Interfaces; @@ -15,4 +16,5 @@ public interface IDatabase { public Task FoundTrainAtStation(string stationName, string trainName); public Task OnTrainData(ITrainScrapeResult trainData); public Task OnStationData(IStationScrapeResult stationData); + public Task OnItineraries(IReadOnlyList itineraries); } diff --git a/server/Utils/IAsyncCusorAsyncAdapter.cs b/server/Utils/IAsyncCusorAsyncAdapter.cs new file mode 100644 index 0000000..7f40968 --- /dev/null +++ b/server/Utils/IAsyncCusorAsyncAdapter.cs @@ -0,0 +1,33 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using MongoDB.Driver; + +namespace Server.Utils; + +public record IAsyncCusorAsyncEnumerator(IAsyncCursor Cursor) { + private IEnumerator? enumerator = null; + + public T Current => enumerator!.Current; + + public async Task MoveNextAsync() { + bool result; + if (enumerator != null) { + result = enumerator.MoveNext(); + if (result) return true; + } + + result = await Cursor.MoveNextAsync(); + if (result) { + enumerator = Cursor.Current.GetEnumerator(); + return true; + } + + return false; + } +} + +public static class IAsyncCursorExtensions { + public static IAsyncCusorAsyncEnumerator GetAsyncEnumerator(this IAsyncCursor cursor) { + return new(cursor); + } +} \ No newline at end of file