@ -2,187 +2,347 @@ using System;
using System.Collections.Generic ; using System.Collections.Generic ;
using System.IO ; using System.IO ;
using System.Linq ; using System.Linq ;
using System.Text.Json ; using System.Threading ;
using System.Text.Json.Nodes ;
using System.Text.Json.Serialization ;
using System.Threading.Tasks ; using System.Threading.Tasks ;
using InfoferScraper.Models.Station ;
using Microsoft.Extensions.Logging ; using Microsoft.Extensions.Logging ;
using Microsoft.Extensions.Options ;
using MongoDB.Bson ;
using MongoDB.Bson.Serialization.Attributes ;
using MongoDB.Driver ;
using Newtonsoft.Json ;
using Newtonsoft.Json.Linq ;
using Newtonsoft.Json.Serialization ;
using scraper.Models.Itinerary ;
using Server.Models.Database ;
using Server.Utils ;
namespace Server.Services.Implementations ; namespace Server.Services.Implementations ;
public class Database : Server . Services . Interfaces . IDatabase { public class Database : Server . Services . Interfaces . IDatabase {
private static readonly JsonSerializerOptions serializerOptions = new ( ) { private static readonly JsonSerializerSettings jsonSerializerSettings = new ( ) {
PropertyNamingPolicy = JsonNamingPolicy . CamelCase , ContractResolver = new DefaultContractResolver {
NamingStrategy = new CamelCaseNamingStrategy ( ) ,
} ,
} ; } ;
private ILogger < Database > Logger { get ; } private ILogger < Database > Logger { get ; }
private bool shouldCommitOnEveryChange = true ; public DbRecord DbData { get ; private set ; } = new ( 3 ) ;
private bool dbDataDirty = false ;
private bool stationsDirty = false ;
private bool trainsDirty = false ;
public DbRecord DbData { get ; private set ; } = new ( 2 ) ; public IReadOnlyList < StationListing > Stations = > stationListingsCollection
private List < StationRecord > stations = new ( ) ; . Aggregate ( PipelineDefinition < StationListing , StationListing > . Create (
private List < TrainRecord > trains = new ( ) ; "{ $addFields: { stoppedAtCount: { $size: \"$stoppedAtBy\" } } }" ,
"{ $sort: { stoppedAtCount: -1 } }" ,
public IReadOnlyList < Server . Services . Interfaces . IStationRecord > Stations = > stations ; "{ $unset: \"stoppedAtCount\" }"
public IReadOnlyList < Server . Services . Interfaces . ITrainRecord > Trains = > trains ; ) )
. ToList ( ) ;
public IReadOnlyList < TrainListing > Trains = > trainListingsCollection . FindSync ( _ = > true ) . ToList ( ) ;
public IReadOnlyList < StationAlias > StationAliases = > stationAliasCollection . FindSync ( _ = > true ) . ToList ( ) ;
private static readonly string DbDir = Environment . GetEnvironmentVariable ( "DB_DIR" ) ? ? Path . Join ( Environment . CurrentDirectory , "db" ) ; private static readonly string DbDir = Environment . GetEnvironmentVariable ( "DB_DIR" ) ? ? Path . Join ( Environment . CurrentDirectory , "db" ) ;
private static readonly string DbFile = Path . Join ( DbDir , "db.json" ) ; private static readonly string DbFile = Path . Join ( DbDir , "db.json" ) ;
private static readonly string StationsFile = Path . Join ( DbDir , "stations.json" ) ; private static readonly string StationsFile = Path . Join ( DbDir , "stations.json" ) ;
private static readonly string TrainsFile = Path . Join ( DbDir , "trains.json" ) ; private static readonly string TrainsFile = Path . Join ( DbDir , "trains.json" ) ;
public IDisposable MakeDbTransaction ( ) { private readonly IMongoDatabase db ;
shouldCommitOnEveryChange = false ; private readonly IMongoCollection < DbRecord > dbRecordCollection ;
return new Server . Utils . ActionDisposable ( ( ) = > { private readonly IMongoCollection < TrainListing > trainListingsCollection ;
if ( dbDataDirty ) File . WriteAllText ( DbFile , JsonSerializer . Serialize ( DbData , serializerOptions ) ) ; private readonly IMongoCollection < StationListing > stationListingsCollection ;
if ( stationsDirty ) { private readonly IMongoCollection < StationAlias > stationAliasCollection ;
stations . Sort ( ( s1 , s2 ) = > s2 . StoppedAtBy . Count . CompareTo ( s1 . StoppedAtBy . Count ) ) ; private readonly AsyncThrottle throttle ;
File . WriteAllText ( StationsFile , JsonSerializer . Serialize ( stations , serializerOptions ) ) ;
}
if ( trainsDirty ) File . WriteAllText ( TrainsFile , JsonSerializer . Serialize ( trains , serializerOptions ) ) ;
dbDataDirty = stationsDirty = trainsDirty = false ;
shouldCommitOnEveryChange = true ;
} ) ;
}
public Database ( ILogger < Database > logger ) { private readonly Dictionary < string , string > trainObjectIds = new ( ) ;
private readonly Dictionary < string , string > stationObjectIds = new ( ) ;
public Database ( ILogger < Database > logger , IOptions < MongoSettings > mongoSettings ) {
Logger = logger ; Logger = logger ;
if ( ! Directory . Exists ( DbDir ) ) { var settings = MongoClientSettings . FromConnectionString ( mongoSettings . Value . ConnectionString ) ;
Logger . LogDebug ( "Creating directory: {DbDir}" , DbDir ) ; settings . ServerApi = new ( ServerApiVersion . V1 ) ;
Directory . CreateDirectory ( DbDir ) ; settings . MaxConnectionPoolSize = 1 0 0 0 0 ;
} MongoClient mongoClient = new ( settings ) ;
Logger . LogDebug ( "Created monogClient" ) ;
throttle = new ( mongoClient . Settings . MaxConnectionPoolSize / 2 ) ;
db = mongoClient . GetDatabase ( mongoSettings . Value . DatabaseName ) ? ? throw new NullReferenceException ( "Unable to get Mongo database" ) ;
Logger . LogDebug ( "Created db" ) ;
dbRecordCollection = db . GetCollection < DbRecord > ( "db" ) ;
trainListingsCollection = db . GetCollection < TrainListing > ( "trainListings" ) ;
stationListingsCollection = db . GetCollection < StationListing > ( "stationListings" ) ;
stationAliasCollection = db . GetCollection < StationAlias > ( "stationAliases" ) ;
Migration ( ) ; Migration ( ) ;
if ( File . Exists ( DbFile ) ) { Task . Run ( async ( ) = > await Initialize ( ) ) ;
DbData = JsonSerializer . Deserialize < DbRecord > ( File . ReadAllText ( DbFile ) , serializerOptions ) ! ;
}
else {
File . WriteAllText ( DbFile , JsonSerializer . Serialize ( DbData , serializerOptions ) ) ;
}
if ( File . Exists ( StationsFile ) ) {
stations = JsonSerializer . Deserialize < List < StationRecord > > ( File . ReadAllText ( StationsFile ) , serializerOptions ) ! ;
}
if ( File . Exists ( TrainsFile ) ) {
trains = JsonSerializer . Deserialize < List < TrainRecord > > ( File . ReadAllText ( TrainsFile ) , serializerOptions ) ! ;
}
} }
private void Migration ( ) { private void Migration ( ) {
if ( ! File . Exists ( DbFile ) ) { if ( ! File . Exists ( DbFile ) & & File . Exists ( TrainsFile ) ) {
// using var _ = Logger.BeginScope("Migrating DB version 1 -> 2");
Logger . LogInformation ( "Migrating DB version 1 -> 2" ) ; Logger . LogInformation ( "Migrating DB version 1 -> 2" ) ;
if ( File . Exists ( StationsFile ) ) { if ( File . Exists ( StationsFile ) ) {
Logger . LogDebug ( "Converting StationsFile" ) ; Logger . LogDebug ( "Converting StationsFile" ) ;
var oldStations = JsonNode . Parse ( File . ReadAllText ( StationsFile ) ) ; var oldStations = JToken . Parse ( File . ReadAllText ( StationsFile ) ) ;
List < StationListing > stations = new ( ) ;
if ( oldStations ! = null ) { if ( oldStations ! = null ) {
Logger . LogDebug ( "Found {StationsCount} stations" , oldStations . AsArray ( ) . Count ) ; Logger . LogDebug ( "Found {StationsCount} stations" , oldStations . Children ( ) . Count ( ) ) ;
foreach ( var station in oldStations . AsArray ( ) ) { foreach ( var station in oldStations . Children ( ) ) {
if ( station = = null ) continue ; if ( station = = null ) continue ;
station [ "stoppedAtBy" ] = new Json Array ( station [ "stoppedAtBy" ] ! . AsArray ( ) . Select ( num = > ( JsonNode ) ( num ! ) . ToString ( ) ! ) . ToArray ( ) ) ; station [ "stoppedAtBy" ] = new JArray ( station [ "stoppedAtBy" ] ! . Children ( ) . Select ( num = > ( JToken ) ( num ! ) . ToString ( ) ! ) . ToArray ( ) ) ;
} }
stations = JsonSerializer . Deserialize < List < StationRecord > > ( oldStations , serializerOptions ) ! ; stations = oldStations . ToObject < List < StationListing > > ( JsonSerializer . Create ( jsonSerializerSettings ) ) ! ;
} }
Logger . LogDebug ( "Rewriting StationsFile" ) ; Logger . LogDebug ( "Rewriting StationsFile" ) ;
File . WriteAllText ( StationsFile , JsonSerializer . Serialize ( stations , serializerOption s) ) ; File . WriteAllText ( StationsFile , JsonConvert . SerializeObject ( stations , jsonSerializerSetting s) ) ;
} }
if ( File . Exists ( TrainsFile ) ) { if ( File . Exists ( TrainsFile ) ) {
Logger . LogDebug ( "Converting TrainsFile" ) ; Logger . LogDebug ( "Converting TrainsFile" ) ;
var oldTrains = JsonNode . Parse ( File . ReadAllText ( TrainsFile ) ) ; var oldTrains = JToken . Parse ( File . ReadAllText ( TrainsFile ) ) ;
List < TrainListing > trains = new ( ) ;
if ( oldTrains ! = null ) { if ( oldTrains ! = null ) {
Logger . LogDebug ( "Found {TrainsCount} trains" , oldTrains . AsArray ( ) . Count ) ; Logger . LogDebug ( "Found {TrainsCount} trains" , oldTrains . Children ( ) . Count ( ) ) ;
foreach ( var train in oldTrains . AsArray ( ) ) { foreach ( var train in oldTrains . Children ( ) ) {
if ( train = = null ) continue ; if ( train = = null ) continue ;
train [ "number" ] = train [ "numberString" ] ; train [ "number" ] = train [ "numberString" ] ;
train . AsObject ( ) . Remove ( "numberString" ) ; train [ "numberString" ] ? . Remove ( ) ;
} }
trains = JsonSerializer . Deserialize < List < TrainRecord > > ( oldTrains , serializerOptions ) ! ; trains = oldTrains . ToObject < List < TrainListing > > ( JsonSerializer . Create ( jsonSerializerSettings ) ) ! ;
} }
Logger . LogDebug ( "Rewriting TrainsFile" ) ; Logger . LogDebug ( "Rewriting TrainsFile" ) ;
File . WriteAllText ( TrainsFile , JsonSerializer . Serialize ( trains , serializerOption s) ) ; File . WriteAllText ( TrainsFile , JsonConvert . SerializeObject ( trains , jsonSerializerSetting s) ) ;
} }
DbData = new ( 2 ) ; DbData = new ( 2 ) ;
File . WriteAllText ( DbFile , JsonSerializer . Serialize ( DbData , serializerOption s) ) ; File . WriteAllText ( DbFile , JsonConvert . SerializeObject ( DbData , jsonSerializerSetting s) ) ;
Migration ( ) ; Migration ( ) ;
} }
else { else if ( File . Exists ( DbFile ) ) {
var oldDbData = JsonNode . Parse ( File . ReadAllText ( DbFile ) ) ; var oldDbData = JToken . Parse ( File . ReadAllText ( DbFile ) ) ;
if ( ( ( int? ) oldDbData ? [ "version" ] ) = = 2 ) { if ( ( ( int? ) oldDbData ? [ "version" ] ) = = 2 ) {
Logger . LogInformation ( "DB Version: 2; noop" ) ; Logger . LogInformation ( "Migrating DB version 2 -> 3 (transition from fs+JSON to MongoDB)" ) ;
if ( File . Exists ( StationsFile ) ) {
Logger . LogDebug ( "Converting StationsFile" ) ;
var stations = JsonConvert . DeserializeObject < List < StationListing > > ( File . ReadAllText ( StationsFile ) ) ;
stationListingsCollection . InsertMany ( stations ) ;
File . Delete ( StationsFile ) ;
}
if ( File . Exists ( TrainsFile ) ) {
Logger . LogDebug ( "Converting TrainsFile" ) ;
var trains = JsonConvert . DeserializeObject < List < TrainListing > > ( File . ReadAllText ( TrainsFile ) ) ;
trainListingsCollection . InsertMany ( trains ) ;
File . Delete ( TrainsFile ) ;
}
File . Delete ( DbFile ) ;
try {
Directory . Delete ( DbDir ) ;
}
catch ( Exception ) {
// Deleting of the directory is optional; may not be allowed in Docker or similar
}
var x = dbRecordCollection . FindSync ( _ = > true ) . ToList ( ) ! ;
if ( x . Count ! = 0 ) {
Logger . LogWarning ( "db collection contained data when migrating to V3" ) ;
using ( var _ = Logger . BeginScope ( "Already existing data:" ) ) {
foreach ( var dbRecord in x ) {
Logger . LogInformation ( "Id: {Id}, Version: {Version}" , dbRecord . Id , dbRecord . Version ) ;
}
}
Logger . LogInformation ( "Backing up existing data" ) ;
var backupDbRecordCollection = db . GetCollection < DbRecord > ( "db-backup" ) ;
backupDbRecordCollection . InsertMany ( x ) ;
Logger . LogDebug ( "Removing existing data" ) ;
dbRecordCollection . DeleteMany ( _ = > true ) ;
}
dbRecordCollection . InsertOne ( new ( 3 ) ) ;
Migration ( ) ;
}
else {
throw new ( "Unexpected Database version, only DB Version 2 uses DbFile" ) ;
}
}
else {
var datas = dbRecordCollection . FindSync ( _ = > true ) . ToList ( ) ;
if ( datas . Count = = 0 ) {
Logger . LogInformation ( "No db record found, new database" ) ;
dbRecordCollection . InsertOne ( DbData ) ;
} }
else { else {
throw new Exception ( "Unexpected Database version" ) ; DbData = datas [ 0 ] ;
}
if ( DbData . Version = = 3 ) {
Logger . LogInformation ( "Using MongoDB Database Version 3; noop" ) ;
}
else {
throw new ( $"Unexpected Database version: {DbData.Version}" ) ;
} }
} }
} }
private async Task Initialize ( ) {
await foreach ( var entry in await stationAliasCollection . FindAsync ( _ = > true ) ) {
if ( entry ? . ListingId is null ) continue ;
stationObjectIds . Add ( entry . Name , entry . ListingId ) ;
}
}
private readonly SemaphoreSlim insertTrainLock = new ( 1 , 1 ) ;
public async Task < string > FoundTrain ( string rank , string number , string company ) { public async Task < string > FoundTrain ( string rank , string number , string company ) {
number = string . Join ( "" , number . TakeWhile ( c = > '0' < = c & & c < = '9' ) ) ; number = string . Join ( "" , number . TakeWhile ( c = > c is > = '0' and < = '9' ) ) ;
if ( ! trains . Where ( train = > train . Number = = number ) . Any ( ) ) { // If there is a matching ObjectId, then it's already in the database
if ( trainObjectIds . ContainsKey ( number ) ) return number ;
await insertTrainLock . WaitAsync ( ) ;
try {
var possibleTrains = await ( await throttle . MakeRequest ( ( ) = > trainListingsCollection . FindAsync (
Builders < TrainListing > . Filter . Eq ( "number" , number )
) ) ) . ToListAsync ( ) ;
if ( possibleTrains . Count = = 0 ) {
Logger . LogDebug ( "Found train {Rank} {Number} from {Company}" , rank , number , company ) ; Logger . LogDebug ( "Found train {Rank} {Number} from {Company}" , rank , number , company ) ;
trains . Add ( new ( number , rank , company ) ) ; TrainListing listing = new ( number : number , rank : rank , company : company ) ;
if ( shouldCommitOnEveryChange ) { await throttle . MakeRequest ( ( ) = > trainListingsCollection . InsertOneAsync ( listing ) ) ;
await File . WriteAllTextAsync ( TrainsFile , JsonSerializer . Serialize ( trains , serializerOptions ) ) ; if ( listing . Id ! = null ) {
trainObjectIds [ number ] = listing . Id ;
}
} }
else { else {
trainsDirty = true ; foreach ( var possibleTrain in possibleTrains ) {
trainObjectIds [ possibleTrain . Number ] = possibleTrain . Id ! ;
}
}
} }
finally {
insertTrainLock . Release ( ) ;
} }
return number ; return number ;
} }
private readonly SemaphoreSlim insertStationLock = new ( 1 , 1 ) ;
public async Task FoundStation ( string name ) { public async Task FoundStation ( string name ) {
if ( ! stations . Where ( station = > station . Name = = name ) . Any ( ) ) { // if (!await throttle.MakeRequest(() => stationListingsCollection.Find(Builders<StationListing>.Filter.Eq("name", name)).AnyAsync())) {
// Logger.LogDebug("Found station {StationName}", name);
// await throttle.MakeRequest(() => stationListingsCollection.InsertOneAsync(new(name, new())));
// }
// If there is a matching ObjectId, then it's already in the database
if ( stationObjectIds . ContainsKey ( name ) ) return ;
await insertStationLock . WaitAsync ( ) ;
UpdateResult update ;
try {
update = await stationListingsCollection . UpdateOneAsync (
Builders < StationListing > . Filter . Eq ( "name" , name ) ,
Builders < StationListing > . Update . Combine (
Builders < StationListing > . Update . SetOnInsert ( "name" , name ) ,
Builders < StationListing > . Update . SetOnInsert ( "stoppedAtBy" , new List < string > ( ) )
) ,
new UpdateOptions {
IsUpsert = true ,
}
) ;
if ( update . IsAcknowledged & & update . ModifiedCount > 0 ) {
var listingId = update . UpsertedId . AsObjectId . ToString ( ) ;
stationObjectIds [ name ] = listingId ;
await stationAliasCollection . UpdateOneAsync (
Builders < StationAlias > . Filter . Eq ( "name" , name ) ,
Builders < StationAlias > . Update . Combine (
Builders < StationAlias > . Update . SetOnInsert ( "name" , name ) ,
Builders < StationAlias > . Update . SetOnInsert ( "listingId" , listingId )
) ,
new UpdateOptions { IsUpsert = true }
) ;
}
}
finally {
insertStationLock . Release ( ) ;
}
if ( update . IsAcknowledged & & update . MatchedCount = = 0 ) {
Logger . LogDebug ( "Found station {StationName}" , name ) ; Logger . LogDebug ( "Found station {StationName}" , name ) ;
stations . Add ( new ( name , new ( ) ) ) ;
if ( shouldCommitOnEveryChange ) {
await File . WriteAllTextAsync ( StationsFile , JsonSerializer . Serialize ( stations , serializerOptions ) ) ;
} }
else {
stationsDirty = true ;
} }
public async Task FoundStations ( IEnumerable < string > names ) {
var unknownStations = names . ToList ( ) ;
if ( unknownStations . All ( s = > stationObjectIds . ContainsKey ( s ) ) ) {
return ;
}
unknownStations . RemoveAll ( s = > stationObjectIds . ContainsKey ( s ) ) ;
var existingStations = await ( await stationListingsCollection . FindAsync (
Builders < StationListing > . Filter . StringIn ( "name" , unknownStations . Select ( ( n ) = > new StringOrRegularExpression ( n ) ) )
) ) . ToListAsync ( ) ;
foreach ( var existingStation in existingStations ) {
stationObjectIds [ existingStation . Name ] = existingStation . Id ! ;
}
unknownStations . RemoveAll ( s = > existingStations . Select ( st = > st . Name ) . Contains ( s ) ) ;
if ( unknownStations . Count = = 0 ) return ;
var unknownStationListings = unknownStations . Select ( ( s ) = > new StationListing ( s , new ( ) ) ) . ToList ( ) ;
await stationListingsCollection . InsertManyAsync ( unknownStationListings ) ;
foreach ( var listing in unknownStationListings ) {
stationObjectIds [ listing . Name ] = listing . Id ! ;
} }
Logger . LogDebug ( "Found stations {StationNames}" , unknownStations ) ;
} }
public async Task FoundTrainAtStation ( string stationName , string trainNumber ) { public async Task FoundTrainAtStation ( string stationName , string trainNumber ) {
trainNumber = string . Join ( "" , trainNumber . TakeWhile ( c = > '0' < = c & & c < = '9' ) ) ; trainNumber = string . Join ( "" , trainNumber . TakeWhile ( c = > c is > = '0' and < = '9' ) ) ;
await FoundStation ( stationName ) ; await FoundStation ( stationName ) ;
var dirty = false ; UpdateResult updateResult ;
for ( var i = 0 ; i < stations . Count ; i + + ) { if ( stationObjectIds . ContainsKey ( stationName ) ) {
if ( stations [ i ] . Name = = stationName ) { updateResult = await throttle . MakeRequest ( ( ) = > stationListingsCollection . UpdateOneAsync (
if ( ! stations [ i ] . StoppedAtBy . Contains ( trainNumber ) ) { Builders < StationListing > . Filter . Eq ( "_id" , ObjectId . Parse ( stationObjectIds [ stationName ] ) ) ,
Logger . LogDebug ( "Found train {TrainNumber} at station {StationName}" , trainNumber , stationName ) ; Builders < StationListing > . Update . AddToSet ( "stoppedAtBy" , trainNumber )
stations [ i ] . ActualStoppedAtBy . Add ( trainNumber ) ; ) ) ;
dirty = true ;
} }
break ; else {
updateResult = await throttle . MakeRequest ( ( ) = > stationListingsCollection . UpdateOneAsync (
Builders < StationListing > . Filter . Eq ( "name" , stationName ) ,
Builders < StationListing > . Update . AddToSet ( "stoppedAtBy" , trainNumber )
) ) ;
} }
if ( updateResult . IsAcknowledged & & updateResult . ModifiedCount > 0 ) {
Logger . LogDebug ( "Found train {TrainNumber} at station {StationName}" , trainNumber , stationName ) ;
} }
if ( dirty ) { }
if ( shouldCommitOnEveryChange ) {
stations . Sort ( ( s1 , s2 ) = > s2 . StoppedAtBy . Count . CompareTo ( s1 . StoppedAtBy . Count ) ) ; public async Task FoundTrainAtStations ( IEnumerable < string > stationNames , string trainNumber ) {
await File . WriteAllTextAsync ( StationsFile , JsonSerializer . Serialize ( stations , serializerOptions ) ) ; trainNumber = string . Join ( "" , trainNumber . TakeWhile ( c = > c is > = '0' and < = '9' ) ) ;
var enumerable = stationNames as string [ ] ? ? stationNames . ToArray ( ) ;
await FoundStations ( enumerable ) ;
var objectIds = enumerable
. Select < string , ObjectId ? > ( ( stationName ) = > stationObjectIds . ContainsKey ( stationName ) ? ObjectId . Parse ( stationObjectIds [ stationName ] ) : null )
. ToList ( ) ;
UpdateResult updateResult ;
if ( ! objectIds . Any ( ( id ) = > id is null ) ) {
updateResult = await throttle . MakeRequest ( ( ) = > stationListingsCollection . UpdateManyAsync (
Builders < StationListing > . Filter . In ( "_id" , objectIds ) ,
Builders < StationListing > . Update . AddToSet ( "stoppedAtBy" , trainNumber )
) ) ;
} }
else { else {
stationsDirty = true ; updateResult = await throttle . MakeRequest ( ( ) = > stationListingsCollection . UpdateManyAsync (
Builders < StationListing > . Filter . StringIn ( "name" , enumerable . Select ( sn = > new StringOrRegularExpression ( sn ) ) ) ,
Builders < StationListing > . Update . AddToSet ( "stoppedAtBy" , trainNumber )
) ) ;
} }
if ( updateResult . IsAcknowledged & & updateResult . ModifiedCount > 0 ) {
Logger . LogDebug ( "Found train {TrainNumber} at stations {StationNames}" , trainNumber , stationNames ) ;
} }
} }
public async Task OnTrainData ( InfoferScraper . Models . Train . ITrainScrapeResult trainData ) { public async Task OnTrainData ( InfoferScraper . Models . Train . ITrainScrapeResult trainData ) {
using var _ = MakeDbTransaction ( ) ;
var trainNumber = await FoundTrain ( trainData . Rank , trainData . Number , trainData . Operator ) ; var trainNumber = await FoundTrain ( trainData . Rank , trainData . Number , trainData . Operator ) ;
foreach ( var group in trainData . Groups ) { await FoundTrainAtStations (
foreach ( var station in group . Stations ) { trainData . Groups
await FoundTrainAtStation ( station . Name , trainNumber ) ; . SelectMany ( g = > g . Stations )
} . Select ( trainStop = > trainStop . Name )
} . Distinct ( ) ,
trainNumber
) ;
} }
public async Task OnStationData ( InfoferScraper . Models . Station . IStationScrapeResult stationData ) { public async Task OnStationData ( InfoferScraper . Models . Station . IStationScrapeResult stationData ) {
@ -191,48 +351,40 @@ public class Database : Server.Services.Interfaces.IDatabase {
async Task ProcessTrain ( InfoferScraper . Models . Station . IStationArrDep train ) { async Task ProcessTrain ( InfoferScraper . Models . Station . IStationArrDep train ) {
var trainNumber = train . Train . Number ; var trainNumber = train . Train . Number ;
trainNumber = await FoundTrain ( train . Train . Rank , trainNumber , train . Train . Operator ) ; trainNumber = await FoundTrain ( train . Train . Rank , trainNumber , train . Train . Operator ) ;
await FoundTrainAtStation ( stationName , trainNumber ) ; await FoundTrainAtStations ( Enumerable . Repeat ( stationName , 1 ) . Concat ( train . Train . Route ) . Distinct ( ) , trainNumber ) ;
if ( train . Train . Route . Count ! = 0 ) {
foreach ( var station in train . Train . Route ) {
await FoundTrainAtStation ( station , trainNumber ) ;
}
} }
}
using var _ = MakeDbTransaction ( ) ;
List < IStationArrDep > arrdep = new ( ) ;
if ( stationData . Arrivals ! = null ) { if ( stationData . Arrivals ! = null ) {
foreach ( var train in stationData . Arrivals ) { arrdep . AddRange ( stationData . Arrivals ) ;
await ProcessTrain ( train ) ;
}
} }
if ( stationData . Departures ! = null ) { if ( stationData . Departures ! = null ) {
foreach ( var train in stationData . Departures ) { arrdep . AddRange ( stationData . Departures ) ;
await ProcessTrain ( train ) ;
} }
foreach ( var train in arrdep . DistinctBy ( ( t ) = > t . Train . Number ) ) {
await ProcessTrain ( train ) ;
} }
} }
}
public record DbRecord ( int Version ) ;
public record StationRecord : Server . Services . Interfaces . IStationRecord {
[JsonPropertyName("stoppedAtBy")]
public List < string > ActualStoppedAtBy { get ; init ; }
public string Name { get ; init ; } public async Task OnItineraries ( IReadOnlyList < IItinerary > itineraries ) {
[JsonIgnore] foreach ( var itinerary in itineraries ) {
public IReadOnlyList < string > StoppedAtBy = > ActualStoppedAtBy ; foreach ( var train in itinerary . Trains ) {
await FoundTrainAtStations (
public StationRecord ( ) { train . IntermediateStops . Concat ( new [ ] { train . From , train . To } ) ,
Name = "" ; train . TrainNumber
ActualStoppedAtBy = new ( ) ; ) ;
}
} }
public StationRecord ( string name , List < string > stoppedAtBy ) {
Name = name ;
ActualStoppedAtBy = stoppedAtBy ;
} }
} }
public record TrainRecord ( string Number , string Rank , string Company ) : Server . Services . Interfaces . ITrainRecord ; public record DbRecord (
[property: BsonId]
[property: BsonRepresentation(BsonType.ObjectId)]
[property: JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
string? Id ,
int Version
) {
public DbRecord ( int version ) : this ( null , version ) { }
}
Kenneth Bruen