Transition from fs+JSON to MongoDB

server/Controllers/V2/StationsController.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using Microsoft.AspNetCore.Mvc;
+using Server.Models.Database;
 using Server.Services.Interfaces;
 
 namespace Server.Controllers.V2;
@@ -15,7 +16,7 @@ public class StationsController : Controller {
 	}
 
 	[HttpGet("")]
-	public ActionResult<IEnumerable<IStationRecord>> ListStations() {
+	public ActionResult<IEnumerable<StationListing>> ListStations() {
 		return Ok(Database.Stations);
 	}
 }

server/Controllers/V2/TrainsController.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using Microsoft.AspNetCore.Mvc;
+using Server.Models.Database;
 using Server.Services.Interfaces;
 
 namespace Server.Controllers.V2;
@@ -15,7 +16,7 @@ public class TrainsController : Controller {
 	}
 
 	[HttpGet("")]
-	public ActionResult<IEnumerable<ITrainRecord>> ListTrains() {
+	public ActionResult<IEnumerable<TrainListing>> ListTrains() {
 		return Ok(Database.Trains);
 	}
 }

server/Controllers/V3/StationsController.cs

@@ -4,6 +4,7 @@ using System.Threading.Tasks;
 using InfoferScraper.Models.Station;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
+using Server.Models.Database;
 using Server.Services.Interfaces;
 
 namespace Server.Controllers.V3;
@@ -21,7 +22,7 @@ public class StationsController : Controller {
 	}
 
 	[HttpGet("")]
-	public ActionResult<IEnumerable<IStationRecord>> ListStations() {
+	public ActionResult<IEnumerable<StationListing>> ListStations() {
 		return Ok(Database.Stations);
 	}
 

server/Controllers/V3/TrainsController.cs

@@ -5,6 +5,7 @@ using InfoferScraper.Models.Train;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
 using scraper.Exceptions;
+using Server.Models.Database;
 using Server.Services.Interfaces;
 
 namespace Server.Controllers.V3;
@@ -22,7 +23,7 @@ public class TrainsController : Controller {
 	}
 
 	[HttpGet("")]
-	public ActionResult<IEnumerable<ITrainRecord>> ListTrains() {
+	public ActionResult<IEnumerable<TrainListing>> ListTrains() {
 		return Ok(Database.Trains);
 	}
 

server/Models/Database/MongoSettings.cs

@@ -0,0 +1,5 @@
+namespace Server.Models.Database;
+
+public record MongoSettings(string ConnectionString, string DatabaseName) {
+    public MongoSettings() : this("", "") { }
+}

server/Models/Database/StationListing.cs

@@ -0,0 +1,17 @@
+using System.Collections.Generic;
+using System.Text.Json.Serialization;
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Attributes;
+
+namespace Server.Models.Database;
+
+public record StationListing(
+    [property: BsonId]
+    [property: BsonRepresentation(BsonType.ObjectId)]
+    [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    string? Id,
+    string Name,
+    List<string> StoppedAtBy
+) {
+    public StationListing(string name, List<string> stoppedAtBy) : this(null, name, stoppedAtBy) { }
+}

server/Models/Database/TrainListing.cs

@@ -0,0 +1,17 @@
+using System.Text.Json.Serialization;
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Attributes;
+
+namespace Server.Models.Database;
+
+public record TrainListing(
+    [property: BsonId]
+    [property: BsonRepresentation(BsonType.ObjectId)]
+    [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    string? Id,
+    string Rank,
+    string Number,
+    string Company
+) {
+    public TrainListing(string rank, string number, string company) : this(null, rank, number, company) { }
+}

server/Services/Implementations/Database.cs

@@ -7,6 +7,11 @@ using System.Text.Json.Nodes;
 using System.Text.Json.Serialization;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Attributes;
+using MongoDB.Driver;
+using Server.Models.Database;
 
 namespace Server.Services.Implementations;
 
@@ -17,77 +22,47 @@ public class Database : Server.Services.Interfaces.IDatabase {
 
 	private ILogger<Database> Logger { get; }
 
-	private bool shouldCommitOnEveryChange = true;
+	public DbRecord DbData { get; private set; } = new(3);
-	private bool dbDataDirty = false;
-	private bool stationsDirty = false;
-	private bool trainsDirty = false;
 
-	public DbRecord DbData { get; private set; } = new(2);
+	public IReadOnlyList<StationListing> Stations => stationListingsCollection.FindSync(_ => true).ToList();
-	private List<StationRecord> stations = new();
+	public IReadOnlyList<TrainListing> Trains => trainListingsCollection.FindSync(_ => true).ToList();
-	private List<TrainRecord> trains = new();
-
-	public IReadOnlyList<Server.Services.Interfaces.IStationRecord> Stations => stations;
-	public IReadOnlyList<Server.Services.Interfaces.ITrainRecord> Trains => trains;
 
 	private static readonly string DbDir = Environment.GetEnvironmentVariable("DB_DIR") ?? Path.Join(Environment.CurrentDirectory, "db");
 	private static readonly string DbFile = Path.Join(DbDir, "db.json");
 	private static readonly string StationsFile = Path.Join(DbDir, "stations.json");
 	private static readonly string TrainsFile = Path.Join(DbDir, "trains.json");
 
-	public IDisposable MakeDbTransaction() {
+	private readonly IMongoDatabase db;
-		shouldCommitOnEveryChange = false;
+	private readonly IMongoCollection<DbRecord> dbRecordCollection;
-		return new Server.Utils.ActionDisposable(() => {
+	private readonly IMongoCollection<TrainListing> trainListingsCollection;
-			if (dbDataDirty) File.WriteAllText(DbFile, JsonSerializer.Serialize(DbData, serializerOptions));
+	private readonly IMongoCollection<StationListing> stationListingsCollection;
-			if (stationsDirty) {
-				stations.Sort((s1, s2) => s2.StoppedAtBy.Count.CompareTo(s1.StoppedAtBy.Count));
-				File.WriteAllText(StationsFile, JsonSerializer.Serialize(stations, serializerOptions));
-			}
-			if (trainsDirty) File.WriteAllText(TrainsFile, JsonSerializer.Serialize(trains, serializerOptions));
-			dbDataDirty = stationsDirty = trainsDirty = false;
-			shouldCommitOnEveryChange = true;
-		});
-	}
 
-	public Database(ILogger<Database> logger) {
+	public Database(ILogger<Database> logger, IOptions<MongoSettings> mongoSettings) {
 		Logger = logger;
 
-		if (!Directory.Exists(DbDir)) {
+		MongoClient mongoClient = new(mongoSettings.Value.ConnectionString);
-			Logger.LogDebug("Creating directory: {DbDir}", DbDir);
+		db = mongoClient.GetDatabase(mongoSettings.Value.DatabaseName) ?? throw new NullReferenceException("Unable to get Mongo database");
-			Directory.CreateDirectory(DbDir);
+		dbRecordCollection = db.GetCollection<DbRecord>("db");
-		}
+		trainListingsCollection = db.GetCollection<TrainListing>("trainListings");
+		stationListingsCollection = db.GetCollection<StationListing>("stationListings");
 
 		Migration();
-
-		if (File.Exists(DbFile)) {
-			DbData = JsonSerializer.Deserialize<DbRecord>(File.ReadAllText(DbFile), serializerOptions)!;
-		}
-		else {
-			File.WriteAllText(DbFile, JsonSerializer.Serialize(DbData, serializerOptions));
-		}
-
-		if (File.Exists(StationsFile)) {
-			stations = JsonSerializer.Deserialize<List<StationRecord>>(File.ReadAllText(StationsFile), serializerOptions)!;
-		}
-
-		if (File.Exists(TrainsFile)) {
-			trains = JsonSerializer.Deserialize<List<TrainRecord>>(File.ReadAllText(TrainsFile), serializerOptions)!;
-		}
 	}
 
 	private void Migration() {
-		if (!File.Exists(DbFile)) {
+		if (!File.Exists(DbFile) && File.Exists(TrainsFile)) {
-// 			using var _ = Logger.BeginScope("Migrating DB version 1 -> 2");
 			Logger.LogInformation("Migrating DB version 1 -> 2");
 			if (File.Exists(StationsFile)) {
 				Logger.LogDebug("Converting StationsFile");
 				var oldStations = JsonNode.Parse(File.ReadAllText(StationsFile));
+				List<StationListing> stations = new();
 				if (oldStations != null) {
 					Logger.LogDebug("Found {StationsCount} stations", oldStations.AsArray().Count);
 					foreach (var station in oldStations.AsArray()) {
 						if (station == null) continue;
 						station["stoppedAtBy"] = new JsonArray(station["stoppedAtBy"]!.AsArray().Select(num => (JsonNode)(num!).ToString()!).ToArray());
 					}
-					stations = JsonSerializer.Deserialize<List<StationRecord>>(oldStations, serializerOptions)!;
+					stations = oldStations.Deserialize<List<StationListing>>(serializerOptions)!;
 				}
 				Logger.LogDebug("Rewriting StationsFile");
 				File.WriteAllText(StationsFile, JsonSerializer.Serialize(stations, serializerOptions));
@@ -95,6 +70,7 @@ public class Database : Server.Services.Interfaces.IDatabase {
 			if (File.Exists(TrainsFile)) {
 				Logger.LogDebug("Converting TrainsFile");
 				var oldTrains = JsonNode.Parse(File.ReadAllText(TrainsFile));
+				List<TrainListing> trains = new();
 				if (oldTrains != null) {
 					Logger.LogDebug("Found {TrainsCount} trains", oldTrains.AsArray().Count);
 					foreach (var train in oldTrains.AsArray()) {
@@ -102,7 +78,7 @@ public class Database : Server.Services.Interfaces.IDatabase {
 						train["number"] = train["numberString"];
 						train.AsObject().Remove("numberString");
 					}
-					trains = JsonSerializer.Deserialize<List<TrainRecord>>(oldTrains, serializerOptions)!;
+					trains = oldTrains.Deserialize<List<TrainListing>>(serializerOptions)!;
 				}
 				Logger.LogDebug("Rewriting TrainsFile");
 				File.WriteAllText(TrainsFile, JsonSerializer.Serialize(trains, serializerOptions));
@@ -111,72 +87,101 @@ public class Database : Server.Services.Interfaces.IDatabase {
 			File.WriteAllText(DbFile, JsonSerializer.Serialize(DbData, serializerOptions));
 			Migration();
 		}
-		else {
+		else if (File.Exists(DbFile)) {
 			var oldDbData = JsonNode.Parse(File.ReadAllText(DbFile));
 			if (((int?)oldDbData?["version"]) == 2) {
-				Logger.LogInformation("DB Version: 2; noop");
+				Logger.LogInformation("Migrating DB version 2 -> 3 (transition from fs+JSON to MongoDB)");
+
+				if (File.Exists(StationsFile)) {
+					Logger.LogDebug("Converting StationsFile");
+					var stations = JsonSerializer.Deserialize<List<StationListing>>(File.ReadAllText(StationsFile));
+					stationListingsCollection.InsertMany(stations);
+					File.Delete(StationsFile);
 				}
-			else {
+
-				throw new Exception("Unexpected Database version");
+				if (File.Exists(TrainsFile)) {
+					Logger.LogDebug("Converting TrainsFile");
+					var trains = JsonSerializer.Deserialize<List<TrainListing>>(File.ReadAllText(TrainsFile));
+					trainListingsCollection.InsertMany(trains);
+					File.Delete(TrainsFile);
 				}
+				
+				File.Delete(DbFile);
+				try {
+					Directory.Delete(DbDir);
 				}
+				catch (Exception) {
+					// Deleting of the directory is optional; may not be allowed in Docker or similar
 				}
 
-	public async Task<string> FoundTrain(string rank, string number, string company) {
+				var x = dbRecordCollection.FindSync(_ => true).ToList()!;
-		number = string.Join("", number.TakeWhile(c => '0' <= c && c <= '9'));
+				if (x.Count != 0) {
-		if (!trains.Where(train => train.Number == number).Any()) {
+					Logger.LogWarning("db collection contained data when migrating to V3");
-			Logger.LogDebug("Found train {Rank} {Number} from {Company}", rank, number, company);
+					using (var _ = Logger.BeginScope("Already existing data:")) {
-			trains.Add(new(number, rank, company));
+						foreach (var dbRecord in x) {
-			if (shouldCommitOnEveryChange) {
+							Logger.LogInformation("Id: {Id}, Version: {Version}", dbRecord.Id, dbRecord.Version);
-				await File.WriteAllTextAsync(TrainsFile, JsonSerializer.Serialize(trains, serializerOptions));
+						}
+					}
+					Logger.LogInformation("Backing up existing data");
+					var backupDbRecordCollection = db.GetCollection<DbRecord>("db-backup");
+					backupDbRecordCollection.InsertMany(x);
+					Logger.LogDebug("Removing existing data");
+					dbRecordCollection.DeleteMany(_ => true);
+				}
+				dbRecordCollection.InsertOne(new(3));
+				Migration();
 			}
 			else {
-				trainsDirty = true;
+				throw new("Unexpected Database version, only DB Version 2 uses DbFile");
 			}
 		}
-		return number;
+		else {
+			var datas = dbRecordCollection.FindSync(_ => true).ToList();
+			if (datas.Count == 0) {
+				Logger.LogInformation("No db record found, new database");
+				dbRecordCollection.InsertOne(DbData);
 			}
-
+			else {
-	public async Task FoundStation(string name) {
+				DbData = datas[0];
-		if (!stations.Where(station => station.Name == name).Any()) {
+			}
-			Logger.LogDebug("Found station {StationName}", name);
+			if (DbData.Version == 3) {
-			stations.Add(new(name, new()));
+				Logger.LogInformation("Using MongoDB Database Version 3; noop");
-			if (shouldCommitOnEveryChange) {
-				await File.WriteAllTextAsync(StationsFile, JsonSerializer.Serialize(stations, serializerOptions));
 			}
 			else {
-				stationsDirty = true;
+				throw new($"Unexpected Database version: {DbData.Version}");
 			}
 		}
 	}
 
-	public async Task FoundTrainAtStation(string stationName, string trainNumber) {
+	public async Task<string> FoundTrain(string rank, string number, string company) {
-		trainNumber = string.Join("", trainNumber.TakeWhile(c => '0' <= c && c <= '9'));
+		number = string.Join("", number.TakeWhile(c => c is >= '0' and <= '9'));
-		await FoundStation(stationName);
+		if (!await (await trainListingsCollection.FindAsync(Builders<TrainListing>.Filter.Eq("number", number))).AnyAsync()) {
-		var dirty = false;
+			Logger.LogDebug("Found train {Rank} {Number} from {Company}", rank, number, company);
-		for (var i = 0; i < stations.Count; i++) {
+			await trainListingsCollection.InsertOneAsync(new(number: number, rank: rank, company: company));
-			if (stations[i].Name == stationName) {
-				if (!stations[i].StoppedAtBy.Contains(trainNumber)) {
-					Logger.LogDebug("Found train {TrainNumber} at station {StationName}", trainNumber, stationName);
-					stations[i].ActualStoppedAtBy.Add(trainNumber);
-					dirty = true;
-				}
-				break;
 		}
+		return number;
 	}
-		if (dirty) {
+
-			if (shouldCommitOnEveryChange) {
+	public async Task FoundStation(string name) {
-				stations.Sort((s1, s2) => s2.StoppedAtBy.Count.CompareTo(s1.StoppedAtBy.Count));
+		if (!await (stationListingsCollection.Find(Builders<StationListing>.Filter.Eq("name", name))).AnyAsync()) {
-				await File.WriteAllTextAsync(StationsFile, JsonSerializer.Serialize(stations, serializerOptions));
+			Logger.LogDebug("Found station {StationName}", name);
+			await stationListingsCollection.InsertOneAsync(new(name, new()));
 		}
-			else {
-				stationsDirty = true;
 	}
+
+	public async Task FoundTrainAtStation(string stationName, string trainNumber) {
+		trainNumber = string.Join("", trainNumber.TakeWhile(c => c is >= '0' and <= '9'));
+		await FoundStation(stationName);
+		var updateResult = await stationListingsCollection.UpdateOneAsync(
+			Builders<StationListing>.Filter.Eq("name", stationName),
+			Builders<StationListing>.Update.AddToSet("stoppedAtBy", trainNumber)
+		);
+		if (updateResult.IsAcknowledged && updateResult.ModifiedCount > 0) {
+			Logger.LogDebug("Found train {TrainNumber} at station {StationName}", trainNumber, stationName);
 		}
 	}
 
 	public async Task OnTrainData(InfoferScraper.Models.Train.ITrainScrapeResult trainData) {
-		using var _ = MakeDbTransaction();
 		var trainNumber = await FoundTrain(trainData.Rank, trainData.Number, trainData.Operator);
 		foreach (var group in trainData.Groups) {
 			foreach (var station in group.Stations) {
@@ -199,8 +204,6 @@ public class Database : Server.Services.Interfaces.IDatabase {
 			}
 		}
 
-		using var _ = MakeDbTransaction();
-
 		if (stationData.Arrivals != null) {
 			foreach (var train in stationData.Arrivals) {
 				await ProcessTrain(train);
@@ -214,25 +217,12 @@ public class Database : Server.Services.Interfaces.IDatabase {
 	}
 }
 
-public record DbRecord(int Version);
+public record DbRecord(
-
+	[property: BsonId]
-public record StationRecord : Server.Services.Interfaces.IStationRecord {
+	[property: BsonRepresentation(BsonType.ObjectId)]
-	[JsonPropertyName("stoppedAtBy")]
+	[property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-	public List<string> ActualStoppedAtBy { get; init; }
+	string? Id,
-
+	int Version
-	public string Name { get; init; }
+) {
-	[JsonIgnore]
+	public DbRecord(int version) : this(null, version) { }
-	public IReadOnlyList<string> StoppedAtBy => ActualStoppedAtBy;
-
-	public StationRecord() {
-		Name = "";
-		ActualStoppedAtBy = new();
 }
-
-	public StationRecord(string name, List<string> stoppedAtBy) {
-		Name = name;
-		ActualStoppedAtBy = stoppedAtBy;
-	}
-}
-
-public record TrainRecord(string Number, string Rank, string Company) : Server.Services.Interfaces.ITrainRecord;

server/Services/Interfaces/IDatabase.cs

@@ -2,12 +2,13 @@ using System.Collections.Generic;
 using System.Threading.Tasks;
 using InfoferScraper.Models.Train;
 using InfoferScraper.Models.Station;
+using Server.Models.Database;
 
 namespace Server.Services.Interfaces;
 
 public interface IDatabase {
-	public IReadOnlyList<IStationRecord> Stations { get; }
+	public IReadOnlyList<StationListing> Stations { get; }
-	public IReadOnlyList<ITrainRecord> Trains { get; }
+	public IReadOnlyList<TrainListing> Trains { get; }
 
 	public Task<string> FoundTrain(string rank, string number, string company);
 	public Task FoundStation(string name);
@@ -15,14 +16,3 @@ public interface IDatabase {
 	public Task OnTrainData(ITrainScrapeResult trainData);
 	public Task OnStationData(IStationScrapeResult stationData);
 }
-
-public interface IStationRecord {
-	public string Name { get; }
-	public IReadOnlyList<string> StoppedAtBy { get; }
-}
-
-public interface ITrainRecord {
-	public string Rank { get; }
-	public string Number { get; }
-	public string Company { get; }
-}

server/Startup.cs

@@ -8,6 +8,8 @@ using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.OpenApi.Models;
+using MongoDB.Bson.Serialization.Conventions;
+using Server.Models.Database;
 using Server.Services.Implementations;
 using Server.Services.Interfaces;
 
@@ -26,6 +28,10 @@ namespace Server {
 					options.KnownProxies.Add(Dns.GetHostAddresses("host.docker.internal")[0]);
 				});
 			}
+
+			services.Configure<MongoSettings>(Configuration.GetSection("TrainDataMongo"));
+			var conventionPack = new ConventionPack { new CamelCaseElementNameConvention() };
+			ConventionRegistry.Register("camelCase", conventionPack, _ => true);
 			services.AddSingleton<IDataManager, DataManager>();
 			services.AddSingleton<IDatabase, Database>();
 			services.AddSingleton<NodaTime.IDateTimeZoneProvider>(NodaTime.DateTimeZoneProviders.Tzdb);

server/appsettings.Development.json

@@ -5,5 +5,9 @@
       "Microsoft": "Warning",
       "Microsoft.Hosting.Lifetime": "Information"
     }
-  }
+  },
+  "TrainDataMongo": {
+    "ConnectionString": "mongodb://localhost:27017",
+    "DatabaseName": "NewInfoferScraper"
+  },
 }

server/appsettings.json

@@ -9,5 +9,9 @@
       "Microsoft.Hosting.Lifetime": "Information"
     }
   },
+  "TrainDataMongo": {
+    "ConnectionString": "mongodb://mongo:27017",
+    "DatabaseName": "NewInfoferScraper"
+  },
   "AllowedHosts": "*"
 }

server/server.csproj

@@ -10,6 +10,7 @@
     <ItemGroup>
         <PackageReference Include="Microsoft.Data.Sqlite" Version="6.0.1" />
         <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="5.0.13" />
+        <PackageReference Include="MongoDB.Driver" Version="2.18.0" />
         <PackageReference Include="Nanoid" Version="2.1.0" />
         <PackageReference Include="Swashbuckle.AspNetCore" Version="5.6.3" />
     </ItemGroup>