kbruen
/
new-infofer-scraper
1
0
Fork 0
Code Pull Requests Releases Activity
Browse Source

Transition from fs+JSON to MongoDB

master
Kenneth Bruen 2 years ago
parent
169e128ca1
commit
145f7b0ee1
Signed by: kbruen
GPG Key ID: C1980A470C3EE5B1
13 changed files with 167 additions and 129 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      server/Controllers/V2/StationsController.cs
  2. 3
      server/Controllers/V2/TrainsController.cs
  3. 3
      server/Controllers/V3/StationsController.cs
  4. 3
      server/Controllers/V3/TrainsController.cs
  5. 5
      server/Models/Database/MongoSettings.cs
  6. 17
      server/Models/Database/StationListing.cs
  7. 17
      server/Models/Database/TrainListing.cs
  8. 210
      server/Services/Implementations/Database.cs
  9. 16
      server/Services/Interfaces/IDatabase.cs
  10. 6
      server/Startup.cs
  11. 6
      server/appsettings.Development.json
  12. 4
      server/appsettings.json
  13. 1
      server/server.csproj

3
server/Controllers/V2/StationsController.cs
Unescape View File

@ -1,5 +1,6 @@
using System.Collections.Generic;
using Microsoft.AspNetCore.Mvc;
using Server.Models.Database;
using Server.Services.Interfaces;
namespace Server.Controllers.V2;
@ -15,7 +16,7 @@ public class StationsController : Controller {
}
[HttpGet("")]
public ActionResult<IEnumerable<IStationRecord>> ListStations() {
public ActionResult<IEnumerable<StationListing>> ListStations() {
return Ok(Database.Stations);
}
}

3
server/Controllers/V2/TrainsController.cs
Unescape View File

@ -1,5 +1,6 @@
using System.Collections.Generic;
using Microsoft.AspNetCore.Mvc;
using Server.Models.Database;
using Server.Services.Interfaces;
namespace Server.Controllers.V2;
@ -15,7 +16,7 @@ public class TrainsController : Controller {
}
[HttpGet("")]
public ActionResult<IEnumerable<ITrainRecord>> ListTrains() {
public ActionResult<IEnumerable<TrainListing>> ListTrains() {
return Ok(Database.Trains);
}
}

3
server/Controllers/V3/StationsController.cs
Unescape View File

@ -4,6 +4,7 @@ using System.Threading.Tasks;
using InfoferScraper.Models.Station;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Server.Models.Database;
using Server.Services.Interfaces;
namespace Server.Controllers.V3;
@ -21,7 +22,7 @@ public class StationsController : Controller {
}
[HttpGet("")]
public ActionResult<IEnumerable<IStationRecord>> ListStations() {
public ActionResult<IEnumerable<StationListing>> ListStations() {
return Ok(Database.Stations);
}

3
server/Controllers/V3/TrainsController.cs
Unescape View File

@ -5,6 +5,7 @@ using InfoferScraper.Models.Train;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using scraper.Exceptions;
using Server.Models.Database;
using Server.Services.Interfaces;
namespace Server.Controllers.V3;
@ -22,7 +23,7 @@ public class TrainsController : Controller {
}
[HttpGet("")]
public ActionResult<IEnumerable<ITrainRecord>> ListTrains() {
public ActionResult<IEnumerable<TrainListing>> ListTrains() {
return Ok(Database.Trains);
}

5
server/Models/Database/MongoSettings.cs
Unescape View File

@ -0,0 +1,5 @@
namespace Server.Models.Database;
public record MongoSettings(string ConnectionString, string DatabaseName) {
public MongoSettings() : this("", "") { }
}

17
server/Models/Database/StationListing.cs
Unescape View File

@ -0,0 +1,17 @@
using System.Collections.Generic;
using System.Text.Json.Serialization;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
namespace Server.Models.Database;
public record StationListing(
[property: BsonId]
[property: BsonRepresentation(BsonType.ObjectId)]
[property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
string? Id,
string Name,
List<string> StoppedAtBy
) {
public StationListing(string name, List<string> stoppedAtBy) : this(null, name, stoppedAtBy) { }
}

17
server/Models/Database/TrainListing.cs
Unescape View File

@ -0,0 +1,17 @@
using System.Text.Json.Serialization;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
namespace Server.Models.Database;
public record TrainListing(
[property: BsonId]
[property: BsonRepresentation(BsonType.ObjectId)]
[property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
string? Id,
string Rank,
string Number,
string Company
) {
public TrainListing(string rank, string number, string company) : this(null, rank, number, company) { }
}

210
server/Services/Implementations/Database.cs
Unescape View File

@ -7,6 +7,11 @@ using System.Text.Json.Nodes;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
using MongoDB.Driver;
using Server.Models.Database;
namespace Server.Services.Implementations;
@ -17,77 +22,47 @@ public class Database : Server.Services.Interfaces.IDatabase {
private ILogger<Database> Logger { get; }
private bool shouldCommitOnEveryChange = true;
private bool dbDataDirty = false;
private bool stationsDirty = false;
private bool trainsDirty = false;
public DbRecord DbData { get; private set; } = new(3);
public DbRecord DbData { get; private set; } = new(2);
private List<StationRecord> stations = new();
private List<TrainRecord> trains = new();
public IReadOnlyList<Server.Services.Interfaces.IStationRecord> Stations => stations;
public IReadOnlyList<Server.Services.Interfaces.ITrainRecord> Trains => trains;
public IReadOnlyList<StationListing> Stations => stationListingsCollection.FindSync(_ => true).ToList();
public IReadOnlyList<TrainListing> Trains => trainListingsCollection.FindSync(_ => true).ToList();
private static readonly string DbDir = Environment.GetEnvironmentVariable("DB_DIR") ?? Path.Join(Environment.CurrentDirectory, "db");
private static readonly string DbFile = Path.Join(DbDir, "db.json");
private static readonly string StationsFile = Path.Join(DbDir, "stations.json");
private static readonly string TrainsFile = Path.Join(DbDir, "trains.json");
public IDisposable MakeDbTransaction() {
shouldCommitOnEveryChange = false;
return new Server.Utils.ActionDisposable(() => {
if (dbDataDirty) File.WriteAllText(DbFile, JsonSerializer.Serialize(DbData, serializerOptions));
if (stationsDirty) {
stations.Sort((s1, s2) => s2.StoppedAtBy.Count.CompareTo(s1.StoppedAtBy.Count));
File.WriteAllText(StationsFile, JsonSerializer.Serialize(stations, serializerOptions));
}
if (trainsDirty) File.WriteAllText(TrainsFile, JsonSerializer.Serialize(trains, serializerOptions));
dbDataDirty = stationsDirty = trainsDirty = false;
shouldCommitOnEveryChange = true;
});
}
private readonly IMongoDatabase db;
private readonly IMongoCollection<DbRecord> dbRecordCollection;
private readonly IMongoCollection<TrainListing> trainListingsCollection;
private readonly IMongoCollection<StationListing> stationListingsCollection;
public Database(ILogger<Database> logger) {
public Database(ILogger<Database> logger, IOptions<MongoSettings> mongoSettings) {
Logger = logger;
if (!Directory.Exists(DbDir)) {
Logger.LogDebug("Creating directory: {DbDir}", DbDir);
Directory.CreateDirectory(DbDir);
}
MongoClient mongoClient = new(mongoSettings.Value.ConnectionString);
db = mongoClient.GetDatabase(mongoSettings.Value.DatabaseName) ?? throw new NullReferenceException("Unable to get Mongo database");
dbRecordCollection = db.GetCollection<DbRecord>("db");
trainListingsCollection = db.GetCollection<TrainListing>("trainListings");
stationListingsCollection = db.GetCollection<StationListing>("stationListings");
Migration();
if (File.Exists(DbFile)) {
DbData = JsonSerializer.Deserialize<DbRecord>(File.ReadAllText(DbFile), serializerOptions)!;
}
else {
File.WriteAllText(DbFile, JsonSerializer.Serialize(DbData, serializerOptions));
}
if (File.Exists(StationsFile)) {
stations = JsonSerializer.Deserialize<List<StationRecord>>(File.ReadAllText(StationsFile), serializerOptions)!;
}
if (File.Exists(TrainsFile)) {
trains = JsonSerializer.Deserialize<List<TrainRecord>>(File.ReadAllText(TrainsFile), serializerOptions)!;
}
}
private void Migration() {
if (!File.Exists(DbFile)) {
// using var _ = Logger.BeginScope("Migrating DB version 1 -> 2");
if (!File.Exists(DbFile) && File.Exists(TrainsFile)) {
Logger.LogInformation("Migrating DB version 1 -> 2");
if (File.Exists(StationsFile)) {
Logger.LogDebug("Converting StationsFile");
var oldStations = JsonNode.Parse(File.ReadAllText(StationsFile));
List<StationListing> stations = new();
if (oldStations != null) {
Logger.LogDebug("Found {StationsCount} stations", oldStations.AsArray().Count);
foreach (var station in oldStations.AsArray()) {
if (station == null) continue;
station["stoppedAtBy"] = new JsonArray(station["stoppedAtBy"]!.AsArray().Select(num => (JsonNode)(num!).ToString()!).ToArray());
}
stations = JsonSerializer.Deserialize<List<StationRecord>>(oldStations, serializerOptions)!;
stations = oldStations.Deserialize<List<StationListing>>(serializerOptions)!;
}
Logger.LogDebug("Rewriting StationsFile");
File.WriteAllText(StationsFile, JsonSerializer.Serialize(stations, serializerOptions));
@ -95,6 +70,7 @@ public class Database : Server.Services.Interfaces.IDatabase {
if (File.Exists(TrainsFile)) {
Logger.LogDebug("Converting TrainsFile");
var oldTrains = JsonNode.Parse(File.ReadAllText(TrainsFile));
List<TrainListing> trains = new();
if (oldTrains != null) {
Logger.LogDebug("Found {TrainsCount} trains", oldTrains.AsArray().Count);
foreach (var train in oldTrains.AsArray()) {
@ -102,7 +78,7 @@ public class Database : Server.Services.Interfaces.IDatabase {
train["number"] = train["numberString"];
train.AsObject().Remove("numberString");
}
trains = JsonSerializer.Deserialize<List<TrainRecord>>(oldTrains, serializerOptions)!;
trains = oldTrains.Deserialize<List<TrainListing>>(serializerOptions)!;
}
Logger.LogDebug("Rewriting TrainsFile");
File.WriteAllText(TrainsFile, JsonSerializer.Serialize(trains, serializerOptions));
@ -111,72 +87,101 @@ public class Database : Server.Services.Interfaces.IDatabase {
File.WriteAllText(DbFile, JsonSerializer.Serialize(DbData, serializerOptions));
Migration();
}
else {
else if (File.Exists(DbFile)) {
var oldDbData = JsonNode.Parse(File.ReadAllText(DbFile));
if (((int?)oldDbData?["version"]) == 2) {
Logger.LogInformation("DB Version: 2; noop");
Logger.LogInformation("Migrating DB version 2 -> 3 (transition from fs+JSON to MongoDB)");
if (File.Exists(StationsFile)) {
Logger.LogDebug("Converting StationsFile");
var stations = JsonSerializer.Deserialize<List<StationListing>>(File.ReadAllText(StationsFile));
stationListingsCollection.InsertMany(stations);
File.Delete(StationsFile);
}
else {
throw new Exception("Unexpected Database version");
if (File.Exists(TrainsFile)) {
Logger.LogDebug("Converting TrainsFile");
var trains = JsonSerializer.Deserialize<List<TrainListing>>(File.ReadAllText(TrainsFile));
trainListingsCollection.InsertMany(trains);
File.Delete(TrainsFile);
}
File.Delete(DbFile);
try {
Directory.Delete(DbDir);
}
catch (Exception) {
// Deleting of the directory is optional; may not be allowed in Docker or similar
}
public async Task<string> FoundTrain(string rank, string number, string company) {
number = string.Join("", number.TakeWhile(c => '0' <= c && c <= '9'));
if (!trains.Where(train => train.Number == number).Any()) {
Logger.LogDebug("Found train {Rank} {Number} from {Company}", rank, number, company);
trains.Add(new(number, rank, company));
if (shouldCommitOnEveryChange) {
await File.WriteAllTextAsync(TrainsFile, JsonSerializer.Serialize(trains, serializerOptions));
var x = dbRecordCollection.FindSync(_ => true).ToList()!;
if (x.Count != 0) {
Logger.LogWarning("db collection contained data when migrating to V3");
using (var _ = Logger.BeginScope("Already existing data:")) {
foreach (var dbRecord in x) {
Logger.LogInformation("Id: {Id}, Version: {Version}", dbRecord.Id, dbRecord.Version);
}
}
Logger.LogInformation("Backing up existing data");
var backupDbRecordCollection = db.GetCollection<DbRecord>("db-backup");
backupDbRecordCollection.InsertMany(x);
Logger.LogDebug("Removing existing data");
dbRecordCollection.DeleteMany(_ => true);
}
dbRecordCollection.InsertOne(new(3));
Migration();
}
else {
trainsDirty = true;
throw new("Unexpected Database version, only DB Version 2 uses DbFile");
}
}
return number;
else {
var datas = dbRecordCollection.FindSync(_ => true).ToList();
if (datas.Count == 0) {
Logger.LogInformation("No db record found, new database");
dbRecordCollection.InsertOne(DbData);
}
public async Task FoundStation(string name) {
if (!stations.Where(station => station.Name == name).Any()) {
Logger.LogDebug("Found station {StationName}", name);
stations.Add(new(name, new()));
if (shouldCommitOnEveryChange) {
await File.WriteAllTextAsync(StationsFile, JsonSerializer.Serialize(stations, serializerOptions));
else {
DbData = datas[0];
}
if (DbData.Version == 3) {
Logger.LogInformation("Using MongoDB Database Version 3; noop");
}
else {
stationsDirty = true;
throw new($"Unexpected Database version: {DbData.Version}");
}
}
}
public async Task FoundTrainAtStation(string stationName, string trainNumber) {
trainNumber = string.Join("", trainNumber.TakeWhile(c => '0' <= c && c <= '9'));
await FoundStation(stationName);
var dirty = false;
for (var i = 0; i < stations.Count; i++) {
if (stations[i].Name == stationName) {
if (!stations[i].StoppedAtBy.Contains(trainNumber)) {
Logger.LogDebug("Found train {TrainNumber} at station {StationName}", trainNumber, stationName);
stations[i].ActualStoppedAtBy.Add(trainNumber);
dirty = true;
}
break;
public async Task<string> FoundTrain(string rank, string number, string company) {
number = string.Join("", number.TakeWhile(c => c is >= '0' and <= '9'));
if (!await (await trainListingsCollection.FindAsync(Builders<TrainListing>.Filter.Eq("number", number))).AnyAsync()) {
Logger.LogDebug("Found train {Rank} {Number} from {Company}", rank, number, company);
await trainListingsCollection.InsertOneAsync(new(number: number, rank: rank, company: company));
}
return number;
}
if (dirty) {
if (shouldCommitOnEveryChange) {
stations.Sort((s1, s2) => s2.StoppedAtBy.Count.CompareTo(s1.StoppedAtBy.Count));
await File.WriteAllTextAsync(StationsFile, JsonSerializer.Serialize(stations, serializerOptions));
public async Task FoundStation(string name) {
if (!await (stationListingsCollection.Find(Builders<StationListing>.Filter.Eq("name", name))).AnyAsync()) {
Logger.LogDebug("Found station {StationName}", name);
await stationListingsCollection.InsertOneAsync(new(name, new()));
}
else {
stationsDirty = true;
}
public async Task FoundTrainAtStation(string stationName, string trainNumber) {
trainNumber = string.Join("", trainNumber.TakeWhile(c => c is >= '0' and <= '9'));
await FoundStation(stationName);
var updateResult = await stationListingsCollection.UpdateOneAsync(
Builders<StationListing>.Filter.Eq("name", stationName),
Builders<StationListing>.Update.AddToSet("stoppedAtBy", trainNumber)
);
if (updateResult.IsAcknowledged && updateResult.ModifiedCount > 0) {
Logger.LogDebug("Found train {TrainNumber} at station {StationName}", trainNumber, stationName);
}
}
public async Task OnTrainData(InfoferScraper.Models.Train.ITrainScrapeResult trainData) {
using var _ = MakeDbTransaction();
var trainNumber = await FoundTrain(trainData.Rank, trainData.Number, trainData.Operator);
foreach (var group in trainData.Groups) {
foreach (var station in group.Stations) {
@ -199,8 +204,6 @@ public class Database : Server.Services.Interfaces.IDatabase {
}
}
using var _ = MakeDbTransaction();
if (stationData.Arrivals != null) {
foreach (var train in stationData.Arrivals) {
await ProcessTrain(train);
@ -214,25 +217,12 @@ public class Database : Server.Services.Interfaces.IDatabase {
}
}
public record DbRecord(int Version);
public record StationRecord : Server.Services.Interfaces.IStationRecord {
[JsonPropertyName("stoppedAtBy")]
public List<string> ActualStoppedAtBy { get; init; }
public string Name { get; init; }
[JsonIgnore]
public IReadOnlyList<string> StoppedAtBy => ActualStoppedAtBy;
public StationRecord() {
Name = "";
ActualStoppedAtBy = new();
public record DbRecord(
[property: BsonId]
[property: BsonRepresentation(BsonType.ObjectId)]
[property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
string? Id,
int Version
) {
public DbRecord(int version) : this(null, version) { }
}
public StationRecord(string name, List<string> stoppedAtBy) {
Name = name;
ActualStoppedAtBy = stoppedAtBy;
}
}
public record TrainRecord(string Number, string Rank, string Company) : Server.Services.Interfaces.ITrainRecord;

16
server/Services/Interfaces/IDatabase.cs
Unescape View File

@ -2,12 +2,13 @@ using System.Collections.Generic;
using System.Threading.Tasks;
using InfoferScraper.Models.Train;
using InfoferScraper.Models.Station;
using Server.Models.Database;
namespace Server.Services.Interfaces;
public interface IDatabase {
public IReadOnlyList<IStationRecord> Stations { get; }
public IReadOnlyList<ITrainRecord> Trains { get; }
public IReadOnlyList<StationListing> Stations { get; }
public IReadOnlyList<TrainListing> Trains { get; }
public Task<string> FoundTrain(string rank, string number, string company);
public Task FoundStation(string name);
@ -15,14 +16,3 @@ public interface IDatabase {
public Task OnTrainData(ITrainScrapeResult trainData);
public Task OnStationData(IStationScrapeResult stationData);
}
public interface IStationRecord {
public string Name { get; }
public IReadOnlyList<string> StoppedAtBy { get; }
}
public interface ITrainRecord {
public string Rank { get; }
public string Number { get; }
public string Company { get; }
}

6
server/Startup.cs
Unescape View File

@ -8,6 +8,8 @@ using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.OpenApi.Models;
using MongoDB.Bson.Serialization.Conventions;
using Server.Models.Database;
using Server.Services.Implementations;
using Server.Services.Interfaces;
@ -26,6 +28,10 @@ namespace Server {
options.KnownProxies.Add(Dns.GetHostAddresses("host.docker.internal")[0]);
});
}
services.Configure<MongoSettings>(Configuration.GetSection("TrainDataMongo"));
var conventionPack = new ConventionPack { new CamelCaseElementNameConvention() };
ConventionRegistry.Register("camelCase", conventionPack, _ => true);
services.AddSingleton<IDataManager, DataManager>();
services.AddSingleton<IDatabase, Database>();
services.AddSingleton<NodaTime.IDateTimeZoneProvider>(NodaTime.DateTimeZoneProviders.Tzdb);

6
server/appsettings.Development.json
Unescape View File

@ -5,5 +5,9 @@
"Microsoft": "Warning",
"Microsoft.Hosting.Lifetime": "Information"
}
}
},
"TrainDataMongo": {
"ConnectionString": "mongodb://localhost:27017",
"DatabaseName": "NewInfoferScraper"
},
}

4
server/appsettings.json
Unescape View File

@ -9,5 +9,9 @@
"Microsoft.Hosting.Lifetime": "Information"
}
},
"TrainDataMongo": {
"ConnectionString": "mongodb://mongo:27017",
"DatabaseName": "NewInfoferScraper"
},
"AllowedHosts": "*"
}

1
server/server.csproj
Unescape View File

@ -10,6 +10,7 @@
<ItemGroup>
<PackageReference Include="Microsoft.Data.Sqlite" Version="6.0.1" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="5.0.13" />
<PackageReference Include="MongoDB.Driver" Version="2.18.0" />
<PackageReference Include="Nanoid" Version="2.1.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="5.6.3" />
</ItemGroup>

Write Preview
Loading…
Cancel
Save