Browse Source

Add debug logging, update mongodb driver

master
Kenneth Bruen 2 years ago
parent
commit
8421b49834
Signed by: kbruen
GPG Key ID: C1980A470C3EE5B1
  1. 6
      docker-compose.yml
  2. 11
      server/Services/Implementations/DataManager.cs
  3. 3
      server/Services/Implementations/Database.cs
  4. 2
      server/server.csproj

6
docker-compose.yml

@ -5,8 +5,6 @@ services:
image: new_infofer_scraper image: new_infofer_scraper
build: . build: .
ports: ports:
- ${PORT:-5000}:80 - ${PORT:-5001}:80
environment: environment:
- DB_DIR=/data DB_DIR: /data
volumes:
- ./data:/data

11
server/Services/Implementations/DataManager.cs

@ -2,13 +2,13 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using System.Threading.Tasks; using System.Threading.Tasks;
using InfoferScraper.Models.Train;
using InfoferScraper.Models.Station;
using Server.Services.Interfaces;
using Server.Utils;
using InfoferScraper; using InfoferScraper;
using InfoferScraper.Models.Station;
using InfoferScraper.Models.Train;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using scraper.Models.Itinerary; using scraper.Models.Itinerary;
using Server.Services.Interfaces;
using Server.Utils;
namespace Server.Services.Implementations { namespace Server.Services.Implementations {
public class DataManager : IDataManager { public class DataManager : IDataManager {
@ -25,6 +25,7 @@ namespace Server.Services.Implementations {
stationCache = new(async (t) => { stationCache = new(async (t) => {
var (stationName, date) = t; var (stationName, date) = t;
Logger.LogDebug("Fetching station {StationName} for date {Date}", stationName, date);
var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone); var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone);
var station = await InfoferScraper.Scrapers.StationScraper.Scrape(stationName, zonedDate.ToDateTimeOffset()); var station = await InfoferScraper.Scrapers.StationScraper.Scrape(stationName, zonedDate.ToDateTimeOffset());
@ -40,6 +41,7 @@ namespace Server.Services.Implementations {
}, TimeSpan.FromMinutes(1)); }, TimeSpan.FromMinutes(1));
trainCache = new(async (t) => { trainCache = new(async (t) => {
var (trainNumber, date) = t; var (trainNumber, date) = t;
Logger.LogDebug("Fetching train {TrainNumber} for date {Date}", trainNumber, date);
var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone); var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone);
var train = await InfoferScraper.Scrapers.TrainScraper.Scrape(trainNumber, zonedDate.ToDateTimeOffset()); var train = await InfoferScraper.Scrapers.TrainScraper.Scrape(trainNumber, zonedDate.ToDateTimeOffset());
@ -55,6 +57,7 @@ namespace Server.Services.Implementations {
}, TimeSpan.FromSeconds(30)); }, TimeSpan.FromSeconds(30));
itinerariesCache = new(async (t) => { itinerariesCache = new(async (t) => {
var (from, to, date) = t; var (from, to, date) = t;
Logger.LogDebug("Fetching itinerary from {From} to {To} for date {Date}", from, to, date);
var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone); var zonedDate = new NodaTime.LocalDate(date.Year, date.Month, date.Day).AtStartOfDayInZone(CfrTimeZone);
var itineraries = await InfoferScraper.Scrapers.RouteScraper.Scrape(from, to, zonedDate.ToDateTimeOffset()); var itineraries = await InfoferScraper.Scrapers.RouteScraper.Scrape(from, to, zonedDate.ToDateTimeOffset());

3
server/Services/Implementations/Database.cs

@ -57,10 +57,13 @@ public class Database : Server.Services.Interfaces.IDatabase {
Logger = logger; Logger = logger;
var settings = MongoClientSettings.FromConnectionString(mongoSettings.Value.ConnectionString); var settings = MongoClientSettings.FromConnectionString(mongoSettings.Value.ConnectionString);
settings.ServerApi = new(ServerApiVersion.V1);
settings.MaxConnectionPoolSize = 10000; settings.MaxConnectionPoolSize = 10000;
MongoClient mongoClient = new(settings); MongoClient mongoClient = new(settings);
Logger.LogDebug("Created monogClient");
throttle = new(mongoClient.Settings.MaxConnectionPoolSize / 2); throttle = new(mongoClient.Settings.MaxConnectionPoolSize / 2);
db = mongoClient.GetDatabase(mongoSettings.Value.DatabaseName) ?? throw new NullReferenceException("Unable to get Mongo database"); db = mongoClient.GetDatabase(mongoSettings.Value.DatabaseName) ?? throw new NullReferenceException("Unable to get Mongo database");
Logger.LogDebug("Created db");
dbRecordCollection = db.GetCollection<DbRecord>("db"); dbRecordCollection = db.GetCollection<DbRecord>("db");
trainListingsCollection = db.GetCollection<TrainListing>("trainListings"); trainListingsCollection = db.GetCollection<TrainListing>("trainListings");
stationListingsCollection = db.GetCollection<StationListing>("stationListings"); stationListingsCollection = db.GetCollection<StationListing>("stationListings");

2
server/server.csproj

@ -12,7 +12,7 @@
<PackageReference Include="Microsoft.Data.Sqlite" Version="6.0.1" /> <PackageReference Include="Microsoft.Data.Sqlite" Version="6.0.1" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="5.0.13" /> <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="5.0.13" />
<PackageReference Include="MongoDB.Analyzer" Version="1.1.0" /> <PackageReference Include="MongoDB.Analyzer" Version="1.1.0" />
<PackageReference Include="MongoDB.Driver" Version="2.18.0" /> <PackageReference Include="MongoDB.Driver" Version="2.19.1" />
<PackageReference Include="Nanoid" Version="2.1.0" /> <PackageReference Include="Nanoid" Version="2.1.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="5.6.3" /> <PackageReference Include="Swashbuckle.AspNetCore" Version="5.6.3" />
</ItemGroup> </ItemGroup>

Loading…
Cancel
Save