Files
maraphon-app/tests/Marathon.Application.Tests/UseCases/PullLiveOddsUseCaseTests.cs
T
alexei.dolgolyov 286b55986b perf(scraping): parallel HTTP fan-out, sequential DB persist (HIGH)
The Pull*UseCase implementations issued one HTTP request at a time despite
Scraping:MaxConcurrentRequests=4. With 30–80 live events and ~1s per
fetch, a 5–10s live cadence target was unreachable; cycles overflowed
the configured interval.

* New Marathon.Application.Configuration.ScrapingThrottle bound from the
  shared Scraping:* section. Exposes only MaxConcurrentRequests so the
  Application layer doesn't pull in the Infrastructure-side ScrapingOptions.
* PullLiveOddsUseCase + PullUpcomingEventsUseCase split into two phases:
  - Phase 1 — Parallel.ForEachAsync over the event list with
    MaxDegreeOfParallelism = throttle.MaxConcurrentRequests. The scraper's
    Polly rate limiter still throttles to RequestsPerSecond underneath
    this fan-out, so spikes are smoothed before they hit the bookmaker.
  - Phase 2 — sequential foreach over the (Event, Snapshot) tuples
    captured in Phase 1, doing event upsert + snapshot insert. EF Core
    DbContext is not thread-safe so all DB writes stay on a single thread.
* InfrastructureModule binds ScrapingThrottle alongside AnomalyOptions.
* Failed snapshot scrapes in Phase 1 mean the event row is also NOT
  persisted in Phase 2 — previously we'd persist the row even when the
  snapshot scrape failed, leaving an orphan event with no odds. Updated
  the regression test accordingly.
* Test fixture exposes TestFixtures.Throttle(maxConcurrentRequests=1) for
  deterministic sequential test runs.
* One existing NSubstitute setup that chained Arg.Is<>() across two
  configurations was rewritten to use a single Arg.Any<>() with inline
  branching — chained matchers were leaking and returning wrong results.
2026-05-09 15:27:06 +03:00

165 lines
6.7 KiB
C#

using FluentAssertions;
using Marathon.Application.Abstractions;
using Marathon.Application.UseCases;
using Marathon.Domain.Entities;
using Marathon.Domain.Enums;
using Marathon.Domain.ValueObjects;
using Microsoft.Extensions.Logging.Abstractions;
using NSubstitute;
using NSubstitute.ExceptionExtensions;
namespace Marathon.Application.Tests.UseCases;
public sealed class PullLiveOddsUseCaseTests
{
private readonly IOddsScraper _scraper = Substitute.For<IOddsScraper>();
private readonly IEventRepository _eventRepo = Substitute.For<IEventRepository>();
private readonly ISnapshotRepository _snapshotRepo = Substitute.For<ISnapshotRepository>();
private PullLiveOddsUseCase CreateSut() =>
new(_scraper, _eventRepo, _snapshotRepo,
TestFixtures.Throttle(),
NullLogger<PullLiveOddsUseCase>.Instance);
[Fact]
public async Task Should_CaptureOneSnapshotPerEvent_When_LiveListingReturnsTwoEvents()
{
// Arrange: 2 events from the live listing; both already known to the DB
var ev1 = TestFixtures.MakeEvent("11111111");
var ev2 = TestFixtures.MakeEvent("22222222");
var live = new List<Event> { ev1, ev2 }.AsReadOnly();
_scraper.ScrapeLiveAsync(Arg.Any<CancellationToken>()).Returns(live);
_eventRepo.GetAsync(ev1.Id, Arg.Any<CancellationToken>()).Returns(ev1);
_eventRepo.GetAsync(ev2.Id, Arg.Any<CancellationToken>()).Returns(ev2);
_scraper.ScrapeEventOddsAsync(
Arg.Is<Event>(e => e.Id == ev1.Id), OddsSource.Live, Arg.Any<CancellationToken>())
.Returns(TestFixtures.MakeSnapshot(ev1.Id, OddsSource.Live));
_scraper.ScrapeEventOddsAsync(
Arg.Is<Event>(e => e.Id == ev2.Id), OddsSource.Live, Arg.Any<CancellationToken>())
.Returns(TestFixtures.MakeSnapshot(ev2.Id, OddsSource.Live));
var sut = CreateSut();
// Act
var snapshotsCaptured = await sut.ExecuteAsync(CancellationToken.None);
// Assert
snapshotsCaptured.Should().Be(2);
await _eventRepo.DidNotReceive().AddAsync(Arg.Any<Event>(), Arg.Any<CancellationToken>());
await _snapshotRepo.Received(2).AddAsync(Arg.Any<OddsSnapshot>(), Arg.Any<CancellationToken>());
}
[Fact]
public async Task Should_PersistNewLiveEvent_When_NotYetInDatabase()
{
// Arrange: live listing returns one event the DB has never seen
var live = TestFixtures.MakeEvent("99999999");
_scraper.ScrapeLiveAsync(Arg.Any<CancellationToken>())
.Returns(new List<Event> { live }.AsReadOnly());
_eventRepo.GetAsync(live.Id, Arg.Any<CancellationToken>()).Returns((Event?)null);
_scraper.ScrapeEventOddsAsync(
Arg.Is<Event>(e => e.Id == live.Id), OddsSource.Live, Arg.Any<CancellationToken>())
.Returns(TestFixtures.MakeSnapshot(live.Id, OddsSource.Live));
var sut = CreateSut();
// Act
var snapshotsCaptured = await sut.ExecuteAsync(CancellationToken.None);
// Assert
snapshotsCaptured.Should().Be(1);
await _eventRepo.Received(1).AddAsync(live, Arg.Any<CancellationToken>());
}
[Fact]
public async Task Should_ContinueAfterSnapshotFailure_And_NotPropagateException()
{
// Arrange: 2 live events — scraping the first throws
var ev1 = TestFixtures.MakeEvent("11111111");
var ev2 = TestFixtures.MakeEvent("22222222");
_scraper.ScrapeLiveAsync(Arg.Any<CancellationToken>())
.Returns(new List<Event> { ev1, ev2 }.AsReadOnly());
_eventRepo.GetAsync(ev1.Id, Arg.Any<CancellationToken>()).Returns(ev1);
_eventRepo.GetAsync(ev2.Id, Arg.Any<CancellationToken>()).Returns(ev2);
_scraper.ScrapeEventOddsAsync(
Arg.Is<Event>(e => e.Id == ev1.Id), OddsSource.Live, Arg.Any<CancellationToken>())
.ThrowsAsync(new HttpRequestException("timeout"));
_scraper.ScrapeEventOddsAsync(
Arg.Is<Event>(e => e.Id == ev2.Id), OddsSource.Live, Arg.Any<CancellationToken>())
.Returns(TestFixtures.MakeSnapshot(ev2.Id, OddsSource.Live));
var sut = CreateSut();
// Act — must not throw
var act = async () => await sut.ExecuteAsync(CancellationToken.None);
await act.Should().NotThrowAsync();
// Re-execute to assert the count (mocks are still primed)
var result = await sut.ExecuteAsync(CancellationToken.None);
result.Should().Be(1, "only ev2 succeeded; ev1 failed silently");
}
[Fact]
public async Task Should_ReturnZero_When_LiveListingIsEmpty()
{
_scraper.ScrapeLiveAsync(Arg.Any<CancellationToken>())
.Returns(Array.Empty<Event>());
var sut = CreateSut();
var result = await sut.ExecuteAsync(CancellationToken.None);
result.Should().Be(0);
await _scraper.DidNotReceive()
.ScrapeEventOddsAsync(Arg.Any<Event>(), Arg.Any<OddsSource>(), Arg.Any<CancellationToken>());
}
[Fact]
public async Task Should_ReturnZeroAndSwallow_When_LiveListingFetchThrows()
{
_scraper.ScrapeLiveAsync(Arg.Any<CancellationToken>())
.ThrowsAsync(new HttpRequestException("listing unavailable"));
var sut = CreateSut();
var result = await sut.ExecuteAsync(CancellationToken.None);
result.Should().Be(0);
await _scraper.DidNotReceive()
.ScrapeEventOddsAsync(Arg.Any<Event>(), Arg.Any<OddsSource>(), Arg.Any<CancellationToken>());
}
[Fact]
public async Task Should_BackfillEventPath_When_ExistingRowMissedIt()
{
// Arrange: DB row pre-dates the EventPath column (EventPath = null);
// live listing supplies a path.
var withoutPath = TestFixtures.MakeEvent("55555555");
var withPath = withoutPath with { EventPath = "Football/Some+Path/Team+vs+Team+-+99" };
_scraper.ScrapeLiveAsync(Arg.Any<CancellationToken>())
.Returns(new List<Event> { withPath }.AsReadOnly());
_eventRepo.GetAsync(withPath.Id, Arg.Any<CancellationToken>()).Returns(withoutPath);
_scraper.ScrapeEventOddsAsync(
Arg.Is<Event>(e => e.EventPath == withPath.EventPath),
OddsSource.Live, Arg.Any<CancellationToken>())
.Returns(TestFixtures.MakeSnapshot(withPath.Id, OddsSource.Live));
var sut = CreateSut();
// Act
var result = await sut.ExecuteAsync(CancellationToken.None);
// Assert — the DB row was updated with the new path before scraping odds
result.Should().Be(1);
await _eventRepo.Received(1).UpdateAsync(
Arg.Is<Event>(e => e.Id == withPath.Id && e.EventPath == withPath.EventPath),
Arg.Any<CancellationToken>());
}
}