alexei.dolgolyov
292223174c
Adds a calibration dashboard that joins persisted SuspensionFlip anomalies with EventResult rows and reports whether the post-flip favourite actually won — the single metric that says whether the detector is doing its job. Domain: - AnomalyEvidenceData + AnomalyEvidenceParser to read the JSON written by AnomalyDetector without re-implementing the schema. - AnomalyOutcomeEvaluator: pure function returning Hit / Miss / Unresolved. Tennis-style two-way markets with a Draw winner are downgraded to Unresolved rather than silently counted as Miss. - AnomalySeverityThresholds: shared Low/Medium/High constants so the UI badge and the report buckets cannot drift. Application: - EvaluateAnomalyOutcomesUseCase orchestrates the join + aggregation. - AnomalyOutcomeReport carries totals, hit rate, three breakdowns (severity / sport / score bins) and a per-event title lookup so the UI needs no second pass over IEventRepository. - Score bins extend below 0.30 automatically when the operator lowers the detector threshold so the histogram total always equals ResolvedCount. UI: - Insights page at /anomalies/insights — hero header, 4-card KPI strip (hit rate tinted by tone), three breakdown grids with bar visualisation, drill-down tables for resolved and unresolved anomalies. Honors prefers-reduced-motion. RU + EN localisation. - Nav entry under Analysis section + chip button on the Anomaly Feed. Tests: +42 across Domain + Application (evaluator boundary cases including tennis two-way and Draw guard, score-bin edges, dynamic floor when threshold is lowered, event-title pass-through). All 324 tests pass. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
309 lines
13 KiB
C#
309 lines
13 KiB
C#
using FluentAssertions;
|
|
using Marathon.Application.Abstractions;
|
|
using Marathon.Application.Reporting;
|
|
using Marathon.Application.UseCases;
|
|
using Marathon.Domain.AnomalyDetection;
|
|
using Marathon.Domain.Entities;
|
|
using Marathon.Domain.Enums;
|
|
using Marathon.Domain.ValueObjects;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using NSubstitute;
|
|
|
|
namespace Marathon.Application.Tests.UseCases;
|
|
|
|
/// <summary>
|
|
/// Unit tests for <see cref="EvaluateAnomalyOutcomesUseCase"/> covering empty
|
|
/// state, mixed hit/miss aggregation, unresolved partitioning, and missing
|
|
/// event metadata fallbacks.
|
|
/// </summary>
|
|
public sealed class EvaluateAnomalyOutcomesUseCaseTests
|
|
{
|
|
private readonly IAnomalyRepository _anomalies = Substitute.For<IAnomalyRepository>();
|
|
private readonly IEventRepository _events = Substitute.For<IEventRepository>();
|
|
private readonly IResultRepository _results = Substitute.For<IResultRepository>();
|
|
|
|
private static readonly TimeSpan MoscowOffset = TimeSpan.FromHours(3);
|
|
private static readonly DateTimeOffset BaseTime =
|
|
new(2026, 5, 10, 18, 0, 0, MoscowOffset);
|
|
|
|
// Flip evidence with Side1 → Side2 reversal.
|
|
private const string FlipEvidence = """
|
|
{
|
|
"suspensionGapSeconds": 90,
|
|
"preSuspension": {
|
|
"capturedAt": "2026-05-10T18:00:00+03:00",
|
|
"p1": 0.55, "pDraw": 0.20, "p2": 0.25,
|
|
"rate1": 1.8, "rateDraw": 4.5, "rate2": 4.0
|
|
},
|
|
"postSuspension": {
|
|
"capturedAt": "2026-05-10T18:02:30+03:00",
|
|
"p1": 0.25, "pDraw": 0.20, "p2": 0.55,
|
|
"rate1": 4.0, "rateDraw": 4.5, "rate2": 1.8
|
|
}
|
|
}
|
|
""";
|
|
|
|
private EvaluateAnomalyOutcomesUseCase CreateSut() =>
|
|
new(_anomalies, _events, _results,
|
|
NullLogger<EvaluateAnomalyOutcomesUseCase>.Instance);
|
|
|
|
private static Anomaly MakeAnomaly(EventId eventId, decimal score) =>
|
|
new(Guid.NewGuid(), eventId, BaseTime, AnomalyKind.SuspensionFlip,
|
|
score, FlipEvidence);
|
|
|
|
private static Event MakeEvent(EventId id, int sportCode) =>
|
|
new(id, new SportCode(sportCode), "BY", "L1", "Cat",
|
|
BaseTime, "Team A", "Team B");
|
|
|
|
[Fact]
|
|
public async Task Should_ReturnEmptyReport_When_NoAnomaliesExist()
|
|
{
|
|
_anomalies.ListAsync(Arg.Any<CancellationToken>())
|
|
.Returns(Array.Empty<Anomaly>().ToList().AsReadOnly());
|
|
|
|
var report = await CreateSut().ExecuteAsync(CancellationToken.None);
|
|
|
|
report.TotalAnomalies.Should().Be(0);
|
|
report.HitRate.Should().BeNull();
|
|
report.Resolved.Should().BeEmpty();
|
|
report.BySport.Should().BeEmpty();
|
|
report.BySeverity.Should().BeEmpty();
|
|
report.ByScoreBin.Should().BeEmpty();
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Should_PartitionAnomalies_Into_ResolvedAndUnresolved()
|
|
{
|
|
var id1 = new EventId("11111111");
|
|
var id2 = new EventId("22222222");
|
|
|
|
_anomalies.ListAsync(Arg.Any<CancellationToken>())
|
|
.Returns(new[]
|
|
{
|
|
MakeAnomaly(id1, score: 0.65m),
|
|
MakeAnomaly(id2, score: 0.40m),
|
|
}.ToList().AsReadOnly());
|
|
|
|
_events.GetAsync(id1, Arg.Any<CancellationToken>()).Returns(MakeEvent(id1, 11));
|
|
_events.GetAsync(id2, Arg.Any<CancellationToken>()).Returns(MakeEvent(id2, 6));
|
|
|
|
// id1 has a result → resolved; id2 has no result → unresolved.
|
|
_results.GetAsync(id1, Arg.Any<CancellationToken>())
|
|
.Returns(new EventResult(id1, 0, 2, Side.Side2, DateTimeOffset.UtcNow));
|
|
_results.GetAsync(id2, Arg.Any<CancellationToken>())
|
|
.Returns((EventResult?)null);
|
|
|
|
var report = await CreateSut().ExecuteAsync(CancellationToken.None);
|
|
|
|
report.TotalAnomalies.Should().Be(2);
|
|
report.ResolvedCount.Should().Be(1);
|
|
report.UnresolvedCount.Should().Be(1);
|
|
report.HitCount.Should().Be(1, "id1's post-flip favourite (Side2) matched the actual winner");
|
|
report.MissCount.Should().Be(0);
|
|
report.HitRate.Should().Be(1.0m);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Should_ComputeHitRate_Across_MixedHitsAndMisses()
|
|
{
|
|
var ids = Enumerable.Range(1, 4)
|
|
.Select(i => new EventId($"event-{i:00000000}"))
|
|
.ToArray();
|
|
|
|
_anomalies.ListAsync(Arg.Any<CancellationToken>())
|
|
.Returns(ids.Select(id => MakeAnomaly(id, score: 0.55m)).ToList().AsReadOnly());
|
|
|
|
foreach (var id in ids)
|
|
{
|
|
_events.GetAsync(id, Arg.Any<CancellationToken>()).Returns(MakeEvent(id, 11));
|
|
}
|
|
|
|
// Three hits (Side2 wins), one miss (Side1 wins).
|
|
_results.GetAsync(ids[0], Arg.Any<CancellationToken>())
|
|
.Returns(new EventResult(ids[0], 0, 2, Side.Side2, DateTimeOffset.UtcNow));
|
|
_results.GetAsync(ids[1], Arg.Any<CancellationToken>())
|
|
.Returns(new EventResult(ids[1], 0, 2, Side.Side2, DateTimeOffset.UtcNow));
|
|
_results.GetAsync(ids[2], Arg.Any<CancellationToken>())
|
|
.Returns(new EventResult(ids[2], 0, 2, Side.Side2, DateTimeOffset.UtcNow));
|
|
_results.GetAsync(ids[3], Arg.Any<CancellationToken>())
|
|
.Returns(new EventResult(ids[3], 2, 0, Side.Side1, DateTimeOffset.UtcNow));
|
|
|
|
var report = await CreateSut().ExecuteAsync(CancellationToken.None);
|
|
|
|
report.HitCount.Should().Be(3);
|
|
report.MissCount.Should().Be(1);
|
|
report.HitRate.Should().Be(0.75m);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Should_BuildSeverityBuckets_Across_LowMediumHigh()
|
|
{
|
|
var idLow = new EventId("low000000");
|
|
var idMed = new EventId("med000000");
|
|
var idHigh = new EventId("high00000");
|
|
|
|
_anomalies.ListAsync(Arg.Any<CancellationToken>())
|
|
.Returns(new[]
|
|
{
|
|
MakeAnomaly(idLow, score: 0.35m),
|
|
MakeAnomaly(idMed, score: 0.50m),
|
|
MakeAnomaly(idHigh, score: 0.75m),
|
|
}.ToList().AsReadOnly());
|
|
|
|
foreach (var id in new[] { idLow, idMed, idHigh })
|
|
{
|
|
_events.GetAsync(id, Arg.Any<CancellationToken>()).Returns(MakeEvent(id, 11));
|
|
_results.GetAsync(id, Arg.Any<CancellationToken>())
|
|
.Returns(new EventResult(id, 0, 2, Side.Side2, DateTimeOffset.UtcNow));
|
|
}
|
|
|
|
var report = await CreateSut().ExecuteAsync(CancellationToken.None);
|
|
|
|
report.BySeverity.Should().HaveCount(3);
|
|
report.BySeverity.Single(b => b.Key == OutcomeBucketKeys.SeverityLow).Total.Should().Be(1);
|
|
report.BySeverity.Single(b => b.Key == OutcomeBucketKeys.SeverityMedium).Total.Should().Be(1);
|
|
report.BySeverity.Single(b => b.Key == OutcomeBucketKeys.SeverityHigh).Total.Should().Be(1);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Should_GroupBySport_When_AnomaliesSpanMultipleSports()
|
|
{
|
|
var idFb = new EventId("fb000000");
|
|
var idBb = new EventId("bb000000");
|
|
|
|
_anomalies.ListAsync(Arg.Any<CancellationToken>())
|
|
.Returns(new[]
|
|
{
|
|
MakeAnomaly(idFb, score: 0.55m),
|
|
MakeAnomaly(idBb, score: 0.55m),
|
|
}.ToList().AsReadOnly());
|
|
|
|
_events.GetAsync(idFb, Arg.Any<CancellationToken>()).Returns(MakeEvent(idFb, 11));
|
|
_events.GetAsync(idBb, Arg.Any<CancellationToken>()).Returns(MakeEvent(idBb, 6));
|
|
|
|
_results.GetAsync(idFb, Arg.Any<CancellationToken>())
|
|
.Returns(new EventResult(idFb, 0, 2, Side.Side2, DateTimeOffset.UtcNow));
|
|
_results.GetAsync(idBb, Arg.Any<CancellationToken>())
|
|
.Returns(new EventResult(idBb, 2, 0, Side.Side1, DateTimeOffset.UtcNow));
|
|
|
|
var report = await CreateSut().ExecuteAsync(CancellationToken.None);
|
|
|
|
report.BySport.Select(b => b.Key)
|
|
.Should().BeEquivalentTo(new[] { "Sport.6", "Sport.11" });
|
|
report.BySport.Single(b => b.Key == "Sport.11").HitRate.Should().Be(1.0m);
|
|
report.BySport.Single(b => b.Key == "Sport.6").HitRate.Should().Be(0.0m);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Should_BuildSevenScoreBins_With_CanonicalKeys()
|
|
{
|
|
var id = new EventId("score000");
|
|
_anomalies.ListAsync(Arg.Any<CancellationToken>())
|
|
.Returns(new[] { MakeAnomaly(id, score: 0.95m) }.ToList().AsReadOnly());
|
|
|
|
_events.GetAsync(id, Arg.Any<CancellationToken>()).Returns(MakeEvent(id, 11));
|
|
_results.GetAsync(id, Arg.Any<CancellationToken>())
|
|
.Returns(new EventResult(id, 0, 2, Side.Side2, DateTimeOffset.UtcNow));
|
|
|
|
var report = await CreateSut().ExecuteAsync(CancellationToken.None);
|
|
|
|
report.ByScoreBin.Should().HaveCount(7, "default buckets cover [0.30, 1.00] in 0.10-wide bins");
|
|
report.ByScoreBin.Select(b => b.Key).Should().BeEquivalentTo(
|
|
new[]
|
|
{
|
|
"Bin.0.30-0.40", "Bin.0.40-0.50", "Bin.0.50-0.60", "Bin.0.60-0.70",
|
|
"Bin.0.70-0.80", "Bin.0.80-0.90", "Bin.0.90-1.00",
|
|
},
|
|
options => options.WithStrictOrdering(),
|
|
"the page reads these literals to render labels");
|
|
report.ByScoreBin.Last().Total.Should().Be(1, "score 0.95 should land in the [0.90, 1.00] bin");
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(0.30, "Bin.0.30-0.40")]
|
|
[InlineData(0.40, "Bin.0.40-0.50")]
|
|
[InlineData(0.5999, "Bin.0.50-0.60")]
|
|
[InlineData(0.60, "Bin.0.60-0.70")]
|
|
[InlineData(1.00, "Bin.0.90-1.00")]
|
|
public async Task Should_PlaceScore_InCorrectBin_AtBoundary(double scoreDouble, string expectedKey)
|
|
{
|
|
var score = (decimal)scoreDouble;
|
|
var id = new EventId("boundary");
|
|
_anomalies.ListAsync(Arg.Any<CancellationToken>())
|
|
.Returns(new[] { MakeAnomaly(id, score) }.ToList().AsReadOnly());
|
|
_events.GetAsync(id, Arg.Any<CancellationToken>()).Returns(MakeEvent(id, 11));
|
|
_results.GetAsync(id, Arg.Any<CancellationToken>())
|
|
.Returns(new EventResult(id, 0, 2, Side.Side2, DateTimeOffset.UtcNow));
|
|
|
|
var report = await CreateSut().ExecuteAsync(CancellationToken.None);
|
|
|
|
var bin = report.ByScoreBin.Single(b => b.Total == 1);
|
|
bin.Key.Should().Be(expectedKey);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Should_ExtendScoreBinsBelow_When_DetectorThresholdIsLowered()
|
|
{
|
|
// Operator lowered Anomaly.OddsFlipThreshold to 0.10 → anomalies with
|
|
// score 0.15 exist. The histogram must still account for them.
|
|
var idLow = new EventId("lowscore");
|
|
var idHigh = new EventId("hicscore");
|
|
|
|
_anomalies.ListAsync(Arg.Any<CancellationToken>())
|
|
.Returns(new[]
|
|
{
|
|
MakeAnomaly(idLow, score: 0.15m),
|
|
MakeAnomaly(idHigh, score: 0.85m),
|
|
}.ToList().AsReadOnly());
|
|
|
|
foreach (var id in new[] { idLow, idHigh })
|
|
{
|
|
_events.GetAsync(id, Arg.Any<CancellationToken>()).Returns(MakeEvent(id, 11));
|
|
_results.GetAsync(id, Arg.Any<CancellationToken>())
|
|
.Returns(new EventResult(id, 0, 2, Side.Side2, DateTimeOffset.UtcNow));
|
|
}
|
|
|
|
var report = await CreateSut().ExecuteAsync(CancellationToken.None);
|
|
|
|
report.ByScoreBin.Sum(b => b.Total).Should().Be(report.ResolvedCount,
|
|
"the histogram total must equal ResolvedCount regardless of detector tuning");
|
|
report.ByScoreBin.First().Key.Should().Be("Bin.0.10-0.20",
|
|
"buckets are extended downward to include the lowest observed score");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Should_PopulateEventTitles_ForJoinedEvents()
|
|
{
|
|
var id = new EventId("title000");
|
|
_anomalies.ListAsync(Arg.Any<CancellationToken>())
|
|
.Returns(new[] { MakeAnomaly(id, 0.55m) }.ToList().AsReadOnly());
|
|
_events.GetAsync(id, Arg.Any<CancellationToken>()).Returns(MakeEvent(id, 11));
|
|
_results.GetAsync(id, Arg.Any<CancellationToken>())
|
|
.Returns(new EventResult(id, 0, 2, Side.Side2, DateTimeOffset.UtcNow));
|
|
|
|
var report = await CreateSut().ExecuteAsync(CancellationToken.None);
|
|
|
|
report.EventTitles.Should().ContainKey(id);
|
|
report.EventTitles[id].Should().Be("Team A vs Team B");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Should_HandleMissingEvent_By_OmittingFromSportBuckets()
|
|
{
|
|
var id = new EventId("orphan00");
|
|
_anomalies.ListAsync(Arg.Any<CancellationToken>())
|
|
.Returns(new[] { MakeAnomaly(id, score: 0.55m) }.ToList().AsReadOnly());
|
|
|
|
_events.GetAsync(id, Arg.Any<CancellationToken>()).Returns((Event?)null);
|
|
_results.GetAsync(id, Arg.Any<CancellationToken>())
|
|
.Returns(new EventResult(id, 0, 2, Side.Side2, DateTimeOffset.UtcNow));
|
|
|
|
var report = await CreateSut().ExecuteAsync(CancellationToken.None);
|
|
|
|
report.Resolved.Should().HaveCount(1,
|
|
"orphan anomalies are still evaluated for hit/miss");
|
|
report.BySport.Should().BeEmpty(
|
|
"missing event metadata excludes the row from sport breakdown");
|
|
}
|
|
}
|