From c9eee9f9074b85474cc250231cb80a5802d668ec Mon Sep 17 00:00:00 2001 From: "alexei.dolgolyov" Date: Fri, 29 May 2026 01:25:16 +0300 Subject: [PATCH] fix(anomaly): exclude non-directional kinds from grading and backtest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up (HIGH): the three detectors fed the same evaluator/backtest, but SuspensionFreeze is non-directional (favourite unchanged) — grading it as "favourite won" polluted the hit-rate with the base favourite-win rate, and its high frozen-ness score always cleared the backtest threshold. - Add AnomalyKind.IsDirectional() (flip + steam = true, freeze = false). - AnomalyOutcomeEvaluator returns Unresolved for non-directional kinds (favourites still surfaced for display) so they don't distort calibration. - RunBacktestUseCase skips non-directional anomalies when building candidates. - Tests for the classification, the evaluator path, and the backtest skip. --- .../UseCases/RunBacktestUseCase.cs | 6 +++++ .../AnomalyOutcomeEvaluator.cs | 19 +++++++++++++++ .../Enums/AnomalyKindExtensions.cs | 24 +++++++++++++++++++ .../UseCases/RunBacktestUseCaseTests.cs | 19 +++++++++++++++ .../AnomalyOutcomeEvaluatorTests.cs | 21 ++++++++++++++++ .../Enums/AnomalyKindExtensionsTests.cs | 16 +++++++++++++ 6 files changed, 105 insertions(+) create mode 100644 src/Marathon.Domain/Enums/AnomalyKindExtensions.cs create mode 100644 tests/Marathon.Domain.Tests/Enums/AnomalyKindExtensionsTests.cs diff --git a/src/Marathon.Application/UseCases/RunBacktestUseCase.cs b/src/Marathon.Application/UseCases/RunBacktestUseCase.cs index 4f3e216..e1eba16 100644 --- a/src/Marathon.Application/UseCases/RunBacktestUseCase.cs +++ b/src/Marathon.Application/UseCases/RunBacktestUseCase.cs @@ -3,6 +3,7 @@ using Marathon.Application.Storage; using Marathon.Domain.AnomalyDetection; using Marathon.Domain.Backtesting; using Marathon.Domain.Entities; +using Marathon.Domain.Enums; using Microsoft.Extensions.Logging; using DomainEventId = Marathon.Domain.ValueObjects.EventId; @@ -95,6 +96,11 @@ public sealed class RunBacktestUseCase { ct.ThrowIfCancellationRequested(); + // Only directional kinds are betting signals; SuspensionFreeze (favourite + // unchanged) is informational and must not be staked or it would skew ROI. + if (!anomaly.Kind.IsDirectional()) + continue; + // Cannot simulate a bet whose event hasn't been graded yet. if (!resultLookup.TryGetValue(anomaly.EventId, out var result)) continue; diff --git a/src/Marathon.Domain/AnomalyDetection/AnomalyOutcomeEvaluator.cs b/src/Marathon.Domain/AnomalyDetection/AnomalyOutcomeEvaluator.cs index 2678562..da7609c 100644 --- a/src/Marathon.Domain/AnomalyDetection/AnomalyOutcomeEvaluator.cs +++ b/src/Marathon.Domain/AnomalyDetection/AnomalyOutcomeEvaluator.cs @@ -63,6 +63,25 @@ public static class AnomalyOutcomeEvaluator var preFav = data.PreSuspension.Favourite; var postFav = data.PostSuspension.Favourite; + // Non-directional kinds (e.g. SuspensionFreeze — the favourite did NOT change) + // make no side prediction. Grading them as "favourite won" would just measure the + // base favourite-win rate, polluting the hit-rate and score-bin calibration, so we + // leave them Unresolved (the favourites are still surfaced for display). + if (!anomaly.Kind.IsDirectional()) + { + return new ResolvedAnomaly( + AnomalyId: anomaly.Id, + EventId: anomaly.EventId, + DetectedAt: anomaly.DetectedAt, + Score: anomaly.Score, + Kind: anomaly.Kind, + Sport: sport, + PreFlipFavourite: preFav, + PostFlipFavourite: postFav, + ActualWinner: result?.WinnerSide, + Outcome: AnomalyOutcomeKind.Unresolved); + } + if (result is null) { return new ResolvedAnomaly( diff --git a/src/Marathon.Domain/Enums/AnomalyKindExtensions.cs b/src/Marathon.Domain/Enums/AnomalyKindExtensions.cs new file mode 100644 index 0000000..830bca6 --- /dev/null +++ b/src/Marathon.Domain/Enums/AnomalyKindExtensions.cs @@ -0,0 +1,24 @@ +namespace Marathon.Domain.Enums; + +/// Semantic classification of anomaly kinds. +public static class AnomalyKindExtensions +{ + /// + /// Whether the kind makes a directional prediction — a specific side/favourite + /// expected to win — that can be graded against the result and bet on in a backtest. + /// + /// + /// and are + /// directional (they point at a favourite). is + /// informational — the line did NOT move — so "predicting" the unchanged favourite would + /// merely measure the base favourite-win rate; it is excluded from outcome grading and + /// from backtest staking so it does not distort detector calibration. + /// + public static bool IsDirectional(this AnomalyKind kind) => kind switch + { + AnomalyKind.SuspensionFlip => true, + AnomalyKind.SteamMove => true, + AnomalyKind.SuspensionFreeze => false, + _ => false, + }; +} diff --git a/tests/Marathon.Application.Tests/UseCases/RunBacktestUseCaseTests.cs b/tests/Marathon.Application.Tests/UseCases/RunBacktestUseCaseTests.cs index 280efd8..3bc7040 100644 --- a/tests/Marathon.Application.Tests/UseCases/RunBacktestUseCaseTests.cs +++ b/tests/Marathon.Application.Tests/UseCases/RunBacktestUseCaseTests.cs @@ -85,6 +85,25 @@ public sealed class RunBacktestUseCaseTests result.BetsPlaced.Should().BeGreaterThan(0, "the in-range graded anomaly produces a bet"); } + [Fact] + public async Task Should_SkipNonDirectionalAnomalies_When_BuildingCandidates() + { + // A SuspensionFreeze anomaly (favourite unchanged) must not be staked. + var id = new EventId("88888888"); + var freeze = new Anomaly( + Guid.NewGuid(), id, BaseTime, AnomalyKind.SuspensionFreeze, 0.9m, FlipEvidence); + + _anomalies.ListAsync(Arg.Any()) + .Returns(new[] { freeze }.ToList().AsReadOnly()); + _events.GetAsync(id, Arg.Any()).Returns(MakeEvent(id)); + _results.GetAsync(id, Arg.Any()) + .Returns(new EventResult(id, 0, 2, Side.Side2, DateTimeOffset.UtcNow)); + + var result = await CreateSut().ExecuteAsync(DefaultStrategy(), CancellationToken.None); + + result.BetsPlaced.Should().Be(0, "SuspensionFreeze is non-directional and must not be staked"); + } + [Fact] public async Task Should_ReturnEmptyResult_When_NoAnomaliesExist() { diff --git a/tests/Marathon.Domain.Tests/AnomalyDetection/AnomalyOutcomeEvaluatorTests.cs b/tests/Marathon.Domain.Tests/AnomalyDetection/AnomalyOutcomeEvaluatorTests.cs index cbaca0b..5c380f7 100644 --- a/tests/Marathon.Domain.Tests/AnomalyDetection/AnomalyOutcomeEvaluatorTests.cs +++ b/tests/Marathon.Domain.Tests/AnomalyDetection/AnomalyOutcomeEvaluatorTests.cs @@ -73,6 +73,27 @@ public sealed class AnomalyOutcomeEvaluatorTests verdict.ActualWinner.Should().Be(Side.Side1); } + [Fact] + public void Should_ReportUnresolved_When_KindIsNonDirectional() + { + // SuspensionFreeze is informational (the favourite did not change) — it must NOT be + // graded hit/miss even when a result exists, or its base favourite-win rate would + // pollute the calibration. Favourites are still surfaced for display. + var anomaly = new Anomaly( + Id: Guid.NewGuid(), + EventId: DefaultEventId, + DetectedAt: new DateTimeOffset(2026, 5, 10, 18, 5, 0, MoscowOffset), + Kind: AnomalyKind.SuspensionFreeze, + Score: 0.9m, + EvidenceJson: ThreeWayFlipJson); + var result = MakeResult(Side.Side2, s1: 0, s2: 2); + + var verdict = AnomalyOutcomeEvaluator.Evaluate(anomaly, new SportCode(6), result); + + verdict.Outcome.Should().Be(AnomalyOutcomeKind.Unresolved); + verdict.PostFlipFavourite.Should().Be(Side.Side2); + } + [Fact] public void Should_ReportMiss_When_DrawOccurred_AndPostFlipFavouriteIsNotDraw() { diff --git a/tests/Marathon.Domain.Tests/Enums/AnomalyKindExtensionsTests.cs b/tests/Marathon.Domain.Tests/Enums/AnomalyKindExtensionsTests.cs new file mode 100644 index 0000000..c2777f8 --- /dev/null +++ b/tests/Marathon.Domain.Tests/Enums/AnomalyKindExtensionsTests.cs @@ -0,0 +1,16 @@ +using FluentAssertions; +using Marathon.Domain.Enums; + +namespace Marathon.Domain.Tests.Enums; + +public sealed class AnomalyKindExtensionsTests +{ + [Theory] + [InlineData(AnomalyKind.SuspensionFlip, true)] + [InlineData(AnomalyKind.SteamMove, true)] + [InlineData(AnomalyKind.SuspensionFreeze, false)] + public void IsDirectional_Should_ClassifyKinds(AnomalyKind kind, bool expected) + { + kind.IsDirectional().Should().Be(expected); + } +}