fix(anomaly): exclude non-directional kinds from grading and backtest
Review follow-up (HIGH): the three detectors fed the same evaluator/backtest, but SuspensionFreeze is non-directional (favourite unchanged) — grading it as "favourite won" polluted the hit-rate with the base favourite-win rate, and its high frozen-ness score always cleared the backtest threshold. - Add AnomalyKind.IsDirectional() (flip + steam = true, freeze = false). - AnomalyOutcomeEvaluator returns Unresolved for non-directional kinds (favourites still surfaced for display) so they don't distort calibration. - RunBacktestUseCase skips non-directional anomalies when building candidates. - Tests for the classification, the evaluator path, and the backtest skip.
This commit is contained in:
@@ -3,6 +3,7 @@ using Marathon.Application.Storage;
|
|||||||
using Marathon.Domain.AnomalyDetection;
|
using Marathon.Domain.AnomalyDetection;
|
||||||
using Marathon.Domain.Backtesting;
|
using Marathon.Domain.Backtesting;
|
||||||
using Marathon.Domain.Entities;
|
using Marathon.Domain.Entities;
|
||||||
|
using Marathon.Domain.Enums;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
using DomainEventId = Marathon.Domain.ValueObjects.EventId;
|
using DomainEventId = Marathon.Domain.ValueObjects.EventId;
|
||||||
|
|
||||||
@@ -95,6 +96,11 @@ public sealed class RunBacktestUseCase
|
|||||||
{
|
{
|
||||||
ct.ThrowIfCancellationRequested();
|
ct.ThrowIfCancellationRequested();
|
||||||
|
|
||||||
|
// Only directional kinds are betting signals; SuspensionFreeze (favourite
|
||||||
|
// unchanged) is informational and must not be staked or it would skew ROI.
|
||||||
|
if (!anomaly.Kind.IsDirectional())
|
||||||
|
continue;
|
||||||
|
|
||||||
// Cannot simulate a bet whose event hasn't been graded yet.
|
// Cannot simulate a bet whose event hasn't been graded yet.
|
||||||
if (!resultLookup.TryGetValue(anomaly.EventId, out var result))
|
if (!resultLookup.TryGetValue(anomaly.EventId, out var result))
|
||||||
continue;
|
continue;
|
||||||
|
|||||||
@@ -63,6 +63,25 @@ public static class AnomalyOutcomeEvaluator
|
|||||||
var preFav = data.PreSuspension.Favourite;
|
var preFav = data.PreSuspension.Favourite;
|
||||||
var postFav = data.PostSuspension.Favourite;
|
var postFav = data.PostSuspension.Favourite;
|
||||||
|
|
||||||
|
// Non-directional kinds (e.g. SuspensionFreeze — the favourite did NOT change)
|
||||||
|
// make no side prediction. Grading them as "favourite won" would just measure the
|
||||||
|
// base favourite-win rate, polluting the hit-rate and score-bin calibration, so we
|
||||||
|
// leave them Unresolved (the favourites are still surfaced for display).
|
||||||
|
if (!anomaly.Kind.IsDirectional())
|
||||||
|
{
|
||||||
|
return new ResolvedAnomaly(
|
||||||
|
AnomalyId: anomaly.Id,
|
||||||
|
EventId: anomaly.EventId,
|
||||||
|
DetectedAt: anomaly.DetectedAt,
|
||||||
|
Score: anomaly.Score,
|
||||||
|
Kind: anomaly.Kind,
|
||||||
|
Sport: sport,
|
||||||
|
PreFlipFavourite: preFav,
|
||||||
|
PostFlipFavourite: postFav,
|
||||||
|
ActualWinner: result?.WinnerSide,
|
||||||
|
Outcome: AnomalyOutcomeKind.Unresolved);
|
||||||
|
}
|
||||||
|
|
||||||
if (result is null)
|
if (result is null)
|
||||||
{
|
{
|
||||||
return new ResolvedAnomaly(
|
return new ResolvedAnomaly(
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
namespace Marathon.Domain.Enums;
|
||||||
|
|
||||||
|
/// <summary>Semantic classification of anomaly kinds.</summary>
|
||||||
|
public static class AnomalyKindExtensions
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Whether the kind makes a <i>directional</i> prediction — a specific side/favourite
|
||||||
|
/// expected to win — that can be graded against the result and bet on in a backtest.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// <see cref="AnomalyKind.SuspensionFlip"/> and <see cref="AnomalyKind.SteamMove"/> are
|
||||||
|
/// directional (they point at a favourite). <see cref="AnomalyKind.SuspensionFreeze"/> is
|
||||||
|
/// informational — the line did NOT move — so "predicting" the unchanged favourite would
|
||||||
|
/// merely measure the base favourite-win rate; it is excluded from outcome grading and
|
||||||
|
/// from backtest staking so it does not distort detector calibration.
|
||||||
|
/// </remarks>
|
||||||
|
public static bool IsDirectional(this AnomalyKind kind) => kind switch
|
||||||
|
{
|
||||||
|
AnomalyKind.SuspensionFlip => true,
|
||||||
|
AnomalyKind.SteamMove => true,
|
||||||
|
AnomalyKind.SuspensionFreeze => false,
|
||||||
|
_ => false,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -85,6 +85,25 @@ public sealed class RunBacktestUseCaseTests
|
|||||||
result.BetsPlaced.Should().BeGreaterThan(0, "the in-range graded anomaly produces a bet");
|
result.BetsPlaced.Should().BeGreaterThan(0, "the in-range graded anomaly produces a bet");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Should_SkipNonDirectionalAnomalies_When_BuildingCandidates()
|
||||||
|
{
|
||||||
|
// A SuspensionFreeze anomaly (favourite unchanged) must not be staked.
|
||||||
|
var id = new EventId("88888888");
|
||||||
|
var freeze = new Anomaly(
|
||||||
|
Guid.NewGuid(), id, BaseTime, AnomalyKind.SuspensionFreeze, 0.9m, FlipEvidence);
|
||||||
|
|
||||||
|
_anomalies.ListAsync(Arg.Any<CancellationToken>())
|
||||||
|
.Returns(new[] { freeze }.ToList().AsReadOnly());
|
||||||
|
_events.GetAsync(id, Arg.Any<CancellationToken>()).Returns(MakeEvent(id));
|
||||||
|
_results.GetAsync(id, Arg.Any<CancellationToken>())
|
||||||
|
.Returns(new EventResult(id, 0, 2, Side.Side2, DateTimeOffset.UtcNow));
|
||||||
|
|
||||||
|
var result = await CreateSut().ExecuteAsync(DefaultStrategy(), CancellationToken.None);
|
||||||
|
|
||||||
|
result.BetsPlaced.Should().Be(0, "SuspensionFreeze is non-directional and must not be staked");
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task Should_ReturnEmptyResult_When_NoAnomaliesExist()
|
public async Task Should_ReturnEmptyResult_When_NoAnomaliesExist()
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -73,6 +73,27 @@ public sealed class AnomalyOutcomeEvaluatorTests
|
|||||||
verdict.ActualWinner.Should().Be(Side.Side1);
|
verdict.ActualWinner.Should().Be(Side.Side1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void Should_ReportUnresolved_When_KindIsNonDirectional()
|
||||||
|
{
|
||||||
|
// SuspensionFreeze is informational (the favourite did not change) — it must NOT be
|
||||||
|
// graded hit/miss even when a result exists, or its base favourite-win rate would
|
||||||
|
// pollute the calibration. Favourites are still surfaced for display.
|
||||||
|
var anomaly = new Anomaly(
|
||||||
|
Id: Guid.NewGuid(),
|
||||||
|
EventId: DefaultEventId,
|
||||||
|
DetectedAt: new DateTimeOffset(2026, 5, 10, 18, 5, 0, MoscowOffset),
|
||||||
|
Kind: AnomalyKind.SuspensionFreeze,
|
||||||
|
Score: 0.9m,
|
||||||
|
EvidenceJson: ThreeWayFlipJson);
|
||||||
|
var result = MakeResult(Side.Side2, s1: 0, s2: 2);
|
||||||
|
|
||||||
|
var verdict = AnomalyOutcomeEvaluator.Evaluate(anomaly, new SportCode(6), result);
|
||||||
|
|
||||||
|
verdict.Outcome.Should().Be(AnomalyOutcomeKind.Unresolved);
|
||||||
|
verdict.PostFlipFavourite.Should().Be(Side.Side2);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public void Should_ReportMiss_When_DrawOccurred_AndPostFlipFavouriteIsNotDraw()
|
public void Should_ReportMiss_When_DrawOccurred_AndPostFlipFavouriteIsNotDraw()
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
using FluentAssertions;
|
||||||
|
using Marathon.Domain.Enums;
|
||||||
|
|
||||||
|
namespace Marathon.Domain.Tests.Enums;
|
||||||
|
|
||||||
|
public sealed class AnomalyKindExtensionsTests
|
||||||
|
{
|
||||||
|
[Theory]
|
||||||
|
[InlineData(AnomalyKind.SuspensionFlip, true)]
|
||||||
|
[InlineData(AnomalyKind.SteamMove, true)]
|
||||||
|
[InlineData(AnomalyKind.SuspensionFreeze, false)]
|
||||||
|
public void IsDirectional_Should_ClassifyKinds(AnomalyKind kind, bool expected)
|
||||||
|
{
|
||||||
|
kind.IsDirectional().Should().Be(expected);
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user