Fix supervisor and proper rate limiting

This commit is contained in:
MasseR committed 2026-09-16 12:28:48 +03:00
1 parent 64561718a5
commit 3d7f6cb3c6
12 files changed
+164 -247

No files matched your search

+19 -47
View File
@@ -2,26 +2,23 @@
module HomeAssistant.Runtime.Supervisor
( supervised
, Backoff(..)
, defaultBackoff
, backoffDelay
, nextAttempt
, Fatal(..)
) where
import Control.Concurrent (threadDelay)
import Control.Exception.Annotated
( Exception
, Handler (..)
, SomeException
, catches
, displayException
, throw
, checkpoint, Annotation (..)
)
import Data.Text (Text)
import qualified Data.Text as T
import Data.Time (NominalDiffTime, diffUTCTime, getCurrentTime)
import Data.Void (Void)
import Control.Monad.IO.Class (MonadIO)
import Control.Monad.Catch (MonadCatch, MonadMask)
import Katip (KatipContext, logFM, ls)
import Control.Retry (RetryStatus(..), recovering, limitRetries, fullJitterBackoff)
import Control.Monad (when)
import Katip.Core (Severity(..))
-- | A failure that cannot be fixed by restarting; propagates out of
-- 'supervised' and terminates the process.
@@ -30,28 +27,11 @@ newtype Fatal = Fatal Text
instance Exception Fatal
data Backoff = Backoff
{ backoffBase :: NominalDiffTime -- ^ Delay before the first restart
, backoffCap :: NominalDiffTime -- ^ Maximum delay between restarts
, backoffQuiet :: NominalDiffTime -- ^ Uptime after which the delay resets
}
deriving (Eq, Show)
defaultBackoff :: Backoff
defaultBackoff = Backoff 0.1 30 30
data SupervisorException = RetriesExhausted
deriving Show
-- | Delay before the @attempt@-th restart: doubles from base, clamped at cap.
backoffDelay :: Backoff -> Int -> NominalDiffTime
backoffDelay (Backoff base cap _) attempt = go (max 0 (attempt - 1)) base
where
go 0 d = d
go n d = go (n - 1) (min cap (d * 2))
-- | Attempt number to use after a crash that ran for the given uptime.
nextAttempt :: Backoff -> NominalDiffTime -> Int -> Int
nextAttempt (Backoff _ _ quiet) uptime attempt
| uptime >= quiet = 1
| otherwise = attempt + 1
instance Exception SupervisorException
-- | Runs the action forever, restarting it with backoff after synchronous
-- exceptions; async exceptions propagate. 'Fatal' is rethrown (crashing the
@@ -59,22 +39,14 @@ nextAttempt (Backoff _ _ quiet) uptime attempt
-- must not be wrapped in checkpoints around 'Fatal'-throwing code: a
-- doubly-wrapped 'Fatal' is indistinguishable from a crash and would be
-- restarted instead of escalated.
supervised :: Text -> Backoff -> IO Void -> IO Void
supervised name backoff action = go 1
supervised :: (MonadIO m, MonadCatch m, KatipContext m, MonadMask m) => Text -> m Void -> m Void
supervised name action = checkpoint (Annotation name) $ do
recovering retryPolicy handlers $ \RetryStatus{rsIterNumber} -> do
when (rsIterNumber > 0) $ logFM WarningS $ ls $ "Child crashed, retry no " <> show rsIterNumber
action
where
go attempt = do
start <- getCurrentTime
outcome <- (action >> pure (Nothing :: Maybe (Either Fatal SomeException))) `catches`
[ Handler $ \(f :: Fatal) -> pure (Just (Left f))
, Handler $ \(e :: SomeException) -> pure (Just (Right e))
handlers =
[ \_retryStatus -> Handler $ \(_ :: Fatal) -> pure False
, \_retryStatus -> Handler $ \(_ :: SomeException) -> pure True
]
end <- getCurrentTime
case outcome of
Nothing -> error "unreachable: supervised action returned"
Just (Left f) -> throw f
Just (Right e) -> do
putStrLn $ "[" <> T.unpack name <> "] attempt " <> show attempt <> " crashed: " <> displayException e
let delay = backoffDelay backoff attempt
putStrLn $ "[" <> T.unpack name <> "] restarting in " <> show delay
threadDelay (round (realToFrac delay * 1000000 :: Double))
go (nextAttempt backoff (diffUTCTime end start) attempt)
retryPolicy = fullJitterBackoff 50000 <> limitRetries 5