Fix supervisor and proper rate limiting
This commit is contained in:
1 parent
64561718a5
commit
3d7f6cb3c6
12 files changed
+164
-247
No files matched your search
@@ -2,26 +2,23 @@
|
||||
|
||||
module HomeAssistant.Runtime.Supervisor
|
||||
( supervised
|
||||
, Backoff(..)
|
||||
, defaultBackoff
|
||||
, backoffDelay
|
||||
, nextAttempt
|
||||
, Fatal(..)
|
||||
) where
|
||||
|
||||
import Control.Concurrent (threadDelay)
|
||||
import Control.Exception.Annotated
|
||||
( Exception
|
||||
, Handler (..)
|
||||
, SomeException
|
||||
, catches
|
||||
, displayException
|
||||
, throw
|
||||
, checkpoint, Annotation (..)
|
||||
)
|
||||
import Data.Text (Text)
|
||||
import qualified Data.Text as T
|
||||
import Data.Time (NominalDiffTime, diffUTCTime, getCurrentTime)
|
||||
import Data.Void (Void)
|
||||
import Control.Monad.IO.Class (MonadIO)
|
||||
import Control.Monad.Catch (MonadCatch, MonadMask)
|
||||
import Katip (KatipContext, logFM, ls)
|
||||
import Control.Retry (RetryStatus(..), recovering, limitRetries, fullJitterBackoff)
|
||||
import Control.Monad (when)
|
||||
import Katip.Core (Severity(..))
|
||||
|
||||
-- | A failure that cannot be fixed by restarting; propagates out of
|
||||
-- 'supervised' and terminates the process.
|
||||
@@ -30,28 +27,11 @@ newtype Fatal = Fatal Text
|
||||
|
||||
instance Exception Fatal
|
||||
|
||||
data Backoff = Backoff
|
||||
{ backoffBase :: NominalDiffTime -- ^ Delay before the first restart
|
||||
, backoffCap :: NominalDiffTime -- ^ Maximum delay between restarts
|
||||
, backoffQuiet :: NominalDiffTime -- ^ Uptime after which the delay resets
|
||||
}
|
||||
deriving (Eq, Show)
|
||||
|
||||
defaultBackoff :: Backoff
|
||||
defaultBackoff = Backoff 0.1 30 30
|
||||
data SupervisorException = RetriesExhausted
|
||||
deriving Show
|
||||
|
||||
-- | Delay before the @attempt@-th restart: doubles from base, clamped at cap.
|
||||
backoffDelay :: Backoff -> Int -> NominalDiffTime
|
||||
backoffDelay (Backoff base cap _) attempt = go (max 0 (attempt - 1)) base
|
||||
where
|
||||
go 0 d = d
|
||||
go n d = go (n - 1) (min cap (d * 2))
|
||||
|
||||
-- | Attempt number to use after a crash that ran for the given uptime.
|
||||
nextAttempt :: Backoff -> NominalDiffTime -> Int -> Int
|
||||
nextAttempt (Backoff _ _ quiet) uptime attempt
|
||||
| uptime >= quiet = 1
|
||||
| otherwise = attempt + 1
|
||||
instance Exception SupervisorException
|
||||
|
||||
-- | Runs the action forever, restarting it with backoff after synchronous
|
||||
-- exceptions; async exceptions propagate. 'Fatal' is rethrown (crashing the
|
||||
@@ -59,22 +39,14 @@ nextAttempt (Backoff _ _ quiet) uptime attempt
|
||||
-- must not be wrapped in checkpoints around 'Fatal'-throwing code: a
|
||||
-- doubly-wrapped 'Fatal' is indistinguishable from a crash and would be
|
||||
-- restarted instead of escalated.
|
||||
supervised :: Text -> Backoff -> IO Void -> IO Void
|
||||
supervised name backoff action = go 1
|
||||
supervised :: (MonadIO m, MonadCatch m, KatipContext m, MonadMask m) => Text -> m Void -> m Void
|
||||
supervised name action = checkpoint (Annotation name) $ do
|
||||
recovering retryPolicy handlers $ \RetryStatus{rsIterNumber} -> do
|
||||
when (rsIterNumber > 0) $ logFM WarningS $ ls $ "Child crashed, retry no " <> show rsIterNumber
|
||||
action
|
||||
where
|
||||
go attempt = do
|
||||
start <- getCurrentTime
|
||||
outcome <- (action >> pure (Nothing :: Maybe (Either Fatal SomeException))) `catches`
|
||||
[ Handler $ \(f :: Fatal) -> pure (Just (Left f))
|
||||
, Handler $ \(e :: SomeException) -> pure (Just (Right e))
|
||||
handlers =
|
||||
[ \_retryStatus -> Handler $ \(_ :: Fatal) -> pure False
|
||||
, \_retryStatus -> Handler $ \(_ :: SomeException) -> pure True
|
||||
]
|
||||
end <- getCurrentTime
|
||||
case outcome of
|
||||
Nothing -> error "unreachable: supervised action returned"
|
||||
Just (Left f) -> throw f
|
||||
Just (Right e) -> do
|
||||
putStrLn $ "[" <> T.unpack name <> "] attempt " <> show attempt <> " crashed: " <> displayException e
|
||||
let delay = backoffDelay backoff attempt
|
||||
putStrLn $ "[" <> T.unpack name <> "] restarting in " <> show delay
|
||||
threadDelay (round (realToFrac delay * 1000000 :: Double))
|
||||
go (nextAttempt backoff (diffUTCTime end start) attempt)
|
||||
retryPolicy = fullJitterBackoff 50000 <> limitRetries 5
|
||||
Reference in new issue
Block a user