Avoid unneeded writes
This commit is contained in:
parent
03860c370a
commit
00e585281c
@ -29,6 +29,7 @@ import qualified Data.ByteString.Lazy.Char8 as LBC
|
||||
import System.IO.Unsafe (unsafeInterleaveIO)
|
||||
import Control.Parallel.Strategies (rseq, parMap)
|
||||
import qualified Data.List as L
|
||||
import Control.Monad (unless)
|
||||
|
||||
combine :: (MonadUnliftIO m, MonadResource m, MonadThrow m, MonadIO m) => ConduitM FilePath Header m ()
|
||||
combine = await >>= \case
|
||||
@ -47,11 +48,12 @@ run = do
|
||||
original <- Set.fromList . map LBS.toStrict . lbsLines <$> LBS.readFile datFile
|
||||
xs <- LBS.getContents >>= stream
|
||||
let set = original `Set.union` F.fold (parMap rseq F.fold (chunks 20 xs))
|
||||
runResourceT $
|
||||
runConduit $
|
||||
CL.sourceList (Set.elems set)
|
||||
.| C.map (<> "\n")
|
||||
.| CB.sinkFileCautious datFile
|
||||
unless (original == set) $
|
||||
runResourceT $
|
||||
runConduit $
|
||||
CL.sourceList (Set.elems set)
|
||||
.| C.map (<> "\n")
|
||||
.| CB.sinkFileCautious datFile
|
||||
where
|
||||
separate = \case
|
||||
From x -> [x]
|
||||
|
Loading…
Reference in New Issue
Block a user