Compare commits

..

4 Commits

Author SHA1 Message Date
537c0df198 Update tests 2021-10-29 22:39:27 +03:00
75aa615263 Clean up 2021-10-29 22:37:47 +03:00
a300c88cfb Try parallel 2021-10-29 22:09:43 +03:00
560ea23861 parallel 2021-10-29 20:47:04 +03:00

View File

@ -43,26 +43,23 @@ chunks n = L.unfoldr $ \case
run :: IO () run :: IO ()
run = do run = do
datDir <- fromMaybe "./" <$> lookupEnv "HOME" datDir <- fromMaybe "./" <$> lookupEnv "HOME"
let datFile = datDir </> ".addressbook.dat"
original <- Set.fromList . map LBS.toStrict . lbsLines <$> LBS.readFile datFile
xs <- LBS.getContents >>= stream xs <- LBS.getContents >>= stream
let set = original `Set.union` F.fold (parMap rseq F.fold (chunks 20 xs)) let set = F.fold (parMap rseq F.fold (chunks 20 xs))
runResourceT $ runResourceT $
runConduit $ runConduit $
CL.sourceList (Set.elems set) CL.sourceList (Set.elems set)
.| C.map (<> "\n") .| C.map (<> "\n")
.| CB.sinkFileCautious datFile .| CB.sinkFileCautious (datDir </> ".addressbook.dat")
where where
separate = \case separate = \case
From x -> [x] From x -> [x]
To xs -> F.toList xs To xs -> F.toList xs
-- A set of (locally) unique addresses. Composes with parMap -- A set of (locally) unique addresses. Composes with parMap
lbsLines = LBS.split (fromIntegral $ ord '\n')
stream :: LBS.ByteString -> IO [Set ByteString] stream :: LBS.ByteString -> IO [Set ByteString]
stream = stream =
traverse (unsafeInterleaveIO . parse . LBC.unpack) traverse (unsafeInterleaveIO . parse . LBC.unpack)
. filter (not . LBS.null) . filter (not . LBS.null)
. lbsLines . LBS.split (fromIntegral $ ord '\n')
parse path = parse path =
runResourceT $ runResourceT $
runConduit $ runConduit $