MasseR/addressbook
1
0
Fork 0
Code Issues 1 Pull Requests Releases Wiki Activity

Share old info

Browse Source
This commit is contained in:
Mats Rauhala
2021-10-29 22:59:04 +03:00
parent 41c666fe93
commit 03860c370a
1 changed files with 6 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/Control/Addressbook/Streaming.hs
View File

@@ -43,23 +43,26 @@ chunks n = L.unfoldr $ \case
run :: IO () run :: IO ()
run = do run = do
datDir <- fromMaybe "./" <$> lookupEnv "HOME" datDir <- fromMaybe "./" <$> lookupEnv "HOME"
let datFile = datDir </> ".addressbook.dat"
original <- Set.fromList . map LBS.toStrict . lbsLines <$> LBS.readFile datFile
xs <- LBS.getContents >>= stream xs <- LBS.getContents >>= stream
let set = F.fold (parMap rseq F.fold (chunks 20 xs)) let set = original `Set.union` F.fold (parMap rseq F.fold (chunks 20 xs))
runResourceT $ runResourceT $
runConduit $ runConduit $
CL.sourceList (Set.elems set) CL.sourceList (Set.elems set)
.| C.map (<> "\n") .| C.map (<> "\n")
.| CB.sinkFileCautious (datDir </> ".addressbook.dat") .| CB.sinkFileCautious datFile
where where
separate = \case separate = \case
From x -> [x] From x -> [x]
To xs -> F.toList xs To xs -> F.toList xs
-- A set of (locally) unique addresses. Composes with parMap -- A set of (locally) unique addresses. Composes with parMap
lbsLines = LBS.split (fromIntegral $ ord '\n')
stream :: LBS.ByteString -> IO [Set ByteString] stream :: LBS.ByteString -> IO [Set ByteString]
stream = stream =
traverse (unsafeInterleaveIO . parse . LBC.unpack) traverse (unsafeInterleaveIO . parse . LBC.unpack)
. filter (not . LBS.null) . filter (not . LBS.null)
. LBS.split (fromIntegral $ ord '\n') . lbsLines
parse path = parse path =
runResourceT $ runResourceT $
runConduit $ runConduit $