Split to multiproject

reddit_pub/src/Data/Config.hs

@@ -0,0 +1,80 @@
+{-# LANGUAGE DataKinds #-}
+{-# LANGUAGE DeriveGeneric #-}
+{-# LANGUAGE DerivingVia #-}
+{-# LANGUAGE TypeOperators #-}
+module Data.Config where
+
+import Control.Lens
+import Data.Text (Text)
+import Dhall (FromDhall, Generic, ToDhall, auto, inputFile)
+import Dhall.Deriving
+import Numeric.Natural (Natural)
+import Data.SubReddit (SubReddit)
+
+data Password
+  = Password Text
+  | File FilePath
+  deriving stock (Generic, Show)
+  deriving (FromDhall, ToDhall)
+    via (Codec AsIs Password)
+
+data AMQP = AMQP
+  { amqpVhost :: Text
+  , amqpUsername :: Text
+  , amqpPassword :: Password
+  , amqpHost :: Text
+  }
+  deriving stock (Generic, Show)
+  deriving (FromDhall, ToDhall)
+    via (Codec (Field (CamelCase <<< DropPrefix "amqp"))) AMQP
+
+host :: Lens' AMQP Text
+host = lens amqpHost (\ am txt -> am{amqpHost=txt})
+
+vhost :: Lens' AMQP Text
+vhost = lens amqpVhost (\ am txt -> am{amqpVhost=txt})
+
+username :: Lens' AMQP Text
+username = lens amqpUsername (\ am txt -> am{amqpUsername=txt})
+
+password :: Lens' AMQP Password
+password = lens amqpPassword (\ am txt -> am{amqpPassword=txt})
+
+data Fetcher = Fetcher
+  { fetcherSubreddit :: SubReddit
+  , fetcherEntries :: Natural
+  , fetcherQualifier :: Maybe Qualifier
+  }
+  deriving stock (Show, Generic)
+  deriving (FromDhall, ToDhall) via Codec (Field (CamelCase <<< DropPrefix "fetcher")) Fetcher
+
+subreddit :: Lens' Fetcher SubReddit
+subreddit = lens fetcherSubreddit (\ fe sr -> fe{fetcherSubreddit=sr})
+
+entries :: Lens' Fetcher Natural
+entries = lens fetcherEntries (\ fe nat -> fe{fetcherEntries=nat})
+
+data Qualifier = Top | Controversial
+  deriving stock (Show, Generic)
+  deriving (FromDhall, ToDhall) via Codec (Constructor TitleCase) Qualifier
+
+data Config = Config
+  { configAmqp :: AMQP
+  , configFetchers :: [Fetcher]
+  , configSqlite :: FilePath
+  }
+  deriving stock (Generic, Show)
+  deriving (FromDhall, ToDhall)
+    via (Codec (Field (CamelCase <<< DropPrefix "config"))) Config
+
+amqp :: Lens' Config AMQP
+amqp = lens configAmqp (\ con am -> con{configAmqp=am})
+
+fetchers :: Lens' Config [Fetcher]
+fetchers = lens configFetchers (\ con fes -> con{configFetchers=fes})
+
+sqlite :: Lens' Config FilePath
+sqlite = lens configSqlite (\ con s -> con{configSqlite=s})
+
+readConfig :: FilePath -> IO Config
+readConfig = inputFile auto

reddit_pub/src/Data/Deriving/Aeson.hs

@@ -0,0 +1,41 @@
+{-# LANGUAGE AllowAmbiguousTypes #-}
+{-# LANGUAGE FlexibleContexts #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+{-# LANGUAGE TypeApplications #-}
+{-# LANGUAGE TypeOperators #-}
+{-# LANGUAGE UndecidableInstances #-}
+module Data.Deriving.Aeson
+  ( AesonCodec(..)
+  , type (<<<)
+  , Field
+  , CamelCase
+  , DropPrefix
+  )
+  where
+
+import Control.Lens
+import Data.Aeson
+import qualified Data.Text.Strict.Lens as T
+import Dhall.Deriving
+import GHC.Generics (Generic, Rep)
+
+newtype AesonCodec codec a = AesonCodec a
+
+class ModifyAesonOptions a where
+  modifyAesonOptions :: Options -> Options
+
+instance (ModifyAesonOptions f, ModifyAesonOptions g) => ModifyAesonOptions (f <<< g) where
+  modifyAesonOptions = modifyAesonOptions @f . modifyAesonOptions @g
+
+instance TextFunction f => ModifyAesonOptions (Field f) where
+  modifyAesonOptions opts = opts{fieldLabelModifier = over T.packed (textFunction @f) }
+
+instance (Generic a, ModifyAesonOptions codec, GToJSON' Value Zero (Rep a))
+  => ToJSON (AesonCodec codec a) where
+  toJSON (AesonCodec a) =
+    genericToJSON (modifyAesonOptions @codec defaultOptions) a
+
+instance (ModifyAesonOptions codec, Generic a, GFromJSON Zero (Rep a))
+  => FromJSON (AesonCodec codec a) where
+  parseJSON va =
+    AesonCodec <$> genericParseJSON (modifyAesonOptions @codec defaultOptions) va

reddit_pub/src/Data/SubReddit.hs

@@ -0,0 +1,9 @@
+{-# LANGUAGE DerivingVia #-}
+module Data.SubReddit where
+
+import Dhall (FromDhall, ToDhall)
+
+newtype SubReddit = SubReddit { getSubReddit :: String }
+  deriving Show
+  deriving (FromDhall, ToDhall) via String
+

reddit_pub/src/Membership.hs

@@ -0,0 +1,20 @@
+{-# LANGUAGE QuasiQuotes #-}
+{-# LANGUAGE TypeApplications #-}
+module Membership where
+
+import Database.SQLite.Simple (Connection, Only (..))
+import Network.Reddit (RedditId)
+import qualified Database.SQLite.Simple as SQL
+import Database.SQLite.Simple.QQ (sql)
+import qualified Data.Foldable as F
+import Data.Monoid (Any(..))
+import Data.Text (Text)
+
+recordSeen :: Connection -> RedditId -> IO ()
+recordSeen conn rid = SQL.execute conn [sql|insert into membership (reddit_id) values (?) on conflict do nothing|] (Only rid)
+
+isSeen :: Connection -> RedditId -> IO Bool
+isSeen conn rid =
+   unwrap <$> SQL.query conn [sql|select reddit_id from membership  where reddit_id = ?|] (Only rid)
+  where
+    unwrap = getAny . F.foldMap' (Any . const @_ @Text True . fromOnly)

reddit_pub/src/MyLib.hs

@@ -0,0 +1,131 @@
+{-# LANGUAGE DataKinds #-}
+{-# LANGUAGE DeriveAnyClass #-}
+{-# LANGUAGE DeriveGeneric #-}
+{-# LANGUAGE DerivingVia #-}
+{-# LANGUAGE NumericUnderscores #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE TypeOperators #-}
+{-# LANGUAGE LambdaCase #-}
+module MyLib (defaultMain) where
+
+import Control.Concurrent (threadDelay)
+import Control.Exception (bracket)
+import Control.Lens
+import Control.Monad (forever, void)
+import Data.Aeson (FromJSON, ToJSON, Value)
+import qualified Data.Aeson as A
+import Data.Aeson.Lens (_String, key)
+import Data.Bool (bool)
+import Data.Config
+import Data.Deriving.Aeson
+import Data.Foldable (for_)
+import Data.Functor.Contravariant ((>$<))
+import Data.Text (Text)
+import qualified Data.Text.IO as TI
+import qualified Data.Text.Strict.Lens as T
+import qualified Database.SQLite.Simple as SQL
+import GHC.Generics (Generic)
+import qualified Membership
+import Network.AMQP
+       ( Channel
+       , DeliveryMode(Persistent)
+       , closeConnection
+       , declareExchange
+       , exchangeName
+       , exchangeType
+       , msgBody
+       , msgDeliveryMode
+       , newExchange
+       , newMsg
+       , openChannel
+       , openConnection
+       , publishMsg
+       )
+import Network.Reddit (RedditId(RedditId), publishEntries)
+import Network.Wreq.Session (newSession)
+import Reddit.Publish (Publish(..))
+import Text.Printf (printf)
+import Data.ByteString.Lazy (ByteString)
+
+data MessageType = Create | Update
+  deriving stock (Show, Eq, Generic)
+  deriving anyclass (ToJSON, FromJSON)
+
+data Message = Message
+  { messageType :: MessageType
+  , messageIdentifier :: RedditId
+  , messageContent :: Value
+  }
+  deriving stock (Show, Eq, Generic)
+  deriving (ToJSON, FromJSON)
+    via AesonCodec (Field (CamelCase <<< DropPrefix "message")) Message
+
+
+toMessage :: SQL.Connection -> Publish IO (Maybe Message) -> Publish IO Value
+toMessage sqlConn (Publish p) = Publish $ \entry -> do
+  case RedditId <$> (entry ^? key "id" . _String) of
+    Nothing -> p Nothing
+    Just redditId -> do
+      event <- bool Create Update <$> Membership.isSeen sqlConn redditId
+      p $ Just $ Message event redditId entry
+
+sqlRecorder :: SQL.Connection -> Publish IO (Maybe RedditId)
+sqlRecorder conn = Publish $ maybe (pure ()) (Membership.recordSeen conn)
+
+amqpPublisher :: Channel -> Text -> Publish IO (Maybe ByteString)
+amqpPublisher channel exchange = Publish $ \case
+  Nothing -> pure ()
+  Just lbs ->
+    void $ publishMsg channel exchange routingKey (message lbs)
+  where
+    routingKey = "doesn't matter on fanout"
+    message lbs = newMsg
+      { msgBody = lbs
+      , msgDeliveryMode = Just Persistent
+      }
+
+stdoutPublisher :: Publish IO String
+stdoutPublisher = Publish putStrLn
+
+data Fetch
+  = Fetch Fetcher
+  | PublishMessage Message
+  | ParseFailed
+
+fetchToLog :: Fetch -> String
+fetchToLog (Fetch fetcher) = printf "Refreshing %s" (show $ fetcherSubreddit fetcher)
+fetchToLog ParseFailed = printf "Failed parsing"
+fetchToLog (PublishMessage msg) = messageToLog msg
+  where
+    messageToLog :: Message -> String
+    messageToLog m = printf "Publishing %s as type %s" (show $ messageIdentifier m) (show $ messageType m)
+
+
+defaultMain :: FilePath -> IO ()
+defaultMain path = do
+  conf <- readConfig path
+  pass <- getPassword (conf ^. amqp . password)
+  let rabbitConnect = openConnection
+                  (conf ^. amqp . host . T.unpacked)
+                  (conf ^. amqp . vhost)
+                  (conf ^. amqp . username)
+                  pass
+  bracket rabbitConnect closeConnection $ \conn -> do
+    SQL.withConnection (conf ^. sqlite) $ \sqlConn -> do
+      SQL.execute_ sqlConn "create table if not exists membership (reddit_id primary key)"
+      chan <- openChannel conn
+      declareExchange chan newExchange { exchangeName = "reddit_posts", exchangeType = "fanout" }
+      sess <- newSession
+      let encoder = amqpPublisher chan "reddit_posts"
+          recorder = sqlRecorder sqlConn
+          publisher = (fmap A.encode >$< encoder) <> (fmap messageIdentifier >$< recorder) <> (maybe ParseFailed PublishMessage >$< logger)
+          logger = fetchToLog >$< stdoutPublisher
+      forever $ do
+        for_ (conf ^. fetchers) $ \fetcher -> do
+            publish logger (Fetch fetcher)
+            publishEntries (toMessage sqlConn publisher) sess fetcher
+        threadDelay (15 * 60_000_000)
+
+getPassword :: Password -> IO Text
+getPassword (Password p) = pure p
+getPassword (File path) = TI.readFile path

reddit_pub/src/Network/Reddit.hs

@@ -0,0 +1,46 @@
+{-# LANGUAGE DataKinds #-}
+{-# LANGUAGE DerivingVia #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE RankNTypes #-}
+module Network.Reddit where
+
+import Control.Lens
+import Data.Aeson (FromJSON, ToJSON, Value)
+import Data.Aeson.Lens
+import Data.Config
+import Data.SubReddit
+import Data.Text (Text)
+import Network.Wreq hiding (getWith)
+import Network.Wreq.Session (Session, getWith)
+import Pipes (Producer, (>->), for, runEffect)
+import qualified Pipes.Prelude as P
+import Reddit.Publish
+import Data.Maybe (maybeToList)
+import Control.Monad.Trans (liftIO)
+import Database.SQLite.Simple.ToField (ToField)
+import Database.SQLite.Simple.FromField (FromField)
+
+
+newtype RedditId = RedditId Text
+  deriving stock (Show, Eq)
+  deriving (ToJSON, FromJSON, ToField, FromField) via Text
+
+
+messages :: Session -> SubReddit -> Producer Value IO ()
+messages sess sre = P.unfoldr go Nothing >-> P.concat
+  where
+    go :: Maybe Text -> IO (Either () ([Value], Maybe Text))
+    go after = do
+      let opts = defaults & header "User-Agent" .~ ["reddit-pubsub"] & param "after" .~ maybeToList after
+      r <- getWith opts sess ("https://www.reddit.com/r/" <> getSubReddit sre <> ".json")
+      let msgs = r ^.. responseBody . key "data" . key "children" . _Array . traversed . key "data"
+          next = r ^? responseBody . key "data" . key "after" . _String
+      print next
+      pure $ Right (msgs, next)
+
+publishEntries :: Publish IO Value -> Session -> Fetcher -> IO ()
+publishEntries publisher sess fetcher =
+  runEffect $
+    for
+      (messages sess (fetcher ^. subreddit) >-> P.take (fromIntegral $ fetcher ^. entries))
+      (liftIO . publish publisher)