Compare commits
13
Commits
b38294c784
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
db94983590 | ||
|
|
8c71d04053 | ||
|
|
cb1972225f | ||
|
|
f4a79b4251 | ||
|
|
f5197caa77 | ||
|
|
f54aaa6e95 | ||
|
|
6ead08af22 | ||
|
|
a4b5b3c8a0 | ||
|
|
66018dca53 | ||
|
|
51a1e49fb7 | ||
|
|
c099a4f105 | ||
|
|
813ec1fda5 | ||
|
|
e87aae3144 |
No files matched your search
+18
-14
@@ -1,10 +1,11 @@
|
||||
{ mkDerivation, aeson, annotated-exception, base, bytestring
|
||||
, containers, directory, ekg-core, exceptions, filepath
|
||||
, generic-lens, http-client, http-types, katip, lens, lens-aeson
|
||||
, lib, monad-control, mtl, optparse-generic, pandoc, pipes
|
||||
, pipes-concurrency, retry, servant, servant-server, stm, streamly
|
||||
, streamly-core, text, time, transformers-base, unliftio
|
||||
, unordered-containers, wai, warp, wreq
|
||||
, cereal, containers, directory, ekg-core, exceptions, filepath
|
||||
, generic-lens, hedgehog, hspec, hspec-hedgehog, http-client
|
||||
, http-types, katip, lens, lens-aeson, lib, monad-control, mtl
|
||||
, optparse-generic, pandoc, pipes, pipes-concurrency, pipes-safe
|
||||
, retry, servant, servant-server, sqlite-simple, stm, streamly
|
||||
, streamly-core, template-haskell, text, time, transformers-base
|
||||
, unliftio, unordered-containers, vector, wai, warp, wreq
|
||||
}:
|
||||
mkDerivation {
|
||||
pname = "wallabag-tagger";
|
||||
@@ -13,15 +14,18 @@ mkDerivation {
|
||||
isLibrary = true;
|
||||
isExecutable = true;
|
||||
libraryHaskellDepends = [
|
||||
aeson annotated-exception base bytestring containers directory
|
||||
ekg-core exceptions filepath generic-lens http-client http-types
|
||||
katip lens lens-aeson monad-control mtl optparse-generic pandoc
|
||||
pipes pipes-concurrency retry servant servant-server stm streamly
|
||||
streamly-core text time transformers-base unliftio
|
||||
unordered-containers wai warp wreq
|
||||
aeson annotated-exception base bytestring cereal containers
|
||||
directory ekg-core exceptions filepath generic-lens http-client
|
||||
http-types katip lens lens-aeson monad-control mtl optparse-generic
|
||||
pandoc pipes pipes-concurrency pipes-safe retry servant
|
||||
servant-server sqlite-simple stm streamly streamly-core
|
||||
template-haskell text time transformers-base unliftio
|
||||
unordered-containers vector wai warp wreq
|
||||
];
|
||||
executableHaskellDepends = [ base ];
|
||||
testHaskellDepends = [ base ];
|
||||
license = lib.licenses.bsd3;
|
||||
testHaskellDepends = [
|
||||
base hedgehog hspec hspec-hedgehog sqlite-simple text vector
|
||||
];
|
||||
license = lib.meta.getLicenseFromSpdxId "BSD-3-Clause";
|
||||
mainProgram = "wallabag-tagger";
|
||||
}
|
||||
Generated
+8
-7
@@ -5,11 +5,11 @@
|
||||
"systems": "systems"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1731533236,
|
||||
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
|
||||
"lastModified": 1791914995,
|
||||
"narHash": "sha256-Zf/xfQEAasnwnmEfHqW/4hjDE/s44yn2hD4RuVIZRjA=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
|
||||
"rev": "b7a79d3fea1e214f6d668d521cecec0c01e26544",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -20,10 +20,11 @@
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 0,
|
||||
"narHash": "sha256-hw63HnwnqU3ZQfsMclLhMvOezpM7RSB0dMAtD5/sOiw=",
|
||||
"path": "/nix/store/4m21g6b3bq78vn5skaxjmqwg7fnir87j-source",
|
||||
"type": "path"
|
||||
"lastModified": 1793977579,
|
||||
"narHash": "sha256-4/Yzr1fWTh/zBAR1tR84ZQ1Zhmor8AXwgD4AS7sPn60=",
|
||||
"rev": "4fe25e12eb9f4c744966dd3de5a05246cf098601",
|
||||
"type": "tarball",
|
||||
"url": "https://releases.nixos.org/nixpkgs/nixpkgs-26.11pre1109744.4fe25e12eb9f/nixexprs.tar.zst"
|
||||
},
|
||||
"original": {
|
||||
"id": "nixpkgs",
|
||||
|
||||
@@ -12,7 +12,11 @@
|
||||
let pkgs = nixpkgs.legacyPackages.${system};
|
||||
hp = nixpkgs.legacyPackages.${system}.haskellPackages.override (old: {
|
||||
overrides = pkgs.lib.composeExtensions (old.overrides or (_: _: {})) (f: p: {
|
||||
wallabag-tagger = f.callPackage ./. {};
|
||||
wallabag-tagger = pkgs.haskell.lib.overrideCabal (f.callPackage ./. {}) (_: {
|
||||
preBuild = ''
|
||||
export COMMIT="${self.rev or "unknown"}"
|
||||
'';
|
||||
});
|
||||
});
|
||||
});
|
||||
in rec {
|
||||
|
||||
@@ -47,7 +47,7 @@ completions c prompt = checkpointCallStack $ do
|
||||
let opts = defaults & header "Authorization" .~ ["Bearer " <> c ^. openrouterKey . re utf8]
|
||||
let j =
|
||||
object
|
||||
[ "model" .= ("openai/gpt-4o-mini" :: Text),
|
||||
[ "model" .= ("~deepseek/deepseek-flash-latest" :: Text),
|
||||
"messages"
|
||||
.= [object ["role" .= ("user" :: Text), "content" .= prompt]]
|
||||
]
|
||||
|
||||
@@ -0,0 +1,360 @@
|
||||
{-# LANGUAGE LambdaCase #-}
|
||||
{-# LANGUAGE OverloadedStrings #-}
|
||||
{-# LANGUAGE StrictData #-}
|
||||
{-# OPTIONS_GHC -Wall #-}
|
||||
{-# LANGUAGE RecordWildCards #-}
|
||||
|
||||
module Embeddings
|
||||
( generateEmbedding,
|
||||
indexEntries,
|
||||
search,
|
||||
chunkArticle,
|
||||
tokensApprox,
|
||||
chunkMaxTokens,
|
||||
Embedding(..), -- for testing
|
||||
) where
|
||||
|
||||
import Control.Exception.Annotated (checkpointCallStack)
|
||||
import Control.Lens (re, traversed, view, (&), (.~), (^.), (^..), (^?))
|
||||
import Control.Monad (when)
|
||||
import Control.Monad.Catch (MonadCatch, MonadThrow, MonadMask)
|
||||
import Control.Monad.Reader (MonadReader, liftIO)
|
||||
import Creds (Creds, openrouterKey)
|
||||
import Data.Aeson (object, (.=))
|
||||
import Data.Aeson.Lens (key, _Array, _Double)
|
||||
import Data.Functor.Contravariant ((>$<))
|
||||
import Data.Generics.Product (HasType, typed)
|
||||
import Data.List (sortOn)
|
||||
import Data.Map.Strict qualified as Map
|
||||
import Data.Ord (Down (..))
|
||||
import Data.Text (Text)
|
||||
import Data.Text qualified as T
|
||||
import Data.Text.Strict.Lens (utf8)
|
||||
import Data.Vector.Unboxed qualified as UV
|
||||
import Database.SQLite.Simple (Only (..), open, close, SQLData (SQLBlob), ResultError (ConversionFailed))
|
||||
import Database.SQLite.Simple qualified as SQL
|
||||
import GHC.Stack (HasCallStack)
|
||||
import HTTP (HTTPMetric, Retry (SafeTransient), Trace, logFormatter, postWith, tracerMetrics)
|
||||
import Katip qualified as K
|
||||
import M (HasEmbeddingMetrics, HasKatipTracer, HasWallabagMetrics, embeddingMetrics, katipTracer, KatipTracer)
|
||||
import Network.Wreq (defaults, header, responseBody)
|
||||
import Network.Wreq.Session (Session)
|
||||
import Pipes (lift, (>->))
|
||||
import Pipes qualified
|
||||
import Tracer (Tracer (..))
|
||||
import UnliftIO (MonadIO, MonadUnliftIO)
|
||||
import Wallabag (Entry, Scan (Everything), content, identifier, title, url)
|
||||
import qualified Wallabag
|
||||
import Wallabag.Auth (Auth)
|
||||
import GHC.Generics (Generic)
|
||||
import Database.SQLite.Simple.ToField (ToField(..))
|
||||
import Database.SQLite.Simple.FromField (FromField(..), fieldData, returnError)
|
||||
import qualified Data.Serialize as Cereal
|
||||
import qualified Pipes.Safe
|
||||
import Pipes.Safe (MonadSafe)
|
||||
import qualified Pipes.Prelude
|
||||
|
||||
embeddingUrl :: String
|
||||
embeddingUrl = "https://openrouter.ai/api/v1/embeddings"
|
||||
|
||||
mkTracerMetrics ::
|
||||
( MonadReader r m,
|
||||
HasEmbeddingMetrics r HTTPMetric,
|
||||
HasKatipTracer r KatipTracer
|
||||
) =>
|
||||
m (Tracer Trace)
|
||||
mkTracerMetrics = do
|
||||
l <- view katipTracer
|
||||
m <- view embeddingMetrics
|
||||
pure $ (logFormatter >$< l) <> tracerMetrics m
|
||||
|
||||
generateEmbedding ::
|
||||
( MonadIO m,
|
||||
HasType Session r,
|
||||
HasType Creds r,
|
||||
MonadReader r m,
|
||||
HasEmbeddingMetrics r HTTPMetric,
|
||||
HasKatipTracer r KatipTracer,
|
||||
MonadCatch m,
|
||||
K.KatipContext m
|
||||
) =>
|
||||
Text ->
|
||||
m (Maybe (UV.Vector Float))
|
||||
generateEmbedding input = checkpointCallStack $ do
|
||||
c <- view $ typed @Creds
|
||||
let opts = defaults & header "Authorization" .~ ["Bearer " <> c ^. openrouterKey . re utf8]
|
||||
let j =
|
||||
object
|
||||
[ "model" .= ("text-embedding-3-small" :: Text),
|
||||
"input" .= input
|
||||
]
|
||||
tm <- mkTracerMetrics
|
||||
session <- view $ typed @Session
|
||||
r <- liftIO $ postWith tm session SafeTransient opts embeddingUrl j
|
||||
case r ^? responseBody . key "data" . _Array . traversed . key "embedding" of
|
||||
Just embeddingVal -> do
|
||||
let floats = embeddingVal ^.. _Array . traversed . _Double
|
||||
if null floats
|
||||
then do
|
||||
K.logFM K.ErrorS $ K.ls $ "Parsing gave empty embedding array from: " <> T.pack (show (r ^. responseBody))
|
||||
pure Nothing
|
||||
else pure $ Just $ UV.fromList (map realToFrac floats)
|
||||
Nothing -> do
|
||||
K.logFM K.ErrorS $ K.ls $ "Couldn't find embedding in response: " <> T.pack (show (r ^. responseBody))
|
||||
pure Nothing
|
||||
|
||||
cosineSimilarity :: UV.Vector Float -> UV.Vector Float -> Float
|
||||
cosineSimilarity a b = dot / (normA * normB)
|
||||
where
|
||||
dot = UV.sum $ UV.zipWith (*) a b
|
||||
normA = sqrt $ UV.sum $ UV.map (^ (2 :: Int)) a
|
||||
normB = sqrt $ UV.sum $ UV.map (^ (2 :: Int)) b
|
||||
|
||||
indexEntries ::
|
||||
( MonadIO m,
|
||||
HasType Session r,
|
||||
HasType Auth r,
|
||||
HasType Creds r,
|
||||
MonadReader r m,
|
||||
K.KatipContext m,
|
||||
HasWallabagMetrics r HTTPMetric,
|
||||
HasEmbeddingMetrics r HTTPMetric,
|
||||
HasKatipTracer r KatipTracer,
|
||||
MonadThrow m,
|
||||
MonadCatch m,
|
||||
HasCallStack,
|
||||
MonadUnliftIO m
|
||||
) =>
|
||||
FilePath ->
|
||||
Bool ->
|
||||
m ()
|
||||
indexEntries dbPath rebuild = do
|
||||
liftIO $ do
|
||||
conn <- open dbPath
|
||||
SQL.execute_ conn
|
||||
"CREATE TABLE IF NOT EXISTS embeddings (entry_id INTEGER NOT NULL, chunk_idx INTEGER NOT NULL, url TEXT NOT NULL, title TEXT NOT NULL, embedding TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT (datetime('now')), PRIMARY KEY (entry_id, chunk_idx))"
|
||||
when rebuild $ SQL.execute_ conn "DELETE FROM embeddings"
|
||||
close conn
|
||||
Pipes.runEffect $
|
||||
Pipes.for
|
||||
(Wallabag.sourceEntries Everything)
|
||||
(lift . processEntry dbPath)
|
||||
|
||||
chunkMaxTokens :: Int
|
||||
chunkMaxTokens = 6000
|
||||
|
||||
tokensApprox :: Text -> Int
|
||||
tokensApprox t = T.length t `div` 3
|
||||
|
||||
stripTags :: Text -> Text
|
||||
stripTags t =
|
||||
case T.breakOn "<" t of
|
||||
(before, rest)
|
||||
| T.null rest -> before
|
||||
| otherwise ->
|
||||
case T.breakOn ">" (T.drop 1 rest) of
|
||||
(_, after) -> stripTags before <> stripTags (T.drop 1 after)
|
||||
|
||||
normalize :: Text -> Text
|
||||
normalize = T.unwords . T.words
|
||||
|
||||
blockEndTags :: [Text]
|
||||
blockEndTags =
|
||||
["</p>", "</div>", "</blockquote>", "</section>", "</li>"]
|
||||
<> map (\n -> "</h" <> T.pack (show n) <> ">") [1 .. 6 :: Int]
|
||||
|
||||
sentinel :: Text
|
||||
sentinel = "\x00BLOCK\x00"
|
||||
|
||||
replaceBlockEnds :: Text -> Text
|
||||
replaceBlockEnds html = foldr (\tag -> T.replace tag sentinel) html blockEndTags
|
||||
|
||||
extractBlocks :: Text -> [Text]
|
||||
extractBlocks html =
|
||||
map normalize
|
||||
. map stripTags
|
||||
. filter (not . T.null)
|
||||
$ T.splitOn sentinel (replaceBlockEnds html)
|
||||
|
||||
splitSentences :: Text -> [Text]
|
||||
splitSentences t =
|
||||
filter (not . T.null)
|
||||
. map T.strip
|
||||
$ T.splitOn sent
|
||||
$ T.replace ". " (". " <> sent)
|
||||
$ T.replace "? " ("? " <> sent)
|
||||
$ T.replace "! " ("! " <> sent)
|
||||
$ t
|
||||
where
|
||||
sent = "\x00SENT\x00"
|
||||
|
||||
hardWrap :: Text -> [Text]
|
||||
hardWrap t
|
||||
| T.null t = []
|
||||
| otherwise = let (a, b) = T.splitAt (chunkMaxTokens * 3) t in a : hardWrap b
|
||||
|
||||
coalesceBlocks :: Int -> [Text] -> [Text]
|
||||
coalesceBlocks titleSize blocks = reverse (go [] 0 blocks)
|
||||
where
|
||||
chunkMaxChars = (chunkMaxTokens * 3) - titleSize
|
||||
|
||||
go :: [Text] -> Int -> [Text] -> [Text]
|
||||
go acc _ [] = [T.concat (reverse acc) | not (null acc)]
|
||||
go acc len (x : xs)
|
||||
| T.length x > chunkMaxChars && null acc =
|
||||
let sentences = splitSentences x
|
||||
splitChunks =
|
||||
if sentences == [x]
|
||||
then hardWrap x
|
||||
else sentences
|
||||
in go acc len (splitChunks ++ xs)
|
||||
| not (null acc) && len + T.length x > chunkMaxChars =
|
||||
T.concat (reverse acc) : go [] 0 (x : xs)
|
||||
| otherwise =
|
||||
go (x : acc) (len + T.length x) xs
|
||||
|
||||
chunkArticle :: Text -> Text -> [Text]
|
||||
chunkArticle title_ html =
|
||||
map (prefixTitle title_) (coalesceBlocks titleSize (extractBlocks html))
|
||||
where
|
||||
prefixTitle t chunk = "Title: " <> t <> "\n\n" <> chunk
|
||||
titleSize = T.length ("Title: " <> title_)
|
||||
|
||||
processEntry ::
|
||||
( MonadIO m,
|
||||
HasType Session r,
|
||||
HasType Creds r,
|
||||
MonadReader r m,
|
||||
HasEmbeddingMetrics r HTTPMetric,
|
||||
HasKatipTracer r KatipTracer,
|
||||
K.KatipContext m,
|
||||
MonadCatch m
|
||||
) =>
|
||||
FilePath ->
|
||||
Entry ->
|
||||
m ()
|
||||
processEntry dbPath e = do
|
||||
conn <- liftIO $ open dbPath
|
||||
existing <- liftIO $ (SQL.query conn "SELECT 1 FROM embeddings WHERE entry_id = ?" (Only (e ^. identifier)) :: IO [Only Int])
|
||||
if null existing
|
||||
then do
|
||||
let chunks = chunkArticle (e ^. title) (e ^. content)
|
||||
if null chunks
|
||||
then do
|
||||
K.logFM K.WarningS $ K.ls $ "Skipping (no extractable content): " <> e ^. title
|
||||
liftIO $ close conn
|
||||
else do
|
||||
K.logFM K.InfoS $ K.ls $ "Indexing: " <> e ^. title <> " (" <> T.pack (show (length chunks)) <> " chunks)"
|
||||
mapM_ (processChunk conn (e ^. identifier) (e ^. url) (e ^. title)) (zip [0 :: Int ..] chunks)
|
||||
liftIO $ close conn
|
||||
else do
|
||||
K.logFM K.DebugS $ K.ls $ "Skipping (already indexed): " <> e ^. title
|
||||
liftIO $ close conn
|
||||
|
||||
|
||||
newtype Embedding = Embedding (UV.Vector Float)
|
||||
deriving (Show, Eq)
|
||||
|
||||
|
||||
instance ToField Embedding where
|
||||
toField (Embedding vec) = toField $ Cereal.encode $ UV.toList vec
|
||||
|
||||
instance FromField Embedding where
|
||||
fromField f = do
|
||||
case fieldData f of
|
||||
SQLBlob bs -> either (returnError ConversionFailed f) (pure . Embedding . UV.fromList) (Cereal.decode bs)
|
||||
_ -> returnError ConversionFailed f "Failed to convert to embedding vector"
|
||||
|
||||
data EmbeddingRow = EmbeddingRow
|
||||
{ entryId :: Integer
|
||||
, chunkIdx :: Int
|
||||
, entryUrl :: Text
|
||||
, entryTitle :: Text
|
||||
, embedding :: Embedding
|
||||
}
|
||||
deriving (Generic)
|
||||
|
||||
instance SQL.ToRow EmbeddingRow
|
||||
|
||||
instance SQL.FromRow EmbeddingRow
|
||||
|
||||
processChunk ::
|
||||
( MonadIO m,
|
||||
HasType Session r,
|
||||
HasType Creds r,
|
||||
MonadReader r m,
|
||||
HasEmbeddingMetrics r HTTPMetric,
|
||||
HasKatipTracer r KatipTracer,
|
||||
K.KatipContext m,
|
||||
MonadCatch m
|
||||
) =>
|
||||
SQL.Connection ->
|
||||
Integer ->
|
||||
Text ->
|
||||
Text ->
|
||||
(Int, Text) ->
|
||||
m ()
|
||||
processChunk conn eid url_ title_ (idx, chunk) = do
|
||||
mVec <- generateEmbedding chunk
|
||||
case mVec of
|
||||
Just vec -> do
|
||||
let embeddingRow = EmbeddingRow eid idx url_ title_ (Embedding vec)
|
||||
liftIO $ SQL.execute conn
|
||||
"INSERT OR REPLACE INTO embeddings (entry_id, chunk_idx, url, title, embedding) VALUES (?, ?, ?, ?, ?)"
|
||||
embeddingRow
|
||||
Nothing ->
|
||||
K.logFM K.WarningS $ K.ls $ "Failed to generate embedding for chunk " <> T.pack (show idx) <> " of: " <> title_
|
||||
|
||||
|
||||
search ::
|
||||
( MonadIO m,
|
||||
HasType Session r,
|
||||
HasType Creds r,
|
||||
MonadReader r m,
|
||||
HasEmbeddingMetrics r HTTPMetric,
|
||||
HasKatipTracer r KatipTracer,
|
||||
MonadCatch m,
|
||||
K.KatipContext m,
|
||||
MonadMask m
|
||||
) =>
|
||||
FilePath ->
|
||||
Text ->
|
||||
Int ->
|
||||
m [(Double, Integer, Text, Text)]
|
||||
search dbPath q limit_ = do
|
||||
queryVec <- generateEmbedding q
|
||||
case queryVec of
|
||||
Nothing -> pure []
|
||||
Just qv -> do
|
||||
conn <- liftIO $ open dbPath
|
||||
results <- Pipes.Safe.runSafeT $ Pipes.runEffect (Pipes.Prelude.fold bestChunk Map.empty topResults $ query_ conn "SELECT entry_id, chunk_idx, url, title, embedding FROM embeddings" >-> Pipes.Prelude.mapMaybe (scoreRow qv))
|
||||
liftIO $ close conn
|
||||
pure results
|
||||
where
|
||||
bestChunk m (s, eid, url_, title_) =
|
||||
Map.insertWith
|
||||
(\(s1, u, t) (s2, _, _) -> (max s1 s2, u, t))
|
||||
eid
|
||||
(s, url_, title_)
|
||||
m
|
||||
sortByScore = sortOn (Down . (\(_, (s, _, _)) -> s)) . Map.toList
|
||||
topResults = map (\(eid, (s, url_, title_)) -> (realToFrac s, eid, url_, title_)) . take limit_ . sortByScore
|
||||
|
||||
|
||||
query_ :: (MonadSafe m, SQL.FromRow a) => SQL.Connection -> SQL.Query -> Pipes.Producer' a m ()
|
||||
query_ conn q = Pipes.Safe.bracket (liftIO openStmt) (liftIO . SQL.closeStatement) eachRow
|
||||
where
|
||||
openStmt = do
|
||||
SQL.openStatement conn q
|
||||
eachRow stmt = do
|
||||
row <- liftIO $ SQL.nextRow stmt
|
||||
maybe (pure ()) (\r -> Pipes.yield r >> eachRow stmt) row
|
||||
|
||||
|
||||
scoreRow ::
|
||||
UV.Vector Float ->
|
||||
EmbeddingRow ->
|
||||
Maybe (Float, Integer, Text, Text)
|
||||
scoreRow qv EmbeddingRow{embedding = Embedding uv, ..} =
|
||||
let sim = cosineSimilarity qv uv
|
||||
in Just (sim, entryId, entryUrl, entryTitle)
|
||||
@@ -12,11 +12,11 @@ import Control.Exception (bracket)
|
||||
import Control.Lens (over, view)
|
||||
import Control.Lens.TH (makeFields)
|
||||
import Control.Monad.Base (MonadBase)
|
||||
import Control.Monad.Catch (MonadCatch, MonadThrow)
|
||||
import Control.Monad.Catch (MonadCatch, MonadThrow, MonadMask)
|
||||
import Control.Monad.Reader (MonadIO, MonadReader, ReaderT, local, runReaderT)
|
||||
import Control.Monad.Trans.Control (MonadBaseControl)
|
||||
import Creds
|
||||
import Data.HashMap.Strict qualified as HM
|
||||
|
||||
import GHC.Generics (Generic)
|
||||
import HTTP (HTTPMetric (HTTPMetric))
|
||||
import Katip qualified as K
|
||||
@@ -37,6 +37,7 @@ data S = S
|
||||
_sCreds :: Creds,
|
||||
_sWallabagMetrics :: HTTPMetric,
|
||||
_sOpenrouterMetrics :: HTTPMetric,
|
||||
_sEmbeddingMetrics :: HTTPMetric,
|
||||
_sKatipTracer :: KatipTracer,
|
||||
_sMetricsStore :: EKG.Store,
|
||||
_sSession :: Wreq.Session
|
||||
@@ -56,7 +57,8 @@ newtype M a = M (ReaderT S IO a)
|
||||
MonadThrow,
|
||||
MonadCatch,
|
||||
MonadBaseControl IO,
|
||||
MonadBase IO
|
||||
MonadBase IO,
|
||||
MonadMask
|
||||
)
|
||||
|
||||
runM :: S -> M a -> IO a
|
||||
@@ -92,6 +94,11 @@ initialS credsPath severity f = do
|
||||
<$> EKG.createDistribution "openrouter.requests" store
|
||||
<*> EKG.createCounter "openrouter.retries" store
|
||||
<*> EKG.createCounter "openrouter.failures" store
|
||||
emMetrics <-
|
||||
HTTPMetric
|
||||
<$> EKG.createDistribution "embeddings.requests" store
|
||||
<*> EKG.createCounter "embeddings.retries" store
|
||||
<*> EKG.createCounter "embeddings.failures" store
|
||||
c <- maybe getDefaultCreds readCreds credsPath
|
||||
handleScribe <- K.mkHandleScribe K.ColorIfTerminal stdout (K.permitItem severity) K.V2
|
||||
let makeLogEnv = K.registerScribe "stdout" handleScribe K.defaultScribeSettings =<< K.initLogEnv "wallabag" "production"
|
||||
@@ -99,7 +106,5 @@ initialS credsPath severity f = do
|
||||
let kTrace = mkKatipTrace le ns
|
||||
ns = "main"
|
||||
cache <- newTVarIO Unauthenticated
|
||||
f (S cache ns le mempty c wbMetrics orMetrics kTrace store wsession)
|
||||
-- TODO: This is just for debug purposes
|
||||
EKG.sampleAll store >>= mapM_ print . HM.toList
|
||||
f (S cache ns le mempty c wbMetrics orMetrics emMetrics kTrace store wsession)
|
||||
pure res
|
||||
+55
-4
@@ -7,14 +7,16 @@ module MyLib (defaultMain, test) where
|
||||
import AI
|
||||
import Control.Exception.Annotated.UnliftIO (Exception, SomeException, catch, throw)
|
||||
import Control.Lens (over, re, strict, view, (&), (.~), (^.))
|
||||
import Control.Monad (forever)
|
||||
import Control.Monad (forM_, forever)
|
||||
import Data.Aeson (decode, encode)
|
||||
import Data.Either (fromRight)
|
||||
import Data.Maybe (fromMaybe)
|
||||
import Data.Set qualified as S
|
||||
import Data.Text (Text)
|
||||
import Data.Text qualified as T
|
||||
import Data.Text.IO qualified as T
|
||||
import Data.Text.Strict.Lens (utf8)
|
||||
import Embeddings (indexEntries, search)
|
||||
import Katip qualified as K
|
||||
import M
|
||||
import Network.Wai.Handler.Warp (defaultSettings, runSettings, setPort)
|
||||
@@ -22,7 +24,11 @@ import Options.Generic (Generic, ParseRecord (..), getRecord, unDefValue, unHelp
|
||||
import Pipes (Pipe, for, lift, runEffect, (>->))
|
||||
import Pipes.Prelude qualified as P
|
||||
import Server (Health (..), app)
|
||||
import System.Directory (createDirectoryIfMissing, getHomeDirectory)
|
||||
import System.Environment (lookupEnv)
|
||||
import System.FilePath (takeDirectory)
|
||||
import Text.Pandoc (PandocError, def, readHtml, runPure, writeMarkdown)
|
||||
import Text.Printf (printf)
|
||||
import UnliftIO (MonadUnliftIO, liftIO)
|
||||
import UnliftIO.Async (mapConcurrently_)
|
||||
import UnliftIO.Concurrent (threadDelay)
|
||||
@@ -30,14 +36,28 @@ import UnliftIO.Exception (displayException, isAsyncException)
|
||||
import UnliftIO.IORef (IORef, newIORef, writeIORef)
|
||||
import Wallabag
|
||||
import Pipes.Concurrent (performGC)
|
||||
import qualified Version
|
||||
|
||||
data Options
|
||||
= Quick {verbose :: Bool}
|
||||
| Full {verbose :: Bool}
|
||||
| Version
|
||||
| Index
|
||||
{ verbose :: Bool,
|
||||
rebuild :: Bool <?> "Rebuild index from scratch" <!> "False",
|
||||
embeddingsDb :: Maybe FilePath <?> "Path to embeddings database"
|
||||
}
|
||||
| Search
|
||||
{ verbose :: Bool,
|
||||
query :: Text <?> "Search query",
|
||||
limit :: Int <?> "Maximum results" <!> "10",
|
||||
embeddingsDb :: Maybe FilePath <?> "Path to embeddings database"
|
||||
}
|
||||
| Daemon
|
||||
{ verbose :: Bool,
|
||||
credentials :: FilePath <?> "Path to the credentials.json",
|
||||
port :: Int <?> "Port to run the diagnostics" <!> "8094"
|
||||
port :: Int <?> "Port to run the diagnostics" <!> "8094",
|
||||
embeddingsDb :: Maybe FilePath <?> "Path to embeddings database"
|
||||
}
|
||||
deriving (Generic, Show)
|
||||
|
||||
@@ -158,16 +178,47 @@ hardLabels =
|
||||
"Gaming",
|
||||
"Plants",
|
||||
"Gadgets",
|
||||
"AI"
|
||||
"AI",
|
||||
"Emergency",
|
||||
"War",
|
||||
"Open Source"
|
||||
]
|
||||
|
||||
resolveEmbeddingsDb :: Maybe FilePath -> IO FilePath
|
||||
resolveEmbeddingsDb (Just fp) = pure fp
|
||||
resolveEmbeddingsDb Nothing = do
|
||||
env <- lookupEnv "WALLABAG_EMBEDDINGS_DB"
|
||||
case env of
|
||||
Just fp -> pure fp
|
||||
Nothing -> do
|
||||
home <- getHomeDirectory
|
||||
let dir = home <> "/.wallabag-tagger"
|
||||
createDirectoryIfMissing True dir
|
||||
pure $ dir <> "/embeddings.db"
|
||||
|
||||
runWithDb :: Maybe FilePath -> K.Severity -> (FilePath -> M ()) -> IO ()
|
||||
runWithDb mbDb sev f = initialS Nothing sev $ \s -> runM s $ do
|
||||
dbPath <- liftIO $ resolveEmbeddingsDb mbDb
|
||||
liftIO $ createDirectoryIfMissing True (takeDirectory dbPath)
|
||||
f dbPath
|
||||
|
||||
defaultMain :: IO ()
|
||||
defaultMain = do
|
||||
opts <- getRecord "Wallabag tagger"
|
||||
case opts of
|
||||
Quick {verbose} -> run (severity verbose) Unarchived
|
||||
Full {verbose} -> run (severity verbose) Everything
|
||||
Daemon {verbose, credentials, port} ->
|
||||
Version -> print Version.compileTimeVersion
|
||||
Index {verbose, rebuild, embeddingsDb} ->
|
||||
runWithDb (unHelpful embeddingsDb) (severity verbose) $ \db ->
|
||||
indexEntries db (unHelpful $ unDefValue rebuild)
|
||||
Search {verbose, query, limit, embeddingsDb} ->
|
||||
runWithDb (unHelpful embeddingsDb) (severity verbose) $ \db -> do
|
||||
entries <- search db (unHelpful query) (unHelpful $ unDefValue limit)
|
||||
liftIO $ forM_ entries $ \(score, _eid, u, t) -> do
|
||||
T.putStrLn $ T.pack $ printf "%.4f %s" (score :: Double) (T.unpack t)
|
||||
T.putStrLn $ T.pack $ printf " %s" (T.unpack u)
|
||||
Daemon {verbose, credentials, port, embeddingsDb} ->
|
||||
runDaemon (unHelpful $ unDefValue port) (unHelpful credentials) (severity verbose)
|
||||
where
|
||||
severity True = K.DebugS
|
||||
|
||||
+8
-2
@@ -21,6 +21,7 @@ import System.Metrics.Distribution qualified as D
|
||||
import UnliftIO (liftIO)
|
||||
import UnliftIO.IORef (readIORef)
|
||||
import Data.Coerce (coerce)
|
||||
import Version (Version, compileTimeVersion)
|
||||
|
||||
data Health
|
||||
= OK
|
||||
@@ -57,7 +58,8 @@ newtype Metrics = Metrics (HashMap Text SampleValue)
|
||||
|
||||
data WallabagRoutes mode = WallabagRoutes
|
||||
{ _getHealth :: mode :- "health" :> Get '[PlainText, JSON] Health,
|
||||
_getMetrics :: mode :- "metrics" :> Get '[JSON] Metrics
|
||||
_getMetrics :: mode :- "metrics" :> Get '[JSON] Metrics,
|
||||
_getVersion :: mode :- "version" :> Get '[JSON] Version
|
||||
}
|
||||
deriving (Generic)
|
||||
|
||||
@@ -67,12 +69,16 @@ wallabagServer :: IORef Health -> WallabagRoutes (AsServerT M)
|
||||
wallabagServer health =
|
||||
WallabagRoutes
|
||||
{ _getHealth = getHealth health,
|
||||
_getMetrics = getMetrics
|
||||
_getMetrics = getMetrics,
|
||||
_getVersion = getVersion
|
||||
}
|
||||
|
||||
getHealth :: IORef Health -> M Health
|
||||
getHealth = readIORef
|
||||
|
||||
getVersion :: M Version
|
||||
getVersion = pure compileTimeVersion
|
||||
|
||||
getMetrics :: M Metrics
|
||||
getMetrics = do
|
||||
sample <- view metricsStore >>= liftIO . EKG.sampleAll
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
{-# LANGUAGE TemplateHaskell #-}
|
||||
{-# OPTIONS_GHC -Wno-unrecognised-pragmas #-}
|
||||
{-# HLINT ignore "Unused LANGUAGE pragma" #-}
|
||||
module Version where
|
||||
|
||||
import qualified Data.Aeson as A
|
||||
import GHC.Generics (Generic)
|
||||
import System.Environment (lookupEnv)
|
||||
import Language.Haskell.TH (runIO, litE, stringL)
|
||||
import Data.Maybe (fromMaybe)
|
||||
|
||||
|
||||
-- Populate the version via the 'githash' package
|
||||
newtype Version = Version
|
||||
{ versionCommit :: String }
|
||||
deriving (Show, Generic)
|
||||
|
||||
instance A.ToJSON Version
|
||||
|
||||
versionFromEnv :: IO Version
|
||||
versionFromEnv = maybe (Version "unknown") Version <$> lookupEnv "COMMIT"
|
||||
|
||||
compileTimeVersion :: Version
|
||||
compileTimeVersion = Version $(runIO (fromMaybe "unknown" <$> lookupEnv "COMMIT") >>= litE . stringL)
|
||||
+8
-6
@@ -46,7 +46,8 @@ data Entry = Entry
|
||||
{ _entryIdentifier :: Integer,
|
||||
_entryContent :: Text,
|
||||
_entryTags :: [Text],
|
||||
_entryTitle :: Text
|
||||
_entryTitle :: Text,
|
||||
_entryUrl :: Text
|
||||
}
|
||||
deriving (Show)
|
||||
|
||||
@@ -151,13 +152,13 @@ sourceEntries s = P.unfoldr (toEither <=< go) (Just startUrl) >-> P.mapFoldable
|
||||
Everything -> "https://app.wallabag.it/api/entries.json"
|
||||
go :: Maybe String -> m (Maybe ([Value], Maybe String))
|
||||
go Nothing = pure Nothing
|
||||
go (Just url) = checkpoint (Annotation url) $ do
|
||||
go (Just u) = checkpoint (Annotation u) $ do
|
||||
metrics <- mkTracerMetrics
|
||||
t <- maybe (throw MissingTokenException) pure =<< getToken
|
||||
let opts = defaults & header "Authorization" .~ ["Bearer " <> t ^. re utf8]
|
||||
K.logFM K.DebugS $ K.ls ("Fetching entries '" <> url <> "'")
|
||||
K.logFM K.DebugS $ K.ls ("Fetching entries '" <> u <> "'")
|
||||
session <- view $ typed @Session
|
||||
r <- liftIO $ getWith metrics session SafeTransient opts url
|
||||
r <- liftIO $ getWith metrics session SafeTransient opts u
|
||||
let items = r ^.. responseBody . key "_embedded" . key "items" . _Array . traversed
|
||||
let next = r ^? responseBody . key "_links" . key "next" . key "href" . _String . unpacked
|
||||
pure $ Just (items, next)
|
||||
@@ -168,6 +169,7 @@ sourceEntries s = P.unfoldr (toEither <=< go) (Just startUrl) >-> P.mapFoldable
|
||||
<*> v ^? key "content" . _String
|
||||
<*> Just (v ^.. key "tags" . _Array . traversed . key "label" . _String)
|
||||
<*> v ^? key "title" . _String
|
||||
<*> v ^? key "url" . _String
|
||||
|
||||
mkTracerMetrics ::
|
||||
( MonadReader r m,
|
||||
@@ -200,8 +202,8 @@ applyTags e = checkpoint (Annotation $ e ^. identifier) $ do
|
||||
t <- maybe (throw MissingTokenException) pure =<< getToken
|
||||
let opts = defaults & header "Authorization" .~ ["Bearer " <> t ^. re utf8]
|
||||
K.logFM K.InfoS (K.ls ("Applying tags for '" <> e ^. title <> "': " <> T.pack (show (e ^. tags))))
|
||||
let url = "https://app.wallabag.it/api/entries/" <> show (e ^. identifier) <> "/tags"
|
||||
let u = "https://app.wallabag.it/api/entries/" <> show (e ^. identifier) <> "/tags"
|
||||
metrics <- mkTracerMetrics
|
||||
session <- view $ typed @Session
|
||||
_ <- liftIO $ postWith metrics session SafeTransient opts url ["tags" := T.intercalate "," (e ^. tags)]
|
||||
_ <- liftIO $ postWith metrics session SafeTransient opts u ["tags" := T.intercalate "," (e ^. tags)]
|
||||
pure ()
|
||||
+39
-1
@@ -1,4 +1,42 @@
|
||||
{-# LANGUAGE OverloadedStrings #-}
|
||||
module Main (main) where
|
||||
|
||||
import Data.Foldable (forM_)
|
||||
import Embeddings (chunkArticle, tokensApprox, chunkMaxTokens, Embedding(..))
|
||||
import Test.Hspec
|
||||
import qualified Data.Text as T
|
||||
|
||||
main :: IO ()
|
||||
main = putStrLn "Test suite not yet implemented."
|
||||
main = hspec $ do
|
||||
describe "Chunking" $ do
|
||||
it "Splits a long article into multiple chunks" testChunking
|
||||
it "Retains a short article as one chunk" testShortArticleOneChunk
|
||||
it "Prefixes each chunk with the title" testTitlePrefix
|
||||
it "Chunks to small enough items" testTokenLimit
|
||||
|
||||
|
||||
testChunking :: IO ()
|
||||
testChunking = do
|
||||
let longHtml = T.replicate 1000 "<p>This is a paragraph with some text content that should be long enough to create multiple chunks when repeated many times.</p>"
|
||||
chunks = chunkArticle "Test Article" longHtml
|
||||
n = length chunks
|
||||
n `shouldBe` 7
|
||||
|
||||
testShortArticleOneChunk :: IO ()
|
||||
testShortArticleOneChunk = do
|
||||
let shortHtml = "<p>A short paragraph.</p>"
|
||||
chunks = chunkArticle "Short" shortHtml
|
||||
length chunks `shouldBe` 1
|
||||
|
||||
testTitlePrefix :: IO ()
|
||||
testTitlePrefix = do
|
||||
let chunks = chunkArticle "My Title" "<p>Some content.</p>"
|
||||
forM_ chunks $ \chunk ->
|
||||
chunk `shouldSatisfy` ("Title: My Title\n\n" `T.isPrefixOf`)
|
||||
|
||||
testTokenLimit :: IO ()
|
||||
testTokenLimit = do
|
||||
let longHtml = T.replicate 1000 "<p>This is a paragraph with some text content that should be long enough to create multiple chunks when repeated many times.</p>"
|
||||
chunks = chunkArticle "Test Article" longHtml
|
||||
counts = map tokensApprox chunks
|
||||
counts `shouldSatisfy` all (<= chunkMaxTokens)
|
||||
+17
-4
@@ -19,6 +19,7 @@ common warnings
|
||||
library
|
||||
import: warnings
|
||||
exposed-modules: AI
|
||||
Embeddings
|
||||
Creds
|
||||
M
|
||||
Wallabag
|
||||
@@ -27,9 +28,12 @@ library
|
||||
HTTP
|
||||
Server
|
||||
Tracer
|
||||
Version
|
||||
-- other-modules:
|
||||
-- other-extensions:
|
||||
build-depends: base ^>=4.18.2.1
|
||||
build-depends: base ^>=4.20
|
||||
, sqlite-simple
|
||||
, vector
|
||||
, wreq
|
||||
, lens
|
||||
, katip
|
||||
@@ -38,6 +42,7 @@ library
|
||||
, ekg-core
|
||||
, retry
|
||||
, pipes
|
||||
, pipes-safe
|
||||
, pipes-concurrency
|
||||
, streamly
|
||||
, streamly-core
|
||||
@@ -64,6 +69,8 @@ library
|
||||
, generic-lens
|
||||
, optparse-generic
|
||||
, containers
|
||||
, template-haskell
|
||||
, cereal
|
||||
hs-source-dirs: src
|
||||
default-language: GHC2021
|
||||
|
||||
@@ -73,7 +80,7 @@ executable wallabag-tagger
|
||||
-- other-modules:
|
||||
-- other-extensions:
|
||||
build-depends:
|
||||
base ^>=4.18.2.1,
|
||||
base ^>=4.20,
|
||||
wallabag-tagger
|
||||
|
||||
hs-source-dirs: app
|
||||
@@ -89,5 +96,11 @@ test-suite wallabag-tagger-test
|
||||
hs-source-dirs: test
|
||||
main-is: Main.hs
|
||||
build-depends:
|
||||
base ^>=4.18.2.1,
|
||||
wallabag-tagger
|
||||
base ^>=4.20,
|
||||
wallabag-tagger,
|
||||
text,
|
||||
hspec,
|
||||
hspec-hedgehog,
|
||||
hedgehog,
|
||||
vector,
|
||||
sqlite-simple
|
||||
Reference in new issue
Block a user