Compare commits

..
13 Commits
Author SHA1 Message Date
Mats Rauhala db94983590 Update nixpkgs 2026-11-11 12:35:16 +02:00
Mats Rauhala 8c71d04053 Replace gpt4o-mini with deepseek 2026-11-11 12:32:54 +02:00
Mats Rauhala cb1972225f More conservative chunking 2026-11-11 11:50:31 +02:00
Mats Rauhala f4a79b4251 Format search results 2026-06-11 20:51:39 +03:00
Mats Rauhala f5197caa77 Chunking 2026-06-11 15:13:29 +03:00
Mats Rauhala f54aaa6e95 Embeddings CLI 2026-06-11 10:45:53 +03:00
MasseR 6ead08af22 Update dependencies 2026-06-10 14:44:43 +03:00
MasseR a4b5b3c8a0 Wrong api path 2026-06-03 15:36:48 +03:00
MasseR 66018dca53 Merge branch 'githash' 2026-06-02 10:19:55 +03:00
MasseR 51a1e49fb7 Version from compile time nix 2026-06-02 10:18:55 +03:00
MasseR c099a4f105 Merge branch 'more-tags' 2026-06-01 09:47:45 +03:00
MasseR 813ec1fda5 New tags 2026-06-01 09:46:51 +03:00
MasseR e87aae3144 Version endpoint 2026-06-01 09:32:01 +03:00
12 changed files with 554 additions and 46 deletions

No files matched your search

+18 -14
View File
@@ -1,10 +1,11 @@
{ mkDerivation, aeson, annotated-exception, base, bytestring
, containers, directory, ekg-core, exceptions, filepath
, generic-lens, http-client, http-types, katip, lens, lens-aeson
, lib, monad-control, mtl, optparse-generic, pandoc, pipes
, pipes-concurrency, retry, servant, servant-server, stm, streamly
, streamly-core, text, time, transformers-base, unliftio
, unordered-containers, wai, warp, wreq
, cereal, containers, directory, ekg-core, exceptions, filepath
, generic-lens, hedgehog, hspec, hspec-hedgehog, http-client
, http-types, katip, lens, lens-aeson, lib, monad-control, mtl
, optparse-generic, pandoc, pipes, pipes-concurrency, pipes-safe
, retry, servant, servant-server, sqlite-simple, stm, streamly
, streamly-core, template-haskell, text, time, transformers-base
, unliftio, unordered-containers, vector, wai, warp, wreq
}:
mkDerivation {
pname = "wallabag-tagger";
@@ -13,15 +14,18 @@ mkDerivation {
isLibrary = true;
isExecutable = true;
libraryHaskellDepends = [
aeson annotated-exception base bytestring containers directory
ekg-core exceptions filepath generic-lens http-client http-types
katip lens lens-aeson monad-control mtl optparse-generic pandoc
pipes pipes-concurrency retry servant servant-server stm streamly
streamly-core text time transformers-base unliftio
unordered-containers wai warp wreq
aeson annotated-exception base bytestring cereal containers
directory ekg-core exceptions filepath generic-lens http-client
http-types katip lens lens-aeson monad-control mtl optparse-generic
pandoc pipes pipes-concurrency pipes-safe retry servant
servant-server sqlite-simple stm streamly streamly-core
template-haskell text time transformers-base unliftio
unordered-containers vector wai warp wreq
];
executableHaskellDepends = [ base ];
testHaskellDepends = [ base ];
license = lib.licenses.bsd3;
testHaskellDepends = [
base hedgehog hspec hspec-hedgehog sqlite-simple text vector
];
license = lib.meta.getLicenseFromSpdxId "BSD-3-Clause";
mainProgram = "wallabag-tagger";
}
Generated
+8 -7
View File
@@ -5,11 +5,11 @@
"systems": "systems"
},
"locked": {
"lastModified": 1731533236,
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
"lastModified": 1791914995,
"narHash": "sha256-Zf/xfQEAasnwnmEfHqW/4hjDE/s44yn2hD4RuVIZRjA=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
"rev": "b7a79d3fea1e214f6d668d521cecec0c01e26544",
"type": "github"
},
"original": {
@@ -20,10 +20,11 @@
},
"nixpkgs": {
"locked": {
"lastModified": 0,
"narHash": "sha256-hw63HnwnqU3ZQfsMclLhMvOezpM7RSB0dMAtD5/sOiw=",
"path": "/nix/store/4m21g6b3bq78vn5skaxjmqwg7fnir87j-source",
"type": "path"
"lastModified": 1793977579,
"narHash": "sha256-4/Yzr1fWTh/zBAR1tR84ZQ1Zhmor8AXwgD4AS7sPn60=",
"rev": "4fe25e12eb9f4c744966dd3de5a05246cf098601",
"type": "tarball",
"url": "https://releases.nixos.org/nixpkgs/nixpkgs-26.11pre1109744.4fe25e12eb9f/nixexprs.tar.zst"
},
"original": {
"id": "nixpkgs",
+5 -1
View File
@@ -12,7 +12,11 @@
let pkgs = nixpkgs.legacyPackages.${system};
hp = nixpkgs.legacyPackages.${system}.haskellPackages.override (old: {
overrides = pkgs.lib.composeExtensions (old.overrides or (_: _: {})) (f: p: {
wallabag-tagger = f.callPackage ./. {};
wallabag-tagger = pkgs.haskell.lib.overrideCabal (f.callPackage ./. {}) (_: {
preBuild = ''
export COMMIT="${self.rev or "unknown"}"
'';
});
});
});
in rec {
+1 -1
View File
@@ -47,7 +47,7 @@ completions c prompt = checkpointCallStack $ do
let opts = defaults & header "Authorization" .~ ["Bearer " <> c ^. openrouterKey . re utf8]
let j =
object
[ "model" .= ("openai/gpt-4o-mini" :: Text),
[ "model" .= ("~deepseek/deepseek-flash-latest" :: Text),
"messages"
.= [object ["role" .= ("user" :: Text), "content" .= prompt]]
]
+360
View File
@@ -0,0 +1,360 @@
{-# LANGUAGE LambdaCase #-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE StrictData #-}
{-# OPTIONS_GHC -Wall #-}
{-# LANGUAGE RecordWildCards #-}
module Embeddings
( generateEmbedding,
indexEntries,
search,
chunkArticle,
tokensApprox,
chunkMaxTokens,
Embedding(..), -- for testing
) where
import Control.Exception.Annotated (checkpointCallStack)
import Control.Lens (re, traversed, view, (&), (.~), (^.), (^..), (^?))
import Control.Monad (when)
import Control.Monad.Catch (MonadCatch, MonadThrow, MonadMask)
import Control.Monad.Reader (MonadReader, liftIO)
import Creds (Creds, openrouterKey)
import Data.Aeson (object, (.=))
import Data.Aeson.Lens (key, _Array, _Double)
import Data.Functor.Contravariant ((>$<))
import Data.Generics.Product (HasType, typed)
import Data.List (sortOn)
import Data.Map.Strict qualified as Map
import Data.Ord (Down (..))
import Data.Text (Text)
import Data.Text qualified as T
import Data.Text.Strict.Lens (utf8)
import Data.Vector.Unboxed qualified as UV
import Database.SQLite.Simple (Only (..), open, close, SQLData (SQLBlob), ResultError (ConversionFailed))
import Database.SQLite.Simple qualified as SQL
import GHC.Stack (HasCallStack)
import HTTP (HTTPMetric, Retry (SafeTransient), Trace, logFormatter, postWith, tracerMetrics)
import Katip qualified as K
import M (HasEmbeddingMetrics, HasKatipTracer, HasWallabagMetrics, embeddingMetrics, katipTracer, KatipTracer)
import Network.Wreq (defaults, header, responseBody)
import Network.Wreq.Session (Session)
import Pipes (lift, (>->))
import Pipes qualified
import Tracer (Tracer (..))
import UnliftIO (MonadIO, MonadUnliftIO)
import Wallabag (Entry, Scan (Everything), content, identifier, title, url)
import qualified Wallabag
import Wallabag.Auth (Auth)
import GHC.Generics (Generic)
import Database.SQLite.Simple.ToField (ToField(..))
import Database.SQLite.Simple.FromField (FromField(..), fieldData, returnError)
import qualified Data.Serialize as Cereal
import qualified Pipes.Safe
import Pipes.Safe (MonadSafe)
import qualified Pipes.Prelude
embeddingUrl :: String
embeddingUrl = "https://openrouter.ai/api/v1/embeddings"
mkTracerMetrics ::
( MonadReader r m,
HasEmbeddingMetrics r HTTPMetric,
HasKatipTracer r KatipTracer
) =>
m (Tracer Trace)
mkTracerMetrics = do
l <- view katipTracer
m <- view embeddingMetrics
pure $ (logFormatter >$< l) <> tracerMetrics m
generateEmbedding ::
( MonadIO m,
HasType Session r,
HasType Creds r,
MonadReader r m,
HasEmbeddingMetrics r HTTPMetric,
HasKatipTracer r KatipTracer,
MonadCatch m,
K.KatipContext m
) =>
Text ->
m (Maybe (UV.Vector Float))
generateEmbedding input = checkpointCallStack $ do
c <- view $ typed @Creds
let opts = defaults & header "Authorization" .~ ["Bearer " <> c ^. openrouterKey . re utf8]
let j =
object
[ "model" .= ("text-embedding-3-small" :: Text),
"input" .= input
]
tm <- mkTracerMetrics
session <- view $ typed @Session
r <- liftIO $ postWith tm session SafeTransient opts embeddingUrl j
case r ^? responseBody . key "data" . _Array . traversed . key "embedding" of
Just embeddingVal -> do
let floats = embeddingVal ^.. _Array . traversed . _Double
if null floats
then do
K.logFM K.ErrorS $ K.ls $ "Parsing gave empty embedding array from: " <> T.pack (show (r ^. responseBody))
pure Nothing
else pure $ Just $ UV.fromList (map realToFrac floats)
Nothing -> do
K.logFM K.ErrorS $ K.ls $ "Couldn't find embedding in response: " <> T.pack (show (r ^. responseBody))
pure Nothing
cosineSimilarity :: UV.Vector Float -> UV.Vector Float -> Float
cosineSimilarity a b = dot / (normA * normB)
where
dot = UV.sum $ UV.zipWith (*) a b
normA = sqrt $ UV.sum $ UV.map (^ (2 :: Int)) a
normB = sqrt $ UV.sum $ UV.map (^ (2 :: Int)) b
indexEntries ::
( MonadIO m,
HasType Session r,
HasType Auth r,
HasType Creds r,
MonadReader r m,
K.KatipContext m,
HasWallabagMetrics r HTTPMetric,
HasEmbeddingMetrics r HTTPMetric,
HasKatipTracer r KatipTracer,
MonadThrow m,
MonadCatch m,
HasCallStack,
MonadUnliftIO m
) =>
FilePath ->
Bool ->
m ()
indexEntries dbPath rebuild = do
liftIO $ do
conn <- open dbPath
SQL.execute_ conn
"CREATE TABLE IF NOT EXISTS embeddings (entry_id INTEGER NOT NULL, chunk_idx INTEGER NOT NULL, url TEXT NOT NULL, title TEXT NOT NULL, embedding TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT (datetime('now')), PRIMARY KEY (entry_id, chunk_idx))"
when rebuild $ SQL.execute_ conn "DELETE FROM embeddings"
close conn
Pipes.runEffect $
Pipes.for
(Wallabag.sourceEntries Everything)
(lift . processEntry dbPath)
chunkMaxTokens :: Int
chunkMaxTokens = 6000
tokensApprox :: Text -> Int
tokensApprox t = T.length t `div` 3
stripTags :: Text -> Text
stripTags t =
case T.breakOn "<" t of
(before, rest)
| T.null rest -> before
| otherwise ->
case T.breakOn ">" (T.drop 1 rest) of
(_, after) -> stripTags before <> stripTags (T.drop 1 after)
normalize :: Text -> Text
normalize = T.unwords . T.words
blockEndTags :: [Text]
blockEndTags =
["</p>", "</div>", "</blockquote>", "</section>", "</li>"]
<> map (\n -> "</h" <> T.pack (show n) <> ">") [1 .. 6 :: Int]
sentinel :: Text
sentinel = "\x00BLOCK\x00"
replaceBlockEnds :: Text -> Text
replaceBlockEnds html = foldr (\tag -> T.replace tag sentinel) html blockEndTags
extractBlocks :: Text -> [Text]
extractBlocks html =
map normalize
. map stripTags
. filter (not . T.null)
$ T.splitOn sentinel (replaceBlockEnds html)
splitSentences :: Text -> [Text]
splitSentences t =
filter (not . T.null)
. map T.strip
$ T.splitOn sent
$ T.replace ". " (". " <> sent)
$ T.replace "? " ("? " <> sent)
$ T.replace "! " ("! " <> sent)
$ t
where
sent = "\x00SENT\x00"
hardWrap :: Text -> [Text]
hardWrap t
| T.null t = []
| otherwise = let (a, b) = T.splitAt (chunkMaxTokens * 3) t in a : hardWrap b
coalesceBlocks :: Int -> [Text] -> [Text]
coalesceBlocks titleSize blocks = reverse (go [] 0 blocks)
where
chunkMaxChars = (chunkMaxTokens * 3) - titleSize
go :: [Text] -> Int -> [Text] -> [Text]
go acc _ [] = [T.concat (reverse acc) | not (null acc)]
go acc len (x : xs)
| T.length x > chunkMaxChars && null acc =
let sentences = splitSentences x
splitChunks =
if sentences == [x]
then hardWrap x
else sentences
in go acc len (splitChunks ++ xs)
| not (null acc) && len + T.length x > chunkMaxChars =
T.concat (reverse acc) : go [] 0 (x : xs)
| otherwise =
go (x : acc) (len + T.length x) xs
chunkArticle :: Text -> Text -> [Text]
chunkArticle title_ html =
map (prefixTitle title_) (coalesceBlocks titleSize (extractBlocks html))
where
prefixTitle t chunk = "Title: " <> t <> "\n\n" <> chunk
titleSize = T.length ("Title: " <> title_)
processEntry ::
( MonadIO m,
HasType Session r,
HasType Creds r,
MonadReader r m,
HasEmbeddingMetrics r HTTPMetric,
HasKatipTracer r KatipTracer,
K.KatipContext m,
MonadCatch m
) =>
FilePath ->
Entry ->
m ()
processEntry dbPath e = do
conn <- liftIO $ open dbPath
existing <- liftIO $ (SQL.query conn "SELECT 1 FROM embeddings WHERE entry_id = ?" (Only (e ^. identifier)) :: IO [Only Int])
if null existing
then do
let chunks = chunkArticle (e ^. title) (e ^. content)
if null chunks
then do
K.logFM K.WarningS $ K.ls $ "Skipping (no extractable content): " <> e ^. title
liftIO $ close conn
else do
K.logFM K.InfoS $ K.ls $ "Indexing: " <> e ^. title <> " (" <> T.pack (show (length chunks)) <> " chunks)"
mapM_ (processChunk conn (e ^. identifier) (e ^. url) (e ^. title)) (zip [0 :: Int ..] chunks)
liftIO $ close conn
else do
K.logFM K.DebugS $ K.ls $ "Skipping (already indexed): " <> e ^. title
liftIO $ close conn
newtype Embedding = Embedding (UV.Vector Float)
deriving (Show, Eq)
instance ToField Embedding where
toField (Embedding vec) = toField $ Cereal.encode $ UV.toList vec
instance FromField Embedding where
fromField f = do
case fieldData f of
SQLBlob bs -> either (returnError ConversionFailed f) (pure . Embedding . UV.fromList) (Cereal.decode bs)
_ -> returnError ConversionFailed f "Failed to convert to embedding vector"
data EmbeddingRow = EmbeddingRow
{ entryId :: Integer
, chunkIdx :: Int
, entryUrl :: Text
, entryTitle :: Text
, embedding :: Embedding
}
deriving (Generic)
instance SQL.ToRow EmbeddingRow
instance SQL.FromRow EmbeddingRow
processChunk ::
( MonadIO m,
HasType Session r,
HasType Creds r,
MonadReader r m,
HasEmbeddingMetrics r HTTPMetric,
HasKatipTracer r KatipTracer,
K.KatipContext m,
MonadCatch m
) =>
SQL.Connection ->
Integer ->
Text ->
Text ->
(Int, Text) ->
m ()
processChunk conn eid url_ title_ (idx, chunk) = do
mVec <- generateEmbedding chunk
case mVec of
Just vec -> do
let embeddingRow = EmbeddingRow eid idx url_ title_ (Embedding vec)
liftIO $ SQL.execute conn
"INSERT OR REPLACE INTO embeddings (entry_id, chunk_idx, url, title, embedding) VALUES (?, ?, ?, ?, ?)"
embeddingRow
Nothing ->
K.logFM K.WarningS $ K.ls $ "Failed to generate embedding for chunk " <> T.pack (show idx) <> " of: " <> title_
search ::
( MonadIO m,
HasType Session r,
HasType Creds r,
MonadReader r m,
HasEmbeddingMetrics r HTTPMetric,
HasKatipTracer r KatipTracer,
MonadCatch m,
K.KatipContext m,
MonadMask m
) =>
FilePath ->
Text ->
Int ->
m [(Double, Integer, Text, Text)]
search dbPath q limit_ = do
queryVec <- generateEmbedding q
case queryVec of
Nothing -> pure []
Just qv -> do
conn <- liftIO $ open dbPath
results <- Pipes.Safe.runSafeT $ Pipes.runEffect (Pipes.Prelude.fold bestChunk Map.empty topResults $ query_ conn "SELECT entry_id, chunk_idx, url, title, embedding FROM embeddings" >-> Pipes.Prelude.mapMaybe (scoreRow qv))
liftIO $ close conn
pure results
where
bestChunk m (s, eid, url_, title_) =
Map.insertWith
(\(s1, u, t) (s2, _, _) -> (max s1 s2, u, t))
eid
(s, url_, title_)
m
sortByScore = sortOn (Down . (\(_, (s, _, _)) -> s)) . Map.toList
topResults = map (\(eid, (s, url_, title_)) -> (realToFrac s, eid, url_, title_)) . take limit_ . sortByScore
query_ :: (MonadSafe m, SQL.FromRow a) => SQL.Connection -> SQL.Query -> Pipes.Producer' a m ()
query_ conn q = Pipes.Safe.bracket (liftIO openStmt) (liftIO . SQL.closeStatement) eachRow
where
openStmt = do
SQL.openStatement conn q
eachRow stmt = do
row <- liftIO $ SQL.nextRow stmt
maybe (pure ()) (\r -> Pipes.yield r >> eachRow stmt) row
scoreRow ::
UV.Vector Float ->
EmbeddingRow ->
Maybe (Float, Integer, Text, Text)
scoreRow qv EmbeddingRow{embedding = Embedding uv, ..} =
let sim = cosineSimilarity qv uv
in Just (sim, entryId, entryUrl, entryTitle)
+11 -6
View File
@@ -12,11 +12,11 @@ import Control.Exception (bracket)
import Control.Lens (over, view)
import Control.Lens.TH (makeFields)
import Control.Monad.Base (MonadBase)
import Control.Monad.Catch (MonadCatch, MonadThrow)
import Control.Monad.Catch (MonadCatch, MonadThrow, MonadMask)
import Control.Monad.Reader (MonadIO, MonadReader, ReaderT, local, runReaderT)
import Control.Monad.Trans.Control (MonadBaseControl)
import Creds
import Data.HashMap.Strict qualified as HM
import GHC.Generics (Generic)
import HTTP (HTTPMetric (HTTPMetric))
import Katip qualified as K
@@ -37,6 +37,7 @@ data S = S
_sCreds :: Creds,
_sWallabagMetrics :: HTTPMetric,
_sOpenrouterMetrics :: HTTPMetric,
_sEmbeddingMetrics :: HTTPMetric,
_sKatipTracer :: KatipTracer,
_sMetricsStore :: EKG.Store,
_sSession :: Wreq.Session
@@ -56,7 +57,8 @@ newtype M a = M (ReaderT S IO a)
MonadThrow,
MonadCatch,
MonadBaseControl IO,
MonadBase IO
MonadBase IO,
MonadMask
)
runM :: S -> M a -> IO a
@@ -92,6 +94,11 @@ initialS credsPath severity f = do
<$> EKG.createDistribution "openrouter.requests" store
<*> EKG.createCounter "openrouter.retries" store
<*> EKG.createCounter "openrouter.failures" store
emMetrics <-
HTTPMetric
<$> EKG.createDistribution "embeddings.requests" store
<*> EKG.createCounter "embeddings.retries" store
<*> EKG.createCounter "embeddings.failures" store
c <- maybe getDefaultCreds readCreds credsPath
handleScribe <- K.mkHandleScribe K.ColorIfTerminal stdout (K.permitItem severity) K.V2
let makeLogEnv = K.registerScribe "stdout" handleScribe K.defaultScribeSettings =<< K.initLogEnv "wallabag" "production"
@@ -99,7 +106,5 @@ initialS credsPath severity f = do
let kTrace = mkKatipTrace le ns
ns = "main"
cache <- newTVarIO Unauthenticated
f (S cache ns le mempty c wbMetrics orMetrics kTrace store wsession)
-- TODO: This is just for debug purposes
EKG.sampleAll store >>= mapM_ print . HM.toList
f (S cache ns le mempty c wbMetrics orMetrics emMetrics kTrace store wsession)
pure res
+55 -4
View File
@@ -7,14 +7,16 @@ module MyLib (defaultMain, test) where
import AI
import Control.Exception.Annotated.UnliftIO (Exception, SomeException, catch, throw)
import Control.Lens (over, re, strict, view, (&), (.~), (^.))
import Control.Monad (forever)
import Control.Monad (forM_, forever)
import Data.Aeson (decode, encode)
import Data.Either (fromRight)
import Data.Maybe (fromMaybe)
import Data.Set qualified as S
import Data.Text (Text)
import Data.Text qualified as T
import Data.Text.IO qualified as T
import Data.Text.Strict.Lens (utf8)
import Embeddings (indexEntries, search)
import Katip qualified as K
import M
import Network.Wai.Handler.Warp (defaultSettings, runSettings, setPort)
@@ -22,7 +24,11 @@ import Options.Generic (Generic, ParseRecord (..), getRecord, unDefValue, unHelp
import Pipes (Pipe, for, lift, runEffect, (>->))
import Pipes.Prelude qualified as P
import Server (Health (..), app)
import System.Directory (createDirectoryIfMissing, getHomeDirectory)
import System.Environment (lookupEnv)
import System.FilePath (takeDirectory)
import Text.Pandoc (PandocError, def, readHtml, runPure, writeMarkdown)
import Text.Printf (printf)
import UnliftIO (MonadUnliftIO, liftIO)
import UnliftIO.Async (mapConcurrently_)
import UnliftIO.Concurrent (threadDelay)
@@ -30,14 +36,28 @@ import UnliftIO.Exception (displayException, isAsyncException)
import UnliftIO.IORef (IORef, newIORef, writeIORef)
import Wallabag
import Pipes.Concurrent (performGC)
import qualified Version
data Options
= Quick {verbose :: Bool}
| Full {verbose :: Bool}
| Version
| Index
{ verbose :: Bool,
rebuild :: Bool <?> "Rebuild index from scratch" <!> "False",
embeddingsDb :: Maybe FilePath <?> "Path to embeddings database"
}
| Search
{ verbose :: Bool,
query :: Text <?> "Search query",
limit :: Int <?> "Maximum results" <!> "10",
embeddingsDb :: Maybe FilePath <?> "Path to embeddings database"
}
| Daemon
{ verbose :: Bool,
credentials :: FilePath <?> "Path to the credentials.json",
port :: Int <?> "Port to run the diagnostics" <!> "8094"
port :: Int <?> "Port to run the diagnostics" <!> "8094",
embeddingsDb :: Maybe FilePath <?> "Path to embeddings database"
}
deriving (Generic, Show)
@@ -158,16 +178,47 @@ hardLabels =
"Gaming",
"Plants",
"Gadgets",
"AI"
"AI",
"Emergency",
"War",
"Open Source"
]
resolveEmbeddingsDb :: Maybe FilePath -> IO FilePath
resolveEmbeddingsDb (Just fp) = pure fp
resolveEmbeddingsDb Nothing = do
env <- lookupEnv "WALLABAG_EMBEDDINGS_DB"
case env of
Just fp -> pure fp
Nothing -> do
home <- getHomeDirectory
let dir = home <> "/.wallabag-tagger"
createDirectoryIfMissing True dir
pure $ dir <> "/embeddings.db"
runWithDb :: Maybe FilePath -> K.Severity -> (FilePath -> M ()) -> IO ()
runWithDb mbDb sev f = initialS Nothing sev $ \s -> runM s $ do
dbPath <- liftIO $ resolveEmbeddingsDb mbDb
liftIO $ createDirectoryIfMissing True (takeDirectory dbPath)
f dbPath
defaultMain :: IO ()
defaultMain = do
opts <- getRecord "Wallabag tagger"
case opts of
Quick {verbose} -> run (severity verbose) Unarchived
Full {verbose} -> run (severity verbose) Everything
Daemon {verbose, credentials, port} ->
Version -> print Version.compileTimeVersion
Index {verbose, rebuild, embeddingsDb} ->
runWithDb (unHelpful embeddingsDb) (severity verbose) $ \db ->
indexEntries db (unHelpful $ unDefValue rebuild)
Search {verbose, query, limit, embeddingsDb} ->
runWithDb (unHelpful embeddingsDb) (severity verbose) $ \db -> do
entries <- search db (unHelpful query) (unHelpful $ unDefValue limit)
liftIO $ forM_ entries $ \(score, _eid, u, t) -> do
T.putStrLn $ T.pack $ printf "%.4f %s" (score :: Double) (T.unpack t)
T.putStrLn $ T.pack $ printf " %s" (T.unpack u)
Daemon {verbose, credentials, port, embeddingsDb} ->
runDaemon (unHelpful $ unDefValue port) (unHelpful credentials) (severity verbose)
where
severity True = K.DebugS
+8 -2
View File
@@ -21,6 +21,7 @@ import System.Metrics.Distribution qualified as D
import UnliftIO (liftIO)
import UnliftIO.IORef (readIORef)
import Data.Coerce (coerce)
import Version (Version, compileTimeVersion)
data Health
= OK
@@ -57,7 +58,8 @@ newtype Metrics = Metrics (HashMap Text SampleValue)
data WallabagRoutes mode = WallabagRoutes
{ _getHealth :: mode :- "health" :> Get '[PlainText, JSON] Health,
_getMetrics :: mode :- "metrics" :> Get '[JSON] Metrics
_getMetrics :: mode :- "metrics" :> Get '[JSON] Metrics,
_getVersion :: mode :- "version" :> Get '[JSON] Version
}
deriving (Generic)
@@ -67,12 +69,16 @@ wallabagServer :: IORef Health -> WallabagRoutes (AsServerT M)
wallabagServer health =
WallabagRoutes
{ _getHealth = getHealth health,
_getMetrics = getMetrics
_getMetrics = getMetrics,
_getVersion = getVersion
}
getHealth :: IORef Health -> M Health
getHealth = readIORef
getVersion :: M Version
getVersion = pure compileTimeVersion
getMetrics :: M Metrics
getMetrics = do
sample <- view metricsStore >>= liftIO . EKG.sampleAll
+24
View File
@@ -0,0 +1,24 @@
{-# LANGUAGE TemplateHaskell #-}
{-# OPTIONS_GHC -Wno-unrecognised-pragmas #-}
{-# HLINT ignore "Unused LANGUAGE pragma" #-}
module Version where
import qualified Data.Aeson as A
import GHC.Generics (Generic)
import System.Environment (lookupEnv)
import Language.Haskell.TH (runIO, litE, stringL)
import Data.Maybe (fromMaybe)
-- Populate the version via the 'githash' package
newtype Version = Version
{ versionCommit :: String }
deriving (Show, Generic)
instance A.ToJSON Version
versionFromEnv :: IO Version
versionFromEnv = maybe (Version "unknown") Version <$> lookupEnv "COMMIT"
compileTimeVersion :: Version
compileTimeVersion = Version $(runIO (fromMaybe "unknown" <$> lookupEnv "COMMIT") >>= litE . stringL)
+8 -6
View File
@@ -46,7 +46,8 @@ data Entry = Entry
{ _entryIdentifier :: Integer,
_entryContent :: Text,
_entryTags :: [Text],
_entryTitle :: Text
_entryTitle :: Text,
_entryUrl :: Text
}
deriving (Show)
@@ -151,13 +152,13 @@ sourceEntries s = P.unfoldr (toEither <=< go) (Just startUrl) >-> P.mapFoldable
Everything -> "https://app.wallabag.it/api/entries.json"
go :: Maybe String -> m (Maybe ([Value], Maybe String))
go Nothing = pure Nothing
go (Just url) = checkpoint (Annotation url) $ do
go (Just u) = checkpoint (Annotation u) $ do
metrics <- mkTracerMetrics
t <- maybe (throw MissingTokenException) pure =<< getToken
let opts = defaults & header "Authorization" .~ ["Bearer " <> t ^. re utf8]
K.logFM K.DebugS $ K.ls ("Fetching entries '" <> url <> "'")
K.logFM K.DebugS $ K.ls ("Fetching entries '" <> u <> "'")
session <- view $ typed @Session
r <- liftIO $ getWith metrics session SafeTransient opts url
r <- liftIO $ getWith metrics session SafeTransient opts u
let items = r ^.. responseBody . key "_embedded" . key "items" . _Array . traversed
let next = r ^? responseBody . key "_links" . key "next" . key "href" . _String . unpacked
pure $ Just (items, next)
@@ -168,6 +169,7 @@ sourceEntries s = P.unfoldr (toEither <=< go) (Just startUrl) >-> P.mapFoldable
<*> v ^? key "content" . _String
<*> Just (v ^.. key "tags" . _Array . traversed . key "label" . _String)
<*> v ^? key "title" . _String
<*> v ^? key "url" . _String
mkTracerMetrics ::
( MonadReader r m,
@@ -200,8 +202,8 @@ applyTags e = checkpoint (Annotation $ e ^. identifier) $ do
t <- maybe (throw MissingTokenException) pure =<< getToken
let opts = defaults & header "Authorization" .~ ["Bearer " <> t ^. re utf8]
K.logFM K.InfoS (K.ls ("Applying tags for '" <> e ^. title <> "': " <> T.pack (show (e ^. tags))))
let url = "https://app.wallabag.it/api/entries/" <> show (e ^. identifier) <> "/tags"
let u = "https://app.wallabag.it/api/entries/" <> show (e ^. identifier) <> "/tags"
metrics <- mkTracerMetrics
session <- view $ typed @Session
_ <- liftIO $ postWith metrics session SafeTransient opts url ["tags" := T.intercalate "," (e ^. tags)]
_ <- liftIO $ postWith metrics session SafeTransient opts u ["tags" := T.intercalate "," (e ^. tags)]
pure ()
+39 -1
View File
@@ -1,4 +1,42 @@
{-# LANGUAGE OverloadedStrings #-}
module Main (main) where
import Data.Foldable (forM_)
import Embeddings (chunkArticle, tokensApprox, chunkMaxTokens, Embedding(..))
import Test.Hspec
import qualified Data.Text as T
main :: IO ()
main = putStrLn "Test suite not yet implemented."
main = hspec $ do
describe "Chunking" $ do
it "Splits a long article into multiple chunks" testChunking
it "Retains a short article as one chunk" testShortArticleOneChunk
it "Prefixes each chunk with the title" testTitlePrefix
it "Chunks to small enough items" testTokenLimit
testChunking :: IO ()
testChunking = do
let longHtml = T.replicate 1000 "<p>This is a paragraph with some text content that should be long enough to create multiple chunks when repeated many times.</p>"
chunks = chunkArticle "Test Article" longHtml
n = length chunks
n `shouldBe` 7
testShortArticleOneChunk :: IO ()
testShortArticleOneChunk = do
let shortHtml = "<p>A short paragraph.</p>"
chunks = chunkArticle "Short" shortHtml
length chunks `shouldBe` 1
testTitlePrefix :: IO ()
testTitlePrefix = do
let chunks = chunkArticle "My Title" "<p>Some content.</p>"
forM_ chunks $ \chunk ->
chunk `shouldSatisfy` ("Title: My Title\n\n" `T.isPrefixOf`)
testTokenLimit :: IO ()
testTokenLimit = do
let longHtml = T.replicate 1000 "<p>This is a paragraph with some text content that should be long enough to create multiple chunks when repeated many times.</p>"
chunks = chunkArticle "Test Article" longHtml
counts = map tokensApprox chunks
counts `shouldSatisfy` all (<= chunkMaxTokens)
+17 -4
View File
@@ -19,6 +19,7 @@ common warnings
library
import: warnings
exposed-modules: AI
Embeddings
Creds
M
Wallabag
@@ -27,9 +28,12 @@ library
HTTP
Server
Tracer
Version
-- other-modules:
-- other-extensions:
build-depends: base ^>=4.18.2.1
build-depends: base ^>=4.20
, sqlite-simple
, vector
, wreq
, lens
, katip
@@ -38,6 +42,7 @@ library
, ekg-core
, retry
, pipes
, pipes-safe
, pipes-concurrency
, streamly
, streamly-core
@@ -64,6 +69,8 @@ library
, generic-lens
, optparse-generic
, containers
, template-haskell
, cereal
hs-source-dirs: src
default-language: GHC2021
@@ -73,7 +80,7 @@ executable wallabag-tagger
-- other-modules:
-- other-extensions:
build-depends:
base ^>=4.18.2.1,
base ^>=4.20,
wallabag-tagger
hs-source-dirs: app
@@ -89,5 +96,11 @@ test-suite wallabag-tagger-test
hs-source-dirs: test
main-is: Main.hs
build-depends:
base ^>=4.18.2.1,
wallabag-tagger
base ^>=4.20,
wallabag-tagger,
text,
hspec,
hspec-hedgehog,
hedgehog,
vector,
sqlite-simple