Store metadata

src/api/syncs.rs

@@ -14,6 +14,13 @@ pub fn syncs_routes() -> Router<AppState> {
         .route("/progress/{document}", routing::get(get_progress))
 }
 
+#[derive(Debug, Deserialize)]
+pub struct Metadata {
+    filename: String,
+    title: String,
+    authors: String,
+}
+
 #[derive(Debug,Deserialize)]
 pub struct PutProgress {
     pub document: String,
@@ -21,6 +28,7 @@ pub struct PutProgress {
     pub percentage: f32,
     pub device: String,
     pub device_id: Option<String>,
+    pub metadata: Option<Metadata>,
 }
 
 #[derive(Debug,Serialize)]
@@ -44,6 +52,10 @@ pub async fn put_progress(State(st) : State<AppState>, AuthUser{username} : Auth
         percentage: progress.percentage,
         device: progress.device,
         device_id: progress.device_id,
+        title: progress.metadata.as_ref().map(|m| m.title.clone()),
+        author: progress.metadata.as_ref().map(|m| m.authors.clone()),
+        filename: progress.metadata.as_ref().map(|m| m.filename.clone()),
+        ..Document::default()
     };
     st.sync.set_progress(&username, &document).await?;
     Ok(Json(ProgressCreated{timestamp: document.timestamp, document: document.document_id}))

src/main.rs

@@ -8,7 +8,7 @@ use tracing_subscriber::{EnvFilter, fmt, layer::SubscriberExt as _, util::Subscr
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
     let level_filter = EnvFilter::builder()
-        .with_default_directive(LevelFilter::INFO.into())
+        .with_default_directive(LevelFilter::DEBUG.into())
         .from_env_lossy();
     tracing_subscriber::registry()
         .with(fmt::layer())

src/sync.rs

@@ -18,7 +18,7 @@ pub enum SyncError {
     DatabaseFailure(#[from] tokio_rusqlite::Error)
 }
 
-#[derive(Debug, PartialEq, Clone)]
+#[derive(Debug, PartialEq, Clone, Default)]
 pub struct Document {
     // document identifier, name of the file, md5 of the file or something
     pub document_id: String,
@@ -28,6 +28,9 @@ pub struct Document {
     pub device: String,
     pub device_id: Option<String>,
     pub timestamp: i64,
+    pub title: Option<String>,
+    pub filename: Option<String>,
+    pub author: Option<String>,
 }
 
 impl ProgressSync {
@@ -39,7 +42,21 @@ impl ProgressSync {
         let username = username.to_owned();
         let doc = document.clone();
         self.writer.connection.call(move |conn| -> rusqlite::Result<()> {
-            conn.execute("insert into progress (user_id, document_id, progress, percentage, device, device_id, timestamp)
+            let t = conn.transaction()?;
+            t.execute("insert into document
+                (id, title, filename, author)
+                values
+                (:document_id, :title, :filename, :author) on conflict(id) do update
+                set title = coalesce(excluded.title, document.title),
+                    author = coalesce(excluded.author, document.author),
+                    filename = coalesce(excluded.filename, document.filename)
+                ", named_params! {
+                    ":document_id": doc.document_id,
+                    ":filename": doc.filename,
+                    ":title": doc.title,
+                    ":author": doc.author,
+                })?;
+            t.execute("insert into progress (user_id, document_id, progress, percentage, device, device_id, timestamp)
                 select id, :document_id, :progress, :percentage, :device, :device_id, :timestamp from users
                 where username = :username", named_params!
                 { ":username": username,
@@ -50,6 +67,7 @@ impl ProgressSync {
                   ":device_id": doc.device_id,
                   ":timestamp": doc.timestamp,
                 })?;
+            t.commit()?;
             Ok(())
         }).await?;
         Ok(())
@@ -60,9 +78,10 @@ impl ProgressSync {
         let document_id = document_id.to_owned();
         let doc = self.reader.connection.call(move |conn| -> rusqlite::Result<Option<Document>> {
             let x : Option<Document> = conn.query_row("select
-                document_id, progress, percentage, device, device_id, timestamp
+                document_id, progress, percentage, device, device_id, timestamp, title, author, filename
                 from progress
                 left join users on progress.user_id = users.id
+                left join document on progress.document_id = document.id
                 where users.username = :username and progress.document_id = :document_id
                 order by progress.rowid desc
                 limit 1", named_params!
@@ -75,7 +94,20 @@ impl ProgressSync {
                     let device : String = row.get(3)?;
                     let device_id : Option<String> = row.get(4)?;
                     let timestamp : i64 = row.get(5)?;
-                    Ok(Document{ document_id, progress, percentage, device, device_id, timestamp })
+                    let title : Option<String> = row.get(6)?;
+                    let author : Option<String> = row.get(7)?;
+                    let filename : Option<String> = row.get(8)?;
+                    Ok(Document{
+                        document_id,
+                        progress,
+                        percentage,
+                        device,
+                        device_id,
+                        timestamp,
+                        title,
+                        author,
+                        filename,
+                    })
                 }).optional()?;
             Ok(x)
         }).await?;
@@ -102,7 +134,15 @@ impl ProgressSync {
                     let device : String = row.get(3)?;
                     let device_id : Option<String> = row.get(4)?;
                     let timestamp : i64 = row.get(5)?;
-                    Ok(Document{ document_id, progress, percentage, device, device_id, timestamp })
+                    Ok(Document{
+                        document_id,
+                        progress,
+                        percentage,
+                        device,
+                        device_id,
+                        timestamp,
+                        ..Document::default()
+                    })
                 })?.collect::<rusqlite::Result<Vec<_>>>()?;
             Ok(x)
         }).await?;
@@ -150,6 +190,7 @@ use secrecy::SecretString;
             device: "kobo".to_string(),
             device_id: None,
             timestamp: now()?,
+            ..Document::default()
         };
 
         sync.set_progress(&username, &doc).await?;
@@ -174,6 +215,7 @@ use secrecy::SecretString;
             device: "kobo".to_string(),
             device_id: None,
             timestamp: now()?,
+            ..Document::default()
         };
         let doc2 = Document {
             document_id: document_id.clone(),
@@ -182,6 +224,7 @@ use secrecy::SecretString;
             device: "kobo".to_string(),
             device_id: None,
             timestamp: now()?,
+            ..Document::default()
         };
 
         sync.set_progress(&username, &doc1).await?;
@@ -208,6 +251,7 @@ use secrecy::SecretString;
             device: "kobo".to_string(),
             device_id: None,
             timestamp: now()?,
+            ..Document::default()
         };
         let doc2 = Document {
             document_id: document_id.clone(),
@@ -216,6 +260,7 @@ use secrecy::SecretString;
             device: "x4".to_string(),
             device_id: None,
             timestamp: now()? + 3,
+            ..Document::default()
         };
 
         sync.set_progress(&username, &doc1).await?;
@@ -241,6 +286,7 @@ use secrecy::SecretString;
             device: "kobo".to_string(),
             device_id: None,
             timestamp: now()?,
+            ..Document::default()
         };
         let doc2 = Document {
             document_id: document_id.clone(),
@@ -249,6 +295,7 @@ use secrecy::SecretString;
             device: "kobo".to_string(),
             device_id: None,
             timestamp: now()? + 3,
+            ..Document::default()
         };
 
         sync.set_progress(&username, &doc1).await?;
@@ -258,4 +305,95 @@ use secrecy::SecretString;
         assert_eq!(got, vec![doc2, doc1]);
         Ok(())
     }
+
+    #[tokio::test]
+    async fn stores_metadata() -> anyhow::Result<()> {
+        let (_t, users, sync) = create_data().await?;
+
+        let username = "foo@example.com";
+        let username = users.create_user(username, SecretString::from("password")).await?;
+        let document_id = "foo".to_string();
+
+        let doc1 = Document {
+            document_id: document_id.clone(),
+            progress: "prog".to_string(),
+            percentage: 0.5,
+            device: "kobo".to_string(),
+            device_id: None,
+            timestamp: now()?,
+            filename: Some("foo.epub".to_string()),
+            title: Some("A Book".to_string()),
+            author: Some("A.N. Author".to_string()),
+        };
+
+        sync.set_progress(&username, &doc1).await?;
+        let got = sync.get_progress(&username, &document_id).await?;
+
+        assert_eq!(got, Some(doc1));
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn updates_metadata() -> anyhow::Result<()> {
+        let (_t, users, sync) = create_data().await?;
+
+        let username = "foo@example.com";
+        let username = users.create_user(username, SecretString::from("password")).await?;
+        let document_id = "foo".to_string();
+
+        let doc1 = Document {
+            document_id: document_id.clone(),
+            progress: "prog".to_string(),
+            percentage: 0.5,
+            device: "kobo".to_string(),
+            device_id: None,
+            timestamp: now()?,
+            filename: Some("foo.epub".to_string()),
+            title: Some("A Book".to_string()),
+            author: Some("A.N. Author".to_string()),
+        };
+        let doc2 = Document {
+            filename: Some("foobar.epub".to_string()),
+            ..doc1.clone()
+        };
+
+        sync.set_progress(&username, &doc1).await?;
+        sync.set_progress(&username, &doc2).await?;
+        let got = sync.get_progress(&username, &document_id).await?;
+
+        assert_eq!(got, Some(doc2));
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn metadata_is_not_renulled() -> anyhow::Result<()> {
+        let (_t, users, sync) = create_data().await?;
+
+        let username = "foo@example.com";
+        let username = users.create_user(username, SecretString::from("password")).await?;
+        let document_id = "foo".to_string();
+
+        let doc1 = Document {
+            document_id: document_id.clone(),
+            progress: "prog".to_string(),
+            percentage: 0.5,
+            device: "kobo".to_string(),
+            device_id: None,
+            timestamp: now()?,
+            filename: Some("foo.epub".to_string()),
+            title: Some("A Book".to_string()),
+            author: Some("A.N. Author".to_string()),
+        };
+        let doc2 = Document {
+            filename: None,
+            ..doc1.clone()
+        };
+
+        sync.set_progress(&username, &doc1).await?;
+        sync.set_progress(&username, &doc2).await?;
+        let got = sync.get_progress(&username, &document_id).await?;
+
+        assert_eq!(got, Some(doc1));
+        Ok(())
+    }
 }

src/writer.rs

@@ -1,6 +1,7 @@
 use std::path::Path;
 
-use tokio_rusqlite::{Connection, rusqlite};
+use sha2::{Digest as _, Sha256};
+use tokio_rusqlite::{Connection, OptionalExtension as _, named_params, rusqlite};
 
 
 pub struct Writer {
@@ -9,21 +10,41 @@ pub struct Writer {
     pub connection: Connection,
 }
 
+const MIGRATIONS : [&str; 8] = [
+    "create table if not exists users (id primary key, username not null, secret not null, unique(username))",
+    "create table if not exists progress (user_id not null, document_id not null, progress not null, percentage not null, device not null, device_id, timestamp not null, unique(user_id, document_id), foreign key (user_id) references users(id))",
+    "create table document (id not null primary key, title, filename, author)",
+    "insert into document (id) select distinct(document_id) from progress",
+    "alter table progress rename to progress_bak",
+    "create table progress (user_id not null, document_id not null, progress not null, percentage not null, device not null, device_id, timestamp not null, foreign key (user_id) references users(id), foreign key (document_id) references document(id))",
+    "insert into progress (user_id, document_id, progress, percentage, device, device_id, timestamp) select user_id, document_id, progress, percentage, device, device_id, timestamp from progress_bak",
+    "drop table progress_bak",
+];
+
+struct Migration { sql: String, fingerprint: Vec<u8> }
+
+fn get_migrations() -> Vec<Migration> {
+    MIGRATIONS
+        .iter()
+        .map(|m| { Migration{sql: m.to_string(), fingerprint: Sha256::digest(m).to_vec() } })
+        .collect()
+}
+
 impl Writer {
+    #[tracing::instrument]
     pub async fn new(path: &Path) -> Result<Self, tokio_rusqlite::Error> {
         let connection = Connection::open(path).await?;
 
         connection.call(|conn| -> rusqlite::Result<()> {
-            conn.execute("create table if not exists users (id primary key, username not null, secret not null, unique(username))", [])?;
+            conn.execute("create table if not exists migrations (hash not null, created_at)", [])?;
-            conn.execute("create table if not exists progress (user_id not null, document_id not null, progress not null, percentage not null, device not null, device_id, timestamp not null, unique(user_id, document_id), foreign key (user_id) references users(id))", [])?;
+            for Migration{sql,fingerprint} in get_migrations() {
+                if let None = conn.query_one("select * from migrations where hash = :hash", named_params! {":hash": fingerprint}, |_| Ok(true)).optional()? {
+                    tracing::debug!("Applying {:?}", sql);
+                    conn.execute(&sql, [])?;
+                    conn.execute("insert into migrations (hash, created_at) values (:hash, :now)", named_params! {":hash": fingerprint, ":now": 0 })?;
+                }
+            }
 
-            // Migrations
-            //
-            // Store history
-            conn.execute("alter table progress rename to progress_bak", [])?;
-            conn.execute("create table if not exists progress (user_id not null, document_id not null, progress not null, percentage not null, device not null, device_id, timestamp not null, foreign key (user_id) references users(id))", [])?;
-            conn.execute("insert into progress (user_id, document_id, progress, percentage, device, device_id, timestamp) select user_id, document_id, progress, percentage, device, device_id, timestamp from progress_bak", [])?;
-            conn.execute("drop table progress_bak", [])?;
             Ok(())
         }).await?;