Lazily delete excluded directories (#809)

CHANGELOG.md

@@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   directory
 - fish: detect infinite loop when using `alias cd=z`.
 - fish / Nushell / PowerShell: handle queries that look like args (e.g. `z -x`).
+- zsh: better cd completions.
+- Lazily delete excluded directories from the database.
 
 ## [0.9.4] - 2024-02-21
 

src/cmd/query.rs

@@ -1,10 +1,10 @@
 use std::io::{self, Write};
 
-use anyhow::{bail, Context, Result};
+use anyhow::{Context, Result};
 
 use crate::cmd::{Query, Run};
 use crate::config;
-use crate::db::{Database, Epoch, Stream};
+use crate::db::{Database, Epoch, Stream, StreamOptions};
 use crate::error::BrokenPipeHandler;
 use crate::util::{self, Fzf, FzfChild};
 
@@ -18,12 +18,22 @@ impl Run for Query {
 impl Query {
     fn query(&self, db: &mut Database) -> Result<()> {
         let now = util::current_time()?;
-        let mut stream = self.get_stream(db, now);
+        let mut stream = self.get_stream(db, now)?;
 
         if self.interactive {
+            self.query_interactive(&mut stream, now)
+        } else if self.list {
+            self.query_list(&mut stream, now)
+        } else {
+            self.query_first(&mut stream, now)
+        }
+    }
+
+    fn query_interactive(&self, stream: &mut Stream, now: Epoch) -> Result<()> {
         let mut fzf = Self::get_fzf()?;
         let selection = loop {
             match stream.next() {
+                Some(dir) if Some(dir.path.as_ref()) == self.exclude.as_deref() => continue,
                 Some(dir) => {
                     if let Some(selection) = fzf.write(dir, now)? {
                         break selection;
@@ -39,38 +49,44 @@ impl Query {
             let path = selection.get(7..).context("could not read selection from fzf")?;
             print!("{path}");
         }
-        } else if self.list {
-            let handle = &mut io::stdout().lock();
-            while let Some(dir) = stream.next() {
-                let dir = if self.score { dir.display().with_score(now) } else { dir.display() };
-                writeln!(handle, "{dir}").pipe_exit("stdout")?;
-            }
-        } else {
-            let handle = &mut io::stdout();
-            let Some(dir) = stream.next() else {
-                bail!(if stream.did_exclude() {
-                    "you are already in the only match"
-                } else {
-                    "no match found"
-                });
-            };
-            let dir = if self.score { dir.display().with_score(now) } else { dir.display() };
-            writeln!(handle, "{dir}").pipe_exit("stdout")?;
-        }
-
         Ok(())
     }
 
-    fn get_stream<'a>(&self, db: &'a mut Database, now: Epoch) -> Stream<'a> {
+    fn query_list(&self, stream: &mut Stream, now: Epoch) -> Result<()> {
-        let mut stream = db.stream(now).with_keywords(&self.keywords);
+        let handle = &mut io::stdout().lock();
+        while let Some(dir) = stream.next() {
+            if Some(dir.path.as_ref()) == self.exclude.as_deref() {
+                continue;
+            }
+            let dir = if self.score { dir.display().with_score(now) } else { dir.display() };
+            writeln!(handle, "{dir}").pipe_exit("stdout")?;
+        }
+        Ok(())
+    }
+
+    fn query_first(&self, stream: &mut Stream, now: Epoch) -> Result<()> {
+        let handle = &mut io::stdout();
+
+        let mut dir = stream.next().context("no match found")?;
+        while Some(dir.path.as_ref()) == self.exclude.as_deref() {
+            dir = stream.next().context("you are already in the only match")?;
+        }
+
+        let dir = if self.score { dir.display().with_score(now) } else { dir.display() };
+        writeln!(handle, "{dir}").pipe_exit("stdout")
+    }
+
+    fn get_stream<'a>(&self, db: &'a mut Database, now: Epoch) -> Result<Stream<'a>> {
+        let mut options = StreamOptions::new(now)
+            .with_keywords(self.keywords.iter().map(|s| s.as_str()))
+            .with_exclude(config::exclude_dirs()?);
         if !self.all {
             let resolve_symlinks = config::resolve_symlinks();
-            stream = stream.with_exists(resolve_symlinks);
+            options = options.with_exists(true).with_resolve_symlinks(resolve_symlinks);
         }
-        if let Some(path) = &self.exclude {
+
-            stream = stream.with_exclude(path);
+        let stream = Stream::new(db, options);
-        }
+        Ok(stream)
-        stream
     }
 
     fn get_fzf() -> Result<FzfChild> {

src/db/mod.rs

@@ -9,7 +9,7 @@ use bincode::Options;
 use ouroboros::self_referencing;
 
 pub use crate::db::dir::{Dir, Epoch, Rank};
-pub use crate::db::stream::Stream;
+pub use crate::db::stream::{Stream, StreamOptions};
 use crate::{config, util};
 
 #[self_referencing]
@@ -135,10 +135,6 @@ impl Database {
         self.with_dirty_mut(|dirty_prev| *dirty_prev |= dirty);
     }
 
-    pub fn stream(&mut self, now: Epoch) -> Stream {
-        Stream::new(self, now)
-    }
-
     pub fn dedup(&mut self) {
         // Sort by path, so that equal paths are next to each other.
         self.sort_by_path();

src/db/stream.rs

@@ -2,80 +2,44 @@ use std::iter::Rev;
 use std::ops::Range;
 use std::{fs, path};
 
+use glob::Pattern;
+
 use crate::db::{Database, Dir, Epoch};
 use crate::util::{self, MONTH};
 
 pub struct Stream<'a> {
-    // State
     db: &'a mut Database,
     idxs: Rev<Range<usize>>,
-    did_exclude: bool,
+    options: StreamOptions,
-
-    // Configuration
-    keywords: Vec<String>,
-    check_exists: bool,
-    expire_below: Epoch,
-    resolve_symlinks: bool,
-    exclude_path: Option<String>,
 }
 
 impl<'a> Stream<'a> {
-    pub fn new(db: &'a mut Database, now: Epoch) -> Self {
+    pub fn new(db: &'a mut Database, options: StreamOptions) -> Self {
-        db.sort_by_score(now);
+        db.sort_by_score(options.now);
         let idxs = (0..db.dirs().len()).rev();
-
+        Stream { db, idxs, options }
-        // If a directory is deleted and hasn't been used for 3 months, delete
-        // it from the database.
-        let expire_below = now.saturating_sub(3 * MONTH);
-
-        Stream {
-            db,
-            idxs,
-            did_exclude: false,
-            keywords: Vec::new(),
-            check_exists: false,
-            expire_below,
-            resolve_symlinks: false,
-            exclude_path: None,
-        }
-    }
-
-    pub fn with_exclude(mut self, path: impl Into<String>) -> Self {
-        self.exclude_path = Some(path.into());
-        self
-    }
-
-    pub fn with_exists(mut self, resolve_symlinks: bool) -> Self {
-        self.check_exists = true;
-        self.resolve_symlinks = resolve_symlinks;
-        self
-    }
-
-    pub fn with_keywords(mut self, keywords: &[impl AsRef<str>]) -> Self {
-        self.keywords = keywords.iter().map(util::to_lowercase).collect();
-        self
     }
 
     pub fn next(&mut self) -> Option<&Dir> {
         while let Some(idx) = self.idxs.next() {
             let dir = &self.db.dirs()[idx];
 
-            if !self.matches_keywords(&dir.path) {
+            if !self.filter_by_keywords(&dir.path) {
                 continue;
             }
 
-            if !self.matches_exists(&dir.path) {
+            if !self.filter_by_exclude(&dir.path) {
-                if dir.last_accessed < self.expire_below {
+                self.db.swap_remove(idx);
+                continue;
+            }
+
+            if !self.filter_by_exists(&dir.path) {
+                if dir.last_accessed < self.options.ttl {
                     self.db.swap_remove(idx);
                 }
                 continue;
             }
 
-            if Some(dir.path.as_ref()) == self.exclude_path.as_deref() {
-                self.did_exclude = true;
-                continue;
-            }
-
             let dir = &self.db.dirs()[idx];
             return Some(dir);
         }
@@ -83,20 +47,8 @@ impl<'a> Stream<'a> {
         None
     }
 
-    pub fn did_exclude(&self) -> bool {
+    fn filter_by_keywords(&self, path: &str) -> bool {
-        self.did_exclude
+        let (keywords_last, keywords) = match self.options.keywords.split_last() {
-    }
-
-    fn matches_exists(&self, path: &str) -> bool {
-        if !self.check_exists {
-            return true;
-        }
-        let resolver = if self.resolve_symlinks { fs::symlink_metadata } else { fs::metadata };
-        resolver(path).map(|m| m.is_dir()).unwrap_or_default()
-    }
-
-    fn matches_keywords(&self, path: &str) -> bool {
-        let (keywords_last, keywords) = match self.keywords.split_last() {
             Some(split) => split,
             None => return true,
         };
@@ -122,6 +74,77 @@ impl<'a> Stream<'a> {
 
         true
     }
+
+    fn filter_by_exclude(&self, path: &str) -> bool {
+        !self.options.exclude.iter().any(|pattern| pattern.matches(path))
+    }
+
+    fn filter_by_exists(&self, path: &str) -> bool {
+        if !self.options.exists {
+            return true;
+        }
+        let resolver =
+            if self.options.resolve_symlinks { fs::metadata } else { fs::symlink_metadata };
+        resolver(path).map(|metadata| metadata.is_dir()).unwrap_or_default()
+    }
+}
+
+pub struct StreamOptions {
+    /// The current time.
+    now: Epoch,
+
+    /// Only directories matching these keywords will be returned.
+    keywords: Vec<String>,
+
+    /// Directories that match any of these globs will be lazily removed.
+    exclude: Vec<Pattern>,
+
+    /// Directories will only be returned if they exist on the filesystem.
+    exists: bool,
+
+    /// Whether to resolve symlinks when checking if a directory exists.
+    resolve_symlinks: bool,
+
+    /// Directories that do not exist and haven't been accessed since TTL will
+    /// be lazily removed.
+    ttl: Epoch,
+}
+
+impl StreamOptions {
+    pub fn new(now: Epoch) -> Self {
+        StreamOptions {
+            now,
+            keywords: Vec::new(),
+            exclude: Vec::new(),
+            exists: false,
+            resolve_symlinks: false,
+            ttl: now.saturating_sub(3 * MONTH),
+        }
+    }
+
+    pub fn with_keywords<I>(mut self, keywords: I) -> Self
+    where
+        I: IntoIterator,
+        I::Item: AsRef<str>,
+    {
+        self.keywords = keywords.into_iter().map(util::to_lowercase).collect();
+        self
+    }
+
+    pub fn with_exclude(mut self, exclude: Vec<Pattern>) -> Self {
+        self.exclude = exclude;
+        self
+    }
+
+    pub fn with_exists(mut self, exists: bool) -> Self {
+        self.exists = exists;
+        self
+    }
+
+    pub fn with_resolve_symlinks(mut self, resolve_symlinks: bool) -> Self {
+        self.resolve_symlinks = resolve_symlinks;
+        self
+    }
 }
 
 #[cfg(test)]
@@ -154,7 +177,8 @@ mod tests {
     #[case(&["/foo/", "/bar"], "/foo/baz/bar", true)]
     fn query(#[case] keywords: &[&str], #[case] path: &str, #[case] is_match: bool) {
         let db = &mut Database::new(PathBuf::new(), Vec::new(), |_| Vec::new(), false);
-        let stream = Stream::new(db, 0).with_keywords(keywords);
+        let options = StreamOptions::new(0).with_keywords(keywords.iter());
-        assert_eq!(is_match, stream.matches_keywords(path));
+        let stream = Stream::new(db, options);
+        assert_eq!(is_match, stream.filter_by_keywords(path));
     }
 }