bobwen-dev
/
hunter
mirror of https://github.com/bobwen-dev/hunter
1
0
Fork 0
Code Issues Releases Wiki Activity

fixed hangs when updating large directories (>10k files)

This commit is contained in:
rabite 2019-12-23 02:08:50 +01:00
parent 03693f96d0
commit ef29c47e88
8 changed files with 557 additions and 323 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
Cargo.lock generated
View File

@ -258,6 +258,16 @@ dependencies = [
"byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "derivative"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.44 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "dirs"
version = "1.0.5"
@ -410,7 +420,7 @@ dependencies = [
"glib 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)",
"glib-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
"muldiv 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -427,21 +437,21 @@ dependencies = [
"glib-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer 0.14.5 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-app-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-app-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-base 0.14.4 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-base-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-base-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "gstreamer-app-sys"
version = "0.8.0"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"glib-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-base-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-base-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -456,19 +466,19 @@ dependencies = [
"glib-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer 0.14.5 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-base-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-base-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "gstreamer-base-sys"
version = "0.8.0"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"glib-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -484,7 +494,7 @@ dependencies = [
"gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer 0.14.5 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-player-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-video 0.14.5 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -496,15 +506,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"glib-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-video-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-video-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "gstreamer-sys"
version = "0.8.0"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"glib-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
@ -524,22 +534,22 @@ dependencies = [
"gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer 0.14.5 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-base 0.14.4 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-base-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-video-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-base-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-video-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "gstreamer-video-sys"
version = "0.8.0"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"glib-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-base-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-base-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -554,7 +564,7 @@ dependencies = [
[[package]]
name = "hermit-abi"
version = "0.1.3"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
@ -569,6 +579,7 @@ dependencies = [
"base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
"chrono 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
"derivative 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
"dirs-2 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
"failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
@ -908,7 +919,7 @@ name = "num_cpus"
version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"hermit-abi 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
"hermit-abi 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -1562,6 +1573,7 @@ dependencies = [
"checksum crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6"
"checksum crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ce446db02cdc3165b94ae73111e570793400d0794e46125cc4056c81cbb039f4"
"checksum deflate 0.7.20 (registry+https://github.com/rust-lang/crates.io-index)" = "707b6a7b384888a70c8d2e8650b3e60170dfc6a67bb4aa67b6dfca57af4bedb4"
"checksum derivative 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "942ca430eef7a3806595a6737bc388bf51adb888d3fc0dd1b50f1c170167ee3a"
"checksum dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901"
"checksum dirs-2 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "50b7e2b65c73137ec48935d50a5ae89b03150df566b7e14a1371df044e76765c"
"checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
@ -1581,16 +1593,16 @@ dependencies = [
"checksum gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31d1a804f62034eccf370006ccaef3708a71c31d561fee88564abe71177553d9"
"checksum gstreamer 0.14.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fa91e470b0cd4b05611f7d0e89caf76e39752156440877f04c23ad34ffc9761c"
"checksum gstreamer-app 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a85485c2db4149ccb24d0b3c6598725743dec254bf757ac7a3684e62b9822c27"
"checksum gstreamer-app-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "41c85ef44d827b9292833203f6623cf6592d5eda06ad1eeefa63bca0cc38ce71"
"checksum gstreamer-app-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bf869ce152c23bca5d761ab62146b47f750d0b28d4d499731857532897d48167"
"checksum gstreamer-base 0.14.4 (registry+https://github.com/rust-lang/crates.io-index)" = "37ed3ddb8f1a65e401340afb5657d2107d78c82f06a081e19393fc8d4c5ee720"
"checksum gstreamer-base-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9ba1955ea091323c17fdf8ff54fd7cf3dfed1a6035193ba08f85eb76bf549056"
"checksum gstreamer-base-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ba384f52174b3c586593fca32642680a9e67961fea9f4cd8419f678965023bed"
"checksum gstreamer-player 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "78395f87de2b954ca3e33a594a4eb85e68246f5f41a70bf0ab52187aacb4d3a9"
"checksum gstreamer-player-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c3608d3e96c8977f31b9b8db7da0b8d0e96758b060e3f05fc3ee9626d75ab1c5"
"checksum gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bdfc2f6cc9b6a1f5159bfd500310fe431cfb0b74b3af17ce3fdf8353cf586975"
"checksum gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1d18da01b97d0ab5896acd5151e4c155acefd0e6c03c3dd24dd133ba054053db"
"checksum gstreamer-video 0.14.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0b48c3edfa6c0419ad7587a557bbed716caf5585c3be65bfbe3584703735df51"
"checksum gstreamer-video-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6b8fcb1e577de93d1ad1e5117234ce64d40f215143d752140719923651608983"
"checksum gstreamer-video-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "615f4298f842f4b4581606e13cf52e1710e2130d989bb99161a5665aa3ccb7cc"
"checksum heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205"
"checksum hermit-abi 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "307c3c9f937f38e3534b1d6447ecf090cafcc9744e4a6360e8b037b2cf5af120"
"checksum hermit-abi 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f629dc602392d3ec14bfc8a09b5e644d7ffd725102b48b81e59f90f2633621d7"
"checksum image 0.21.3 (registry+https://github.com/rust-lang/crates.io-index)" = "35371e467cd7b0b3d1d6013d619203658467df12d61b0ca43cd67b743b1965eb"
"checksum inflate 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "1cdb29978cc5797bd8dcc8e5bf7de604891df2a8dc576973d71a281e916db2ff"
"checksum inotify 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "40b54539f3910d6f84fbf9a643efd6e3aa6e4f001426c0329576128255994718"

3
Cargo.toml
View File

@ -27,7 +27,7 @@ chrono = "0.4"
libc = "0.2.51"
failure = "0.1.5"
failure_derive = "0.1.1"
notify = "4.0.12"
notify = "4.0.14"
parse-ansi = "0.1.6"
signal-notify = "0.1.3"
systemstat = "0.1.5"
@ -41,6 +41,7 @@ base64 = "0.10.1"
strum = "0.15"
strum_macros = "0.15"
rust-ini = "0.13"
derivative = "1.0.3"
image = { version = "0.21.1", optional = true }

52
src/file_browser.rs
View File

@ -168,21 +168,6 @@ impl Tabbable for TabView<FileBrowser> {
}
fn on_refresh(&mut self) -> HResult<()> {
let fs_changes = self.active_tab_()
.fs_cache
.fs_changes
.write()?
.drain(..)
.collect::<Vec<_>>();
for tab in &mut self.widgets {
for (dir, old_file, new_file) in fs_changes.iter() {
tab.replace_file(&dir,
old_file.as_ref(),
new_file.as_ref()).log()
}
}
let open_dirs = self.widgets
.iter()
.fold(HashSet::new(), |mut dirs, tab| {
@ -503,7 +488,7 @@ impl FileBrowser {
}
pub fn main_widget_goto(&mut self, dir: &File) -> HResult<()> {
self.cache_files().log();
self.save_tab_settings().log();
let dir = dir.clone();
let cache = self.fs_cache.clone();
@ -733,22 +718,15 @@ impl FileBrowser {
Ok(&self.left_widget()?.content)
}
pub fn cache_files(&mut self) -> HResult<()> {
if self.main_widget().is_ok() {
pub fn save_tab_settings(&mut self) -> HResult<()> {
if !self.main_async_widget_mut()?.ready() { return Ok(()) }
if self.main_widget()?.content.len() > 0 {
let files = self.get_files()?;
let selected_file = self.selected_file().ok();
self.fs_cache.put_files(files, selected_file).log();
self.main_widget_mut()?.content.meta_updated = false;
self.fs_cache.save_settings(files, selected_file).log();
}
// if self.cwd.parent().is_some() {
// let left_selection = self.left_widget()?.clone_selected_file();
// let left_files = self.get_left_files()?;
// self.fs_cache.put_files(left_files, Some(left_selection)).log();
// self.left_widget_mut()?.content.meta_updated = false;
// }
Ok(())
}
@ -769,22 +747,6 @@ impl FileBrowser {
Ok(dir)
}
fn replace_file(&mut self,
dir: &File,
old: Option<&File>,
new: Option<&File>) -> HResult<()> {
if &self.cwd == dir {
self.main_widget_mut()?.content.replace_file(old, new.cloned()).log();
}
self.preview_widget_mut()?.replace_file(dir, old, new).ok();
if &self.left_dir()? == &dir {
self.left_widget_mut()?.content.replace_file(old, new.cloned()).log();
}
Ok(())
}
pub fn selected_file(&self) -> HResult<File> {
let widget = self.main_widget()?;
let file = widget.selected_file().clone();
@ -1288,7 +1250,7 @@ impl Widget for FileBrowser {
self.set_cwd().log();
if !self.columns.zoom_active { self.update_preview().log(); }
self.columns.refresh().log();
self.cache_files().log();
self.save_tab_settings().log();
Ok(())
}

299
src/files.rs
View File

@ -1,4 +1,5 @@
use std::cmp::{Ord, Ordering};
use std::collections::{HashMap, HashSet};
use std::ops::Index;
use std::fs::Metadata;
use std::os::unix::fs::MetadataExt;
@ -16,7 +17,6 @@ use users::{get_current_username,
get_group_by_gid};
use chrono::TimeZone;
use failure::Error;
use notify::DebouncedEvent;
use rayon::{ThreadPool, ThreadPoolBuilder};
use alphanumeric_sort::compare_str;
use mime_guess;
@ -28,6 +28,7 @@ use crate::fail::{HResult, HError, ErrorLog};
use crate::dirty::{AsyncDirtyBit, DirtyBit, Dirtyable};
use crate::widget::Events;
use crate::icon::Icons;
use crate::fscache::FsEvent;
lazy_static! {
@ -99,11 +100,147 @@ pub fn tags_loaded() -> HResult<()> {
#[derive(PartialEq, Eq, Hash, Clone, Debug)]
pub struct RefreshPackage {
pub new_files: Option<Vec<File>>,
pub new_buffer: Option<Vec<String>>,
}
impl RefreshPackage {
fn new(mut files: Files,
old_buffer: Vec<String>,
events: Vec<FsEvent>,
render_fn: impl Fn(&File) -> String) -> RefreshPackage {
use FsEvent::*;
// If there is only a placeholder at this point, remove it now
if files.len() == 1 {
files.remove_placeholder();
}
//To preallocate collections
let event_count = events.len();
// Need at least one copy for the hashmaps
let static_files = files.clone();
// Optimization to speed up access to array
let file_pos_map: HashMap<&File, usize> = static_files
.files
.iter()
.enumerate()
.map(|(i, file)| (file, i))
.collect();
// Need to know which line of the ListView buffer belongs to which file
let list_pos_map: HashMap<&File, usize> = static_files
.iter_files()
.enumerate()
.take_while(|&(i, _)| i < old_buffer.len())
.map(|(i, file)| (file, i))
.collect();
// Save new files to add them all at once later
let mut new_files = Vec::with_capacity(event_count);
// Files that need rerendering to make all changes visible (size, etc.)
let mut changed_files = HashSet::with_capacity(event_count);
// Save deletions to delete them efficiently later
let mut deleted_files = HashSet::with_capacity(event_count);
for event in events.into_iter() {
match event {
Create(mut file) => {
let dirty_meta = files.dirty_meta.clone();
file.dirty_meta = Some(dirty_meta);
file.meta_sync().log();
new_files.push(file);
}
Change(file) => {
if let Some(&fpos) = file_pos_map.get(&file) {
files.files[fpos].meta_sync().log();
changed_files.insert(file);
}
}
Rename(old, new) => {
if let Some(&fpos) = file_pos_map.get(&old) {
files.files[fpos].rename(&new.path).log();
files.files[fpos].meta_sync().log();
}
}
Remove(file) => {
if let Some(_) = file_pos_map.get(&file) {
deleted_files.insert(file);
}
}
}
}
if deleted_files.len() > 0 {
files.files.retain(|file| !deleted_files.contains(file));
}
// Finally add all new files
files.files.extend(new_files);
// Need to unpack this to prevent issue with recursive Files type
// Also, if no files remain add placeholder
let files = if files.len() > 0 {
// Sort again to make sure new/changed files are in correct order
files.sort();
files.files
} else {
let placeholder = File::new_placeholder(&files.directory.path).unwrap();
files.files.push(placeholder);
// Need to sort because of possible hidden files
files.sort();
files.files
};
let mut old_buffer = old_buffer;
// Prerender new buffer in current thread
let new_buffer = files.iter()
.map(|file| {
match list_pos_map.get(&file) {
Some(&old_pos) =>
match changed_files.contains(&file) {
true => render_fn(&file),
false => std::mem::take(&mut old_buffer[old_pos])
}
None => render_fn(&file)
}
}).collect();
RefreshPackage {
new_files: Some(files),
new_buffer: Some(new_buffer),
}
}
}
#[derive(Derivative)]
#[derivative(PartialEq, Eq, Hash, Clone, Debug)]
pub struct Files {
pub directory: File,
pub files: Vec<File>,
#[derivative(Debug="ignore")]
#[derivative(PartialEq="ignore")]
#[derivative(Hash="ignore")]
pub pending_events: Arc<RwLock<Vec<FsEvent>>>,
#[derivative(Debug="ignore")]
#[derivative(PartialEq="ignore")]
#[derivative(Hash="ignore")]
pub refresh: Option<Async<RefreshPackage>>,
pub meta_upto: Option<usize>,
pub meta_updated: bool,
pub sort: SortBy,
pub dirs_first: bool,
pub reverse: bool,
@ -158,8 +295,9 @@ impl Files {
let mut files = Files {
directory: File::new_from_path(&path, None)?,
files: files,
pending_events: Arc::new(RwLock::new(vec![])),
refresh: None,
meta_upto: None,
meta_updated: false,
sort: SortBy::Name,
dirs_first: true,
reverse: false,
@ -216,8 +354,9 @@ impl Files {
let mut files = Files {
directory: File::new_from_path(&path, None)?,
files: files,
pending_events: Arc::new(RwLock::new(vec![])),
refresh: None,
meta_upto: None,
meta_updated: false,
sort: SortBy::Name,
dirs_first: true,
reverse: false,
@ -375,77 +514,69 @@ impl Files {
}
}
pub fn replace_file(&mut self,
old: Option<&File>,
new: Option<File>) -> HResult<()> {
let (tag, selected) = if let Some(old) = old {
if let Some(old) = self.find_file_with_path(&old.path) {
(old.tag, old.selected)
} else {
(None, false)
}
} else {
(None, false)
};
old.map(|old| self.files.remove_item(old));
new.map(|mut new| {
new.tag = tag;
new.selected = selected;
self.files.push(new);
});
self.sort();
if self.len() == 0 {
let placeholder = File::new_placeholder(&self.directory.path)?;
self.files.push(placeholder);
} else {
self.remove_placeholder();
}
Ok(())
}
fn remove_placeholder(&mut self) {
let dirpath = self.directory.path.clone();
self.find_file_with_path(&dirpath).cloned()
.map(|placeholder| self.files.remove_item(&placeholder));
}
pub fn handle_event(&mut self,
event: &DebouncedEvent) -> HResult<()> {
match event {
DebouncedEvent::Create(path) => {
self.path_in_here(&path)?;
let file = File::new_from_path(&path,
Some(self.dirty_meta.clone()))?;
self.files.push(file);
self.sort();
},
DebouncedEvent::Write(path) | DebouncedEvent::Chmod(path) => {
self.path_in_here(&path)?;
let file = self.find_file_with_path(&path)?;
file.reload_meta()?;
},
DebouncedEvent::Remove(path) => {
self.path_in_here(&path)?;
let file = self.find_file_with_path(&path)?.clone();
self.files.remove_item(&file);
},
DebouncedEvent::Rename(old_path, new_path) => {
self.path_in_here(&new_path)?;
let mut file = self.find_file_with_path(&old_path)?;
file.name = new_path.file_name()?.to_string_lossy().to_string();
file.path = new_path.into();
file.reload_meta()?;
},
DebouncedEvent::Error(err, _path) => {
// Never seen this happen. Should reload affected dirs
HError::log::<()>(&format!("{}", err))?;
},
_ => {},
pub fn ready_to_refresh(&self) -> HResult<bool> {
let pending = self.pending_events.read()?.len();
let running = self.refresh.is_some();
Ok(pending > 0 && !running)
}
pub fn get_refresh(&mut self) -> HResult<Option<RefreshPackage>> {
if let Some(mut refresh) = self.refresh.take() {
if refresh.is_ready() {
refresh.pull_async()?;
let mut refresh = refresh.value?;
self.files = refresh.new_files.take()?;
return Ok(Some(refresh));
} else {
self.refresh.replace(refresh);
}
}
self.set_dirty();
return Ok(None)
}
pub fn process_fs_events(&mut self,
buffer: Vec<String>,
sender: Sender<Events>,
render_fn: impl Fn(&File) -> String + Send + 'static)
-> HResult<()> {
let pending = self.pending_events.read()?.len();
if pending > 0 {
let pending = if pending >= 1000 {
1000
} else {
pending
};
let events = self.pending_events
.write()?
.drain(0..pending)
.collect::<Vec<_>>();
let files = self.clone();
let mut refresh = Async::new(move |_| {
let refresh = RefreshPackage::new(files,
buffer,
events,
render_fn);
Ok(refresh)
});
refresh.on_ready(move |_,_| {
Ok(sender.send(Events::WidgetReady)?)
})?;
refresh.run()?;
self.refresh = Some(refresh);
}
Ok(())
}
@ -465,17 +596,16 @@ impl Files {
}
pub fn find_file_with_path(&mut self, path: &Path) -> Option<&mut File> {
self.files.iter_mut().find(|file| file.path == path)
self.iter_files_mut().find(|file| file.path == path)
}
pub fn meta_all_sync(&mut self) -> HResult<()> {
for file in self.files.iter_mut() {
for file in self.iter_files_mut() {
if !file.meta_processed {
file.meta_sync().log();
}
}
self.set_dirty();
self.meta_updated = true;
Ok(())
}
@ -508,10 +638,10 @@ impl Files {
.filter(|f| !(!show_hidden && f.name.starts_with(".")))
.take(meta_files) {
if !file.meta_processed {
file.take_meta(&meta_pool, &mut self.meta_updated).ok();
file.take_meta(&meta_pool).ok();
}
if file.is_dir() {
file.take_dirsize(&meta_pool, &mut self.meta_updated).ok();
file.take_dirsize(&meta_pool).ok();
}
}
@ -519,7 +649,7 @@ impl Files {
}
pub fn meta_set_fresh(&self) -> HResult<()> {
self.files.get(0)?.meta.set_fresh()?;
self.iter_files().nth(0)?.meta.set_fresh()?;
Ok(())
}
@ -697,6 +827,12 @@ impl File {
Ok(file)
}
pub fn rename(&mut self, new_path: &Path) -> HResult<()> {
self.name = new_path.file_name()?.to_string_lossy().to_string();
self.path = new_path.into();
Ok(())
}
pub fn meta_sync(&mut self) -> HResult<()> {
let stale = self.meta.get_stale();
let meta = std::fs::metadata(&self.path)?;
@ -754,36 +890,32 @@ impl File {
}
fn take_dirsize(&mut self,
pool: &ThreadPool,
meta_updated: &mut bool) -> HResult<()> {
pool: &ThreadPool) -> HResult<()> {
let dirsize = self.dirsize.as_mut()?;
if let Ok(_) = dirsize.value { return Ok(()) }
if !dirsize.is_running() {
if !dirsize.is_running() && !dirsize.is_ready() {
dirsize.run_pooled(Some(&*pool))?;
}
if dirsize.is_ready() {
dirsize.pull_async()?;
*meta_updated = true;
}
Ok(())
}
pub fn take_meta(&mut self,
pool: &ThreadPool,
meta_updated: &mut bool) -> HResult<()> {
pool: &ThreadPool) -> HResult<()> {
if self.meta_processed { return Ok(()) }
if !self.meta.is_running() {
if !self.meta.is_running() && !self.meta.is_ready() {
self.meta.run_pooled(Some(&*pool))?;
}
if self.meta.is_ready() {
self.meta.pull_async()?;
self.process_meta()?;
*meta_updated = true;
}
Ok(())
@ -857,10 +989,11 @@ impl File {
mime
} else {
// Fix crash in tree_magic when called on non-regular file
// Also fix crash when a file doesn't exist any more
self.meta()
.ok()
.and_then(|meta| {
if meta.is_file() {
if meta.is_file() && self.path.exists() {
let mime = tree_magic::from_filepath(&self.path);
mime::Mime::from_str(&mime).ok()
} else { None }

266
src/fscache.rs
View File

@ -2,7 +2,7 @@ use notify::{RecommendedWatcher, Watcher, DebouncedEvent, RecursiveMode};
use async_value::{Async, Stale};
use std::sync::{Arc, RwLock};
use std::sync::{Arc, RwLock, Weak};
use std::sync::mpsc::{channel, Sender, Receiver};
use std::collections::{HashMap, HashSet};
use std::time::Duration;
@ -12,6 +12,8 @@ use crate::files::{Files, File, SortBy};
use crate::widget::Events;
use crate::fail::{HResult, HError, ErrorLog, Backtrace, ArcBacktrace};
pub type CachedFiles = (Option<File>, Async<Files>);
#[derive(Debug, Clone)]
pub struct DirSettings {
@ -64,6 +66,63 @@ impl std::fmt::Debug for FsCache {
}
}
#[derive(Clone)]
struct FsEventDispatcher {
targets: Arc<RwLock<HashMap<File, Vec<Weak<RwLock<Vec<FsEvent>>>>>>>
}
impl FsEventDispatcher {
fn new() -> Self {
FsEventDispatcher {
targets: Arc::new(RwLock::new(HashMap::new()))
}
}
fn add_target(&self,
dir: &File,
target: &Arc<RwLock<Vec<FsEvent>>>) -> HResult<()> {
let target = Arc::downgrade(target);
self.targets
.write()
.map(|mut targets| {
match targets.get_mut(dir) {
Some(targets) => targets.push(target),
None => { targets.insert(dir.clone(), vec![target]); }
}
})?;
Ok(())
}
fn remove_target(&self, dir: &File) -> HResult<()> {
self.targets
.write()?
.get_mut(dir)
.map(|targets| {
targets.retain(|t| t.upgrade().is_some());
});
Ok(())
}
fn dispatch(&self, events: HashMap<File, Vec<FsEvent>>) -> HResult<()> {
for (dir, events) in events {
for target_dirs in self.targets
.read()?
.get(&dir) {
for target in target_dirs {
if let Some(target) = target.upgrade() {
let events = events.clone();
target.write()?.extend(events)
}
}
}
}
Ok(())
}
// fn remove_unnecessary
}
#[derive(Clone)]
pub struct FsCache {
@ -71,14 +130,14 @@ pub struct FsCache {
pub tab_settings: Arc<RwLock<HashMap<File, TabSettings>>>,
watched_dirs: Arc<RwLock<HashSet<File>>>,
watcher: Arc<RwLock<RecommendedWatcher>>,
pub fs_changes: Arc<RwLock<Vec<(File, Option<File>, Option<File>)>>>,
fs_event_dispatcher: FsEventDispatcher
}
impl FsCache {
pub fn new(sender: Sender<Events>) -> FsCache {
let (tx_fs_event, rx_fs_event) = channel();
let watcher = RecommendedWatcher::new(tx_fs_event,
Duration::from_secs(2)).unwrap();
Duration::from_secs(2)).unwrap();
let fs_cache = FsCache {
@ -86,12 +145,11 @@ impl FsCache {
tab_settings: Arc::new(RwLock::new(HashMap::new())),
watched_dirs: Arc::new(RwLock::new(HashSet::new())),
watcher: Arc::new(RwLock::new(watcher)),
fs_changes: Arc::new(RwLock::new(vec![])),
fs_event_dispatcher: FsEventDispatcher::new()
};
watch_fs(rx_fs_event,
fs_cache.files.clone(),
fs_cache.fs_changes.clone(),
fs_cache.fs_event_dispatcher.clone(),
sender);
fs_cache
@ -104,8 +162,6 @@ impl FsCache {
}
}
pub type CachedFiles = (Option<File>, Async<Files>);
impl FsCache {
pub fn get_files(&self, dir: &File, stale: Stale) -> HResult<CachedFiles> {
if self.files.read()?.contains_key(dir) {
@ -117,6 +173,8 @@ impl FsCache {
let files = Async::new(move |_| {
let mut files = Files::new_from_path_cancellable(&dir.path, stale)?;
cache.add_watch(&dir).log();
cache.fs_event_dispatcher.add_target(&dir,
&files.pending_events).log();
FsCache::apply_settingss(&cache, &mut files).ok();
Ok(files)
});
@ -129,7 +187,6 @@ impl FsCache {
let mut files = files.run_sync()?;
FsCache::apply_settingss(&self, &mut files).ok();
let files = FsCache::ensure_not_empty(files)?;
self.add_watch(&dir).log();
Ok(files)
}
@ -144,28 +201,6 @@ impl FsCache {
Ok(())
}
pub fn put_files(&self, files: &Files, selection: Option<File>) -> HResult<()> {
let dir = files.directory.clone();
let tab_settings = FsCache::extract_tab_settings(&files, selection);
self.tab_settings.write()?.insert(dir.clone(), tab_settings);
// let mut file_cache = self.files.write()?;
// if file_cache.contains_key(&files.directory) {
// if files.meta_updated {
// let mut files = files.clone();
// files.meta_updated = false;
// file_cache.insert(dir, files);
// }
// } else {
// file_cache.insert(dir, files.clone());
// }
Ok(())
}
pub fn is_cached(&self, dir: &File) -> HResult<bool> {
Ok(self.files.read()?.contains_key(dir))
}
@ -298,65 +333,134 @@ impl FsCache {
}
#[derive(PartialEq, Eq, Hash, Clone, Debug)]
pub enum FsEvent {
Create(File),
Change(File),
Rename(File, File),
Remove(File)
}
impl FsEvent {
pub fn file(&self) -> &File {
use FsEvent::*;
match self {
Create(event_file) |
Change(event_file) |
Remove(event_file) |
Rename(_, event_file) => &event_file
}
}
pub fn for_file(&self, file: &File) -> bool {
use FsEvent::*;
match self {
Create(event_file) |
Change(event_file) |
Remove(event_file) |
Rename(_, event_file) => event_file.path == file.path
}
}
}
use std::convert::TryFrom;
impl TryFrom<DebouncedEvent> for FsEvent {
type Error = HError;
fn try_from(event: DebouncedEvent) -> HResult<Self> {
let event = match event {
DebouncedEvent::Create(path)
=> FsEvent::Create(File::new_from_path(&path, None)?),
DebouncedEvent::Remove(path)
=> FsEvent::Remove(File::new_from_path(&path, None)?),
DebouncedEvent::Write(path) |
DebouncedEvent::Chmod(path)
=> FsEvent::Change(File::new_from_path(&path, None)?),
DebouncedEvent::Rename(old_path, new_path)
=> FsEvent::Rename(File::new_from_path(&old_path, None)?,
File::new_from_path(&new_path, None)?),
DebouncedEvent::Error(err, path)
=> Err(HError::INotifyError(format!("{}, {:?}", err, path),
Backtrace::new_arced()))?,
DebouncedEvent::Rescan
=> Err(HError::INotifyError("Need to rescan".to_string(),
Backtrace::new_arced()))?,
// Ignore NoticeRemove/NoticeWrite
_ => None?,
};
Ok(event)
}
}
fn watch_fs(rx_fs_events: Receiver<DebouncedEvent>,
fs_cache: Arc<RwLock<HashMap<File, Files>>>,
fs_changes: Arc<RwLock<Vec<(File, Option<File>, Option<File>)>>>,
fs_event_dispatcher: FsEventDispatcher,
sender: Sender<Events>) {
std::thread::spawn(move || -> HResult<()> {
for event in rx_fs_events.iter() {
apply_event(&fs_cache, &fs_changes, event).log();
let transform_event =
move |event: DebouncedEvent| -> HResult<(File, FsEvent)> {
let path = event.get_source_path()?;
let dirpath = path.parent()
.map(|path| path)
.unwrap_or(std::path::Path::new("/"));
let dir = File::new_from_path(&dirpath, None)?;
let event = FsEvent::try_from(event)?;
Ok((dir, event))
};
sender.send(Events::WidgetReady).ok();
let collect_events =
move || -> HResult<HashMap<File, Vec<FsEvent>>> {
let event = loop {
use DebouncedEvent::*;
let event = rx_fs_events.recv()?;
match event {
NoticeWrite(_) => continue,
NoticeRemove(_) => continue,
_ => break std::iter::once(event)
}
};
// Wait a bit to batch up more events
std::thread::sleep(std::time::Duration::from_millis(100));
// Batch up all other remaining events received so far
let events = event.chain(rx_fs_events.try_iter())
.map(transform_event)
.flatten()
.fold(HashMap::with_capacity(1000), |mut events, (dir, event)| {
events.entry(dir)
.or_insert(vec![])
.push(event);
events
});
Ok(events)
};
let dispatch_events =
move |events| -> HResult<()> {
fs_event_dispatcher.dispatch(events)?;
sender.send(Events::WidgetReady)?;
Ok(())
};
loop {
if let Ok(events) = collect_events().log_and() {
dispatch_events(events).log();
}
}
Ok(())
});
}
fn apply_event(_fs_cache: &Arc<RwLock<HashMap<File, Files>>>,
fs_changes: &Arc<RwLock<Vec<(File, Option<File>, Option<File>)>>>,
event: DebouncedEvent)
-> HResult<()> {
let path = &event.get_source_path()?;
let dirpath = path.parent()
.map(|path| path.to_path_buf())
.unwrap_or_else(|| PathBuf::from("/"));
let dir = File::new_from_path(&dirpath, None)?;
let old_file = File::new_from_path(&path, None)?;
let mut new_file = match event {
DebouncedEvent::Remove(_) => None,
_ => Some(File::new_from_path(&path, None)?)
};
new_file.as_mut().map(|file| file.meta_sync());
fs_changes.write()?.push((dir,
Some(old_file),
new_file));
// for dir in fs_cache.write()?.values_mut() {
// if dir.path_in_here(&path).unwrap_or(false) {
// let old_file = dir.find_file_with_path(&path).cloned();
// let dirty_meta = old_file
// .as_ref()
// .map(|f| f.dirty_meta.clone())
// .unwrap_or(None);
// let mut new_file = match event {
// DebouncedEvent::Remove(_) => None,
// _ => Some(File::new_from_path(&path, dirty_meta)?)
// };
// new_file.as_mut().map(|file| file.meta_sync());
// dir.replace_file(old_file.as_ref(), new_file.clone()).log();
// fs_changes.write()?.push((dir.directory.clone(),
// old_file,
// new_file));
// }
// }
Ok(())
}
trait PathFromEvent {
fn get_source_path(&self) -> HResult<&PathBuf>;

173
src/listview.rs
View File

@ -97,6 +97,8 @@ impl Listable for ListView<Files> {
self.content.meta_upto(visible_files, Some(sender.clone()));
self.refresh_files().log();
if self.content.is_dirty() {
self.content.set_clean();
self.core.set_dirty();
@ -239,7 +241,7 @@ impl ListView<Files>
// Work around annoying restriction until polonius borrow checker becomes default
// Since we only ever return one mutable borrow this is perfectly safe
// Since only ever one mutable borrow is returned this is perfectly safe
// See also: https://github.com/rust-lang/rust/issues/21906
match file {
Some(file) => unsafe { return file.as_mut().unwrap() },
@ -581,77 +583,109 @@ impl ListView<Files>
}
fn render_line(&self, file: &File) -> String {
let icon = if self.core.config().icons {
file.icon()
} else { "" };
let name = String::from(icon) + &file.name;
let (size, unit) = file.calculate_size().unwrap_or((0, "".to_string()));
let tag = match file.is_tagged() {
Ok(true) => term::color_red() + "*",
_ => "".to_string()
};
let tag_len = if tag != "" { 1 } else { 0 };
let selection_gap = " ".to_string();
let (name, selection_color) = if file.is_selected() {
(selection_gap + &name, crate::term::color_yellow())
} else { (name.clone(), "".to_string()) };
let (link_indicator, link_indicator_len) = if file.target.is_some() {
(format!("{}{}{}",
term::color_yellow(),
"--> ".to_string(),
term::highlight_color()),
4)
} else { ("".to_string(), 0) };
let xsize = self.get_coordinates().unwrap().xsize();
let sized_string = term::sized_string(&name, xsize);
let size_pos = xsize - (size.to_string().len() as u16
+ unit.to_string().len() as u16
+ link_indicator_len);
let padding = sized_string.len() - sized_string.width_cjk();
let padding = xsize - padding as u16;
let padding = padding - tag_len;
format!(
"{}{}{}{}{}{}{}{}",
termion::cursor::Save,
match &file.color {
Some(color) => format!("{}{}{}{:padding$}{}",
tag,
term::from_lscolor(color),
selection_color,
&sized_string,
term::normal_color(),
padding = padding as usize),
_ => format!("{}{}{}{:padding$}{}",
tag,
term::normal_color(),
selection_color,
&sized_string,
term::normal_color(),
padding = padding as usize),
} ,
termion::cursor::Restore,
termion::cursor::Right(size_pos),
link_indicator,
term::highlight_color(),
size,
unit
)
let render_fn = self.render_line_fn();
render_fn(file)
}
#[allow(trivial_bounds)]
fn render_line_fn(&self) -> impl Fn(&File) -> String {
let xsize = self.get_coordinates().unwrap().xsize();
let icons = self.core.config().icons;
move |file| -> String {
let icon = if icons {
file.icon()
} else { "" };
let name = String::from(icon) + &file.name;
let (size, unit) = file.calculate_size().unwrap_or((0, "".to_string()));
let tag = match file.is_tagged() {
Ok(true) => term::color_red() + "*",
_ => "".to_string()
};
let tag_len = if tag != "" { 1 } else { 0 };
let selection_gap = " ".to_string();
let (name, selection_color) = if file.is_selected() {
(selection_gap + &name, crate::term::color_yellow())
} else { (name.clone(), "".to_string()) };
let (link_indicator, link_indicator_len) = if file.target.is_some() {
(format!("{}{}{}",
term::color_yellow(),
"--> ".to_string(),
term::highlight_color()),
4)
} else { ("".to_string(), 0) };
let sized_string = term::sized_string(&name, xsize);
let size_pos = xsize - (size.to_string().len() as u16
+ unit.to_string().len() as u16
+ link_indicator_len);
let padding = sized_string.len() - sized_string.width_cjk();
let padding = xsize - padding as u16;
let padding = padding - tag_len;
format!(
"{}{}{}{}{}{}{}{}",
termion::cursor::Save,
match &file.color {
Some(color) => format!("{}{}{}{:padding$}{}",
tag,
term::from_lscolor(color),
selection_color,
&sized_string,
term::normal_color(),
padding = padding as usize),
_ => format!("{}{}{}{:padding$}{}",
tag,
term::normal_color(),
selection_color,
&sized_string,
term::normal_color(),
padding = padding as usize),
} ,
termion::cursor::Restore,
termion::cursor::Right(size_pos),
link_indicator,
term::highlight_color(),
size,
unit
)
}
}
fn render(&self) -> Vec<String> {
self.content
.iter_files()
.map(|file| self.render_line(file))
.collect()
}
fn refresh_files(&mut self) -> HResult<()> {
if let Ok(Some(mut refresh)) = self.content.get_refresh() {
let file = self.clone_selected_file();
self.buffer = refresh.new_buffer.take()?;
self.lines = self.buffer.len() - 1;
self.select_file(&file);
}
if self.content.ready_to_refresh()? {
let render_fn = self.render_line_fn();
self.content.process_fs_events(self.buffer.clone(),
self.core.get_sender(),
render_fn)?;
}
Ok(())
}
}
@ -664,13 +698,16 @@ impl<T> Widget for ListView<T> where ListView<T>: Listable {
}
fn refresh(&mut self) -> HResult<()> {
self.on_refresh().log();
self.lines = self.len();
if self.selection >= self.len() && self.selection != 0 {
self.selection = self.len() - 1;
let buffer_len = self.buffer.len();
self.lines = buffer_len;
if self.selection >= self.buffer.len() && self.buffer.len() != 0 {
self.selection = self.buffer.len() - 1;
}
if self.core.is_dirty() || self.buffer.len() != self.len() {
if self.core.is_dirty() || buffer_len != self.len() {
self.buffer = self.render();
self.core.set_clean();
}

2
src/main.rs
View File

@ -27,6 +27,8 @@ extern crate clap;
extern crate strum;
#[macro_use]
extern crate strum_macros;
#[macro_use]
extern crate derivative;
extern crate osstrtools;
extern crate pathbuftools;

17
src/preview.rs
View File

@ -324,23 +324,6 @@ impl Previewer {
}
}
pub fn replace_file(&mut self, dir: &File,
old: Option<&File>,
new: Option<&File>) -> HResult<()> {
if self.file.as_ref() != Some(dir) { return Ok(()) }
self.widget.widget_mut().map(|widget| {
match widget {
PreviewWidget::FileList(filelist) => {
filelist.content.replace_file(old, new.cloned()).map(|_| {
filelist.refresh().ok();
}).ok();
}
_ => {}
}
})
}
pub fn put_preview_files(&mut self, files: Files) {
let core = self.core.clone();
let dir = files.directory.clone();