looking good

Cargo.lock

@@ -641,6 +641,7 @@ dependencies = [
  "gstreamer-player 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "gstreamer-video 0.14.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "image 0.21.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "is_utf8 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "itertools 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -721,6 +722,11 @@ dependencies = [
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "is_utf8"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "itertools"
 version = "0.8.2"
@@ -1808,6 +1814,7 @@ dependencies = [
 "checksum inotify 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "24e40d6fd5d64e2082e0c796495c8ef5ad667a96d03e5aaa0becfd9d47bcbfb8"
 "checksum inotify-sys 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e74a1aa87c59aeff6ef2cc2fa62d41bc43f54952f55652656b18a02fd5e356c0"
 "checksum iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e"
+"checksum is_utf8 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "30b4274c37cc88a8e26c12c331819b20e6d1edf5838f0e216487c418c211638b"
 "checksum itertools 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484"
 "checksum jpeg-decoder 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "0256f0aec7352539102a9efbcb75543227b7ab1117e0f95450023af730128451"
 "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"

Cargo.toml

@@ -49,6 +49,7 @@ crossbeam = "0.7"
 bumpalo = { versioon = "*", features = [ "collections" ] }
 splay_tree = { path = "../splay_tree" }
 dmsort = "0.1.3"
+is_utf8 = { version = "0.1.4", target_feature = [ "avx", "avx2" ] }
 
 image = { version = "0.21.1", optional = true }
 gstreamer = { version = "0.14", optional = true }

src/file_browser.rs

@@ -673,7 +673,6 @@ impl FileBrowser {
     }
 
     pub fn update_preview(&mut self) -> HResult<()> {
-        return Ok(());
         if !self.main_async_widget_mut()?.ready() { return Ok(()) }
         if self.main_widget()?
             .content
@@ -1383,9 +1382,9 @@ impl FileBrowser {
         let count_xpos = xsize - file_count.len() as u16;
         let count_ypos = ypos + self.get_coordinates()?.ysize();
 
-        //let fs = self.fs_stat.read()?.find_fs(&file.path)?.clone();
+        let fs = self.fs_stat.read()?.find_fs(&file.path)?.clone();
 
-        let dev = String::from("");
+        let dev = fs.get_dev().unwrap_or(String::from(""));
         let free_space = 0;
         let total_space = 0;
         let space = format!("{}{} / {}",

src/files.rs

@@ -1,10 +1,9 @@
-#![feature(vec_into_raw_parts)]
-
 use std::cmp::Ord;
 use std::collections::{HashMap, HashSet};
 use std::ops::Index;
 use std::fs::Metadata;
-use std::os::unix::fs::MetadataExt;
+use std::os::unix::{fs::MetadataExt,
+                    ffi::OsStrExt};
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, RwLock, Mutex};
 use std::sync::mpsc::Sender;
@@ -12,8 +11,7 @@ use std::hash::{Hash, Hasher};
 use std::str::FromStr;
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
 use std::ffi::OsStr;
-use std::cell::RefCell;
+use std::default::Default;
-
 
 use failure;
 use failure::Fail;
@@ -28,20 +26,21 @@ use failure::Error;
 use rayon::{ThreadPool, ThreadPoolBuilder};
 use natord::compare;
 use mime_guess;
-use rayon::prelude::*;
 use nix::{dir::*,
           fcntl::OFlag,
           sys::stat::Mode};
+use bumpalo::Bump;
 
 use pathbuftools::PathBufTools;
 use async_value::{Async, Stale, StopIter};
-use bumpalo::Bump;
+
 
 use crate::fail::{HResult, HError, ErrorLog};
 use crate::dirty::{DirtyBit, Dirtyable};
 use crate::widget::Events;
 use crate::icon::Icons;
 use crate::fscache::{FsCache, FsEvent};
+use crate::iter::{FileIter, FileIterMut};
 
 lazy_static! {
     static ref COLORS: LsColors = LsColors::from_env().unwrap_or_default();
@@ -297,17 +296,13 @@ pub type Job = (PathBuf,
                 Option<Arc<RwLock<Option<Metadata>>>>,
                 Option<Arc<(AtomicBool, AtomicUsize)>>);
 
-// unsafe impl Sync for Files {}
-// unsafe impl Send for Files {}
-// unsafe impl Sync for File {}
-// unsafe impl Send for File {}
-
 #[derive(Derivative)]
 #[derivative(PartialEq, Eq, Hash, Clone, Debug)]
 pub struct Files {
     pub directory: File,
     pub files: SplaySet<File>,
     pub len: usize,
+    pub current_raw_pos: usize,
     #[derivative(Debug="ignore")]
     #[derivative(PartialEq="ignore")]
     #[derivative(Hash="ignore")]
@@ -364,13 +359,14 @@ impl Dirtyable for Files {
     }
 }
 
-use std::default::Default;
+
 
 impl Default for Files {
     fn default() -> Files {
         Files {
             directory: File::new_placeholder(Path::new("")).unwrap(),
             files: SplaySet::new(),
+            current_raw_pos: 0,
             len: 0,
             pending_events: Arc::new(RwLock::new(vec![])),
             refresh: None,
@@ -396,7 +392,7 @@ impl Drop for Files {
         self.stale
             .as_ref()
             .map(|s| s.set_stale());
-    }
+   }
 }
 
 
@@ -470,11 +466,11 @@ pub fn from_getdents(fd: i32,
     use libc::SYS_getdents64;
 
     // Buffer size was chosen after measuring different sizes and 4k seemed best
-    const BUFFER_SIZE: usize = 1024 * 1024 * 2 * 4;
+    const BUFFER_SIZE: usize = 1024 * 1024 * 4;
-    const BUFFER_SUBSLICE_SIZE: usize = BUFFER_SIZE / 4;
+
+    // To hold temporary allocations only necessary for this function call
+    let tmpalloc = &Allocator::new();
 
-    // Abuse Vec<u8> as byte buffer
-    let mut buf: Vec<u8> = vec![0; BUFFER_SIZE];
 
     let path_ostr = path.as_os_str();
     let path_len = path_ostr.len();
@@ -485,17 +481,13 @@ pub fn from_getdents(fd: i32,
         let files = &files;
 
         loop {
-            // let (buf, localfiles, used) = get_buffers();
+            let buf = tmpalloc.dentbuf(BUFFER_SIZE) as usize;
-            let bufptr = buf.as_mut_ptr() as usize;
-            // let mut localfiles: &mut Vec<_> = localfiles;
-
-            //dbg!(&bufptr);
 
             // Returns number of bytes written to buffer
             let nread = unsafe { libc::syscall(SYS_getdents64,
                                                fd,
-                                               bufptr,
+                                               buf,
-                                               BUFFER_SUBSLICE_SIZE) };
+                                               BUFFER_SIZE) };
 
             // 0 means done, -1 means an error happened
             if dbg!(nread) == 0 {
@@ -507,19 +499,14 @@ pub fn from_getdents(fd: i32,
                 return;
             }
 
-
+            s.spawn(move |_| {
-            // // Clone buffer for parallel processing in another thread
-            let mut buf: Vec<u8> = Vec::from(&buf[..nread as usize + 1]);
-
-            let files = s.spawn(move |_| {
-                //let bump = std::sync::Mutex::new(Bump::new());
                 // Rough approximation of the number of entries. Actual
                 // size changes from entry to entry due to variable string
                 // size.
                 let cap = nread as usize / std::mem::size_of::<linux_dirent>();
                 // Use a local Vec to avoid contention on Mutex
-                let mut localfiles = Vec::with_capacity(dbg!(cap));
+                let mut tmpfiles = tmpalloc.tmpfiles(cap);
-                let bufptr = buf.as_mut_ptr() as usize;
+
 
                 let mut bpos: usize = 0;
 
@@ -531,43 +518,29 @@ pub fn from_getdents(fd: i32,
                     // long as the kernel doesn't provide wrong values and
                     // the calculations are corrent this is safe to do.
                     // It's bascally (buffer[n] -> buffer[n + len(buffer[n])
+                    let buf = buf as *mut u8;
                     let d: &linux_dirent = unsafe {
-                        std::mem::transmute::<usize, &linux_dirent>(bufptr  + bpos )
+                        buf.add(bpos).cast::<linux_dirent>().as_ref().unwrap()
                     };
 
-                    // Name lenegth is overallocated, true length can be found by checking with strlen
-                    let name_len = d.d_reclen as usize -
-                        std::mem::size_of::<u64>() -
-                        std::mem::size_of::<u64>() -
-                        std::mem::size_of::<u16>() -
-                        std::mem::size_of::<u8>();
-
-                    // OOB!!!
-                    if bpos + name_len > BUFFER_SIZE {
-                        panic!("LOLWUT");
-                        HError::log::<()>(&format!("WARNING: Name for file was out of bounds in: {}",
-                                                   path.to_string_lossy())).ok();
-                        return Err(FileError::GetDents(path.to_string_lossy().to_string()));
-                    }
-
                     // Add length of current dirent to the current offset
                     // tbuffer[n] -> buffer[n + len(buffer[n])
                     bpos = bpos + d.d_reclen as usize;
 
-                    let (name_ostr, name_bytes, name_len): (&OsStr, &[u8], usize) = {
+                    let (name_ostr, name_bytes, name_len): (&OsStr, &[u8], usize) = unsafe {
                         // Safe as long as d_name is NULL terminated
-                        let true_len = unsafe { libc::strlen(d.d_name.as_ptr() as *const i8) };
+                        let true_len = libc::strlen(d.d_name.as_ptr() as *const i8);
-                        // Safe if strlen returned without SEGFAULT on OOB (if d_name weren't NULL terminated)
-                        let bytes: &[u8] = unsafe { std::slice::from_raw_parts(d.d_name.as_ptr() as *const u8,
-                                                                               true_len) };
 
-                        // Don't want this
+                        // Safe if strlen returned without SEGFAULT on OOB (if d_name weren't NULL terminated)
+                        let bytes: &[u8] = std::slice::from_raw_parts(d.d_name.as_ptr(), true_len);
+
+                        // Don't want this crap
                         if true_len == 0  || bytes == b"." || bytes == b".." {
                             continue;
                         }
 
                         // A bit sketchy maybe, but if all checks passed, should be OK.
-                        (unsafe { std::mem::transmute::<&[u8], &OsStr>(bytes) },
+                        (OsStr::from_bytes(bytes),
                          bytes,
                          true_len)
                     };
@@ -624,7 +597,7 @@ pub fn from_getdents(fd: i32,
                     // Avoid reallocation on push
                     let path = {
                         let pathlen = path_len;
-                        let totallen = pathlen + name_len + 2;
+                        let totallen = pathlen + name_len + 1;
 
                         let mut path = alloc.pathbuf(totallen);
                         path.write(path_ostr.as_bytes());
@@ -659,12 +632,12 @@ pub fn from_getdents(fd: i32,
                         tag: None,
                     };
 
-                    localfiles.push(file)
+                    tmpfiles.push(file)
                 }
 
                 // Now add all files from this worker thread
                 files.lock().map(|mut f| {
-                    f.extend(localfiles);
+                    f.extend(tmpfiles);
                 }).unwrap();
 
                 Ok(())
@@ -672,26 +645,14 @@ pub fn from_getdents(fd: i32,
         }
     });
 
-
-    dbg!(files.lock().unwrap().len());
-
     match result {
         Ok(()) => {
-            // let len = workset.iter().map(|(_, f, _)| f.len()).sum();
-            // let mut files = Vec::with_capacity(len);
-
-            // for i in 0..4 {
-            //     files.par_extend(std::mem::take(&mut workset[i].1));
-            // }
-
             return Ok(std::mem::take(&mut *files.lock().unwrap()))
         }
         Err(_) => Err(FileError::GetDents(path.to_string_lossy().to_string()))
     }
 }
 
-
-
 impl Files {
     // Use getdents64 on Linux
     #[cfg(target_os = "linux")]
@@ -771,7 +732,7 @@ impl Files {
     }
 
     pub fn enqueue_jobs(&mut self, n: usize) {
-        return;
+        use rayon::prelude::*;
         let from = self.meta_upto.unwrap_or(0);
         self.meta_upto = Some(from + n);
 
@@ -780,19 +741,18 @@ impl Files {
             None => return
         };
 
-        // let mut jobs = self.iter_files_mut()
+        let mut jobs = self.iter_files_mut()
-        //                    .collect::<Vec<&mut File>>()
+                           .collect::<Vec<&mut File>>()
-        //                    .into_par_iter()
+                           .into_par_iter()
-        //                    .skip(from)
+                           .skip(from)
-        //                    .take(n)
+                           .take(n)
-        //                    .filter_map(|f| f.prepare_meta_job(&cache))
+                           .filter_map(|f| f.prepare_meta_job(&cache))
-        //                    .collect::<Vec<_>>();
+                           .collect::<Vec<_>>();
 
-        // self.jobs.append(&mut jobs);
+        self.jobs.append(&mut jobs);
     }
 
     pub fn run_jobs(&mut self, sender: Sender<Events>) {
-        return;
         let jobs = std::mem::take(&mut self.jobs);
         let stale = self.stale
                         .clone()
@@ -841,99 +801,21 @@ impl Files {
         self.len = self.iter_files().count();
     }
 
-    pub fn get_file_mut(&mut self, index: usize) -> Option<&mut File> {
+    pub fn par_iter_files(&self) -> FileIter<'_> {
-        // Need actual length of self.files for this
+        FileIter::new(&self.files,
-        let hidden_in_between = self.files_in_between(index, self.files.len());
+                      Box::new(self.filter_fn()))
-
-        let  file = self.files.as_vec_like_mut().get(index + hidden_in_between);
-        None
     }
 
-    pub fn par_iter_files(&self) -> impl Iterator<Item=&File> {
+    pub fn iter_files(&self) -> FileIter<'_> {
-        let filter_fn = self.filter_fn();
+        FileIter::new(&self.files,
-
+                      Box::new(self.filter_fn()))
-        self.files
-            .as_vec_like()
-            .iter()
-            .filter(move |f| filter_fn(f))
     }
 
-    pub fn iter_files(&self) -> impl Iterator<Item=&File> {
+
+    pub fn iter_files_mut(&mut self) -> FileIterMut<'_> {
         let filter_fn = self.filter_fn();
-
+        FileIterMut::new(&mut self.files,
-        self.files
+                         Box::new(filter_fn))
-            .as_vec_like()
-            .iter()
-            .filter(move |&f| filter_fn(f))
-    }
-
-    #[allow(mutable_transmutes)]
-    pub fn files_in_between(&self, pos: usize, n_before: usize) -> usize {
-        let filter_fn = self.filter_fn();
-
-        unsafe {std::mem::transmute(self.files
-                                    .iter()
-                                    .take(pos)
-                                    .collect::<Vec<_>>()
-                                    .iter()
-                                    .rev()
-                                    .enumerate()
-                                    .filter(|(_, f)| filter_fn(f))
-                                    .take(n_before)
-                                    .map(|(i, _)| i + 1)
-                                    .last()
-                                    .unwrap_or(0)) }
-
-    }
-
-    pub fn iter_files_from(&mut self, from: &File, n_before: usize) -> impl Iterator<Item=&File> {
-        // let fpos = self.find_file(from).unwrap_or(0);
-
-        // let files_in_between = self.files_in_between(fpos, n_before);
-
-        let filter_fn = self.filter_fn();
-
-        use splay_tree::set::SuperIter;
-        self.files
-            .better_iter_from(n_before)
-            .filter(move |f| filter_fn(f))
-    }
-
-    pub fn iter_files_mut_from(&mut self, from: &File, n_before: usize) -> impl Iterator<Item=&mut File> {
-        let fpos = self.find_file(from).unwrap_or(0);
-        let files_in_between = self.files_in_between(fpos, n_before);
-
-        let filter_fn = self.filter_fn();
-
-        self.iter_files_mut()
-            .skip(fpos.saturating_sub(files_in_between))
-            .filter(move |f| filter_fn(f))
-    }
-
-    #[allow(mutable_transmutes)]
-    pub fn iter_files_mut(&mut self) -> impl Iterator<Item=&mut File>  {
-        let filter_fn = self.filter_fn();
-        self.files
-            .iter()
-            .filter(move |f| filter_fn(f))
-            .map(|f| unsafe { std::mem::transmute(f) })
-        //let wut = &mut self;
-
-        // std::iter::from_fn(move || -> Option<&mut File> {
-        //     // match wut.files.take_smallest() {
-        //     //     Some(f) => {
-        //     //         wut.taken.push(f);
-        //     //         wut.taken.last_mut()
-        //     //     }
-        //     //     None => {
-        //     //         for file in std::mem::take(&mut wut.taken).into_iter() {
-        //     //             wut.files.insert(file.clone());
-        //     //         }
-        //     //         None
-        //     //     }
-        //     // }
-        //     None
-        // })
     }
 
     #[allow(trivial_bounds)]
@@ -1152,10 +1034,7 @@ impl Files {
     }
 
     pub fn find_file(&mut self, file: &File) -> Option<usize> {
-        let comp = self.sorter();
         let pos = self.files.as_vec_like_mut().find_index(file)?;
-            //.// binary_search_by(|probe| comp(probe, file))
-            // .ok()?;
 
         debug_assert_eq!(file.path, self[pos].path);
 
@@ -1233,14 +1112,9 @@ pub enum SortBy {
 }
 
 
-use std::os::unix::ffi::OsStrExt;
 impl PartialEq for File {
     fn eq(&self, other: &File) -> bool {
-        if unsafe { self.path.as_os_str().as_bytes() == other.path.as_os_str().as_bytes() } {
+        self.path == other.path
-            true
-        } else {
-            false
-        }
     }
 }
 
@@ -1676,8 +1550,6 @@ impl File {
         let state = self.tag.unwrap();
 
         std::thread::spawn(move || -> HResult<()> {
-            use std::os::unix::ffi::OsStrExt;
-
             let tagfile_path = crate::paths::tagfile_path()?;
             let mut tags = TAGS.write()?;
 

src/listview.rs

@@ -3,7 +3,6 @@ use std::path::PathBuf;
 
 use termion::event::Key;
 use unicode_width::UnicodeWidthStr;
-use rayon::prelude::*;
 
 use async_value::Stale;
 
@@ -111,13 +110,13 @@ impl Listable for ListView<Files> {
         let meta_upto = self.content.meta_upto.unwrap_or(0);
         let ysize = self.core.coordinates.ysize_u();
 
-        // if  self.offset + ysize >= meta_upto {
+        if  self.offset + ysize >= meta_upto {
-        //     let sender = self.core.get_sender();
+            let sender = self.core.get_sender();
-        //     let njobs = self.offset + ysize;
+            let njobs = self.offset + ysize;
 
-        //     self.content.enqueue_jobs(njobs);
+            self.content.enqueue_jobs(njobs);
-        //     self.content.run_jobs(sender);
+            self.content.run_jobs(sender);
-        // }
+        }
 
         // self.refresh_files().log();
 
@@ -347,8 +346,8 @@ impl FileListBuilder {
         self.cache.map(|c| view.content.cache = Some(c));
         view.content.set_clean();
         view.core.set_clean();
-        let  len = view.content.len();
+        // let  len = view.content.len();
-        view.content.meta_upto = Some(len);
+        // view.content.meta_upto = Some(len);
 
         crate::files::stop_ticking();
 
@@ -364,28 +363,36 @@ impl ListView<Files>
         FileListBuilder::new(core, source)
     }
 
-    pub fn update_selected_file(&mut self, oldpos: usize) {
+    pub fn update_selected_file(&mut self, oldsel: usize) {
-        let newpos = self.get_selection();
+        let newsel = self.get_selection();
+
         let skip =
-            match newpos > oldpos {
+            match newsel > oldsel {
-                true => newpos - oldpos,
+                true => newsel - oldsel,
                 false => 0
             };
 
         let seek_back =
-            match newpos < oldpos {
+            match newsel < oldsel {
-                true => oldpos - newpos,
+                true => oldsel - newsel,
                 false => 0
             };
 
-        let sel = self.selected_file().clone();
+        let oldfile = self.selected_file().clone();
+        let fpos = self.content.find_file(&oldfile).unwrap_or(0);
 
         let file = self.content
-                       .iter_files_from(&sel, seek_back)
+                       .iter_files()
-                       .skip(skip)
+                       .set_raw_pos(fpos)
-                       .nth(0);
+                       .seek_back(seek_back)
+                       .nth(skip)
+                       .unwrap()
+                       .clone();
 
-        self.current_item = file.cloned();
+        let new_fpos = self.content.find_file(&file).unwrap_or(0);
+        self.content.current_raw_pos = new_fpos;
+
+        self.current_item = Some(file);
     }
 
     pub fn selected_file(&self) -> &File {
@@ -396,10 +403,11 @@ impl ListView<Files>
     }
 
     pub fn selected_file_mut(&mut self) -> &mut File {
-        let selected_file = self.selected_file().clone();
+        let raw_pos = self.content.current_raw_pos;
 
         let file = self.content
-            .iter_files_mut_from(&selected_file, 0)
+            .iter_files_mut()
+            .set_raw_pos(raw_pos)
             .nth(0)
             .map(|f| f as *mut File);
 
@@ -914,25 +922,19 @@ impl ListView<Files>
         }
     }
 
-    #[allow(mutable_transmutes)]
     fn render(&self) -> Vec<String> {
         let render_fn = self.render_line_fn();
         let ysize = self.get_coordinates().unwrap().ysize_u();
-        // let files_above_selection = self.get_selection() - self.offset;
+        let files_above_selection = self.get_selection() - self.offset;
-        // let selected_file = self.selected_file();
+        let current_raw_pos = self.content.current_raw_pos;
 
-        use splay_tree::set::SuperIter;
+        self.content
-
+            .iter_files()
-        unsafe {
+            .set_raw_pos(current_raw_pos)
-            std::mem::transmute::<&Self, &mut Self>(self)
+            .seek_back(files_above_selection)
-                .content
+            .take(ysize+1)
-                .files
+            .map(|file| render_fn(file))
-                .better_iter_from(self.offset)
+            .collect()
-                //.inspect(|f| { dbg!(&f.name); })
-                .take(ysize+1)
-                .map(|file| render_fn(file))
-                .collect()
-        }
     }
 
     fn refresh_files(&mut self) -> HResult<()> {

src/main.rs

@@ -36,6 +36,7 @@ extern crate crossbeam;
 extern crate bumpalo;
 extern crate splay_tree;
 extern crate dmsort;
+extern crate is_utf8;
 
 extern crate osstrtools;
 extern crate pathbuftools;
@@ -75,7 +76,7 @@ mod imgview;
 mod mediaview;
 mod keybind;
 mod alloc;
-
+mod iter;
 
 
 

src/term.rs

@@ -239,7 +239,7 @@ pub fn sized_string(string: &str, xsize: u16) -> &str {
                     .last()
                     .unwrap_or(0);
 
-    string
+    &string[0..len]
 }
 
 #[derive(Debug)]