mirror of https://github.com/bobwen-dev/hunter
another day, another boost: 30% faster on linux with getdents64
This commit is contained in:
rabite
parent
f10f7d5e32
commit
963e0ae1cb
|
|
@ -214,6 +214,27 @@ name = "constant_time_eq"
|
|||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-channel 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-queue 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-channel"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.7.2"
|
||||
|
|
@ -594,6 +615,7 @@ dependencies = [
|
|||
"base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"chrono 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"derivative 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"dirs-2 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
|
|
@ -1716,6 +1738,8 @@ dependencies = [
|
|||
"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
|
||||
"checksum color_quant 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0dbbb57365263e881e805dc77d94697c9118fd94d8da011240555aa7b23445bd"
|
||||
"checksum constant_time_eq 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
|
||||
"checksum crossbeam 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "69323bff1fb41c635347b8ead484a5ca6c3f11914d784170b158d8449ab07f8e"
|
||||
"checksum crossbeam-channel 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "acec9a3b0b3559f15aee4f90746c4e5e293b701c0f7d3925d24e01645267b68c"
|
||||
"checksum crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3aa945d63861bfe624b55d153a39684da1e8c0bc8fba932f7ee3a3c16cea3ca"
|
||||
"checksum crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5064ebdbf05ce3cb95e45c8b086f72263f4166b29b97f6baff7ef7fe047b55ac"
|
||||
"checksum crossbeam-queue 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c695eeca1e7173472a32221542ae469b3e9aac3a4fc81f7696bcad82029493db"
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ derivative = "1.0.3"
|
|||
itertools = "0.8"
|
||||
nix = "0.17"
|
||||
strip-ansi-escapes = "0.1"
|
||||
crossbeam = "0.7"
|
||||
|
||||
|
||||
image = { version = "0.21.1", optional = true }
|
||||
|
|
|
|||
174
src/files.rs
174
src/files.rs
|
|
@ -104,6 +104,8 @@ pub enum FileError {
|
|||
OpenDir(#[cause] nix::Error),
|
||||
#[fail(display = "Couldn't read files! Error: {}", _0)]
|
||||
ReadFiles(#[cause] nix::Error),
|
||||
#[fail(display = "Had problems with getdents64 in directory: {}", _0)]
|
||||
GetDents(String),
|
||||
}
|
||||
|
||||
pub fn get_pool() -> ThreadPool {
|
||||
|
|
@ -383,7 +385,179 @@ impl Drop for Files {
|
|||
}
|
||||
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct linux_dirent {
|
||||
pub d_ino: u64,
|
||||
pub d_off: u64,
|
||||
pub d_reclen: u16,
|
||||
pub d_type: u8,
|
||||
pub d_name: [u8; 0],
|
||||
}
|
||||
|
||||
|
||||
// This arcane spell hastens the target by around 30%.
|
||||
|
||||
// It uses quite a bit of usafe code, mostly to call into libc and
|
||||
// dereference raw pointers inherent to the getdents API, but also to
|
||||
// avoid some of the overhead built into Rust's default conversion
|
||||
// methods. How the getdents64 syscall is intended to be used was
|
||||
// mostly looked up in man 2 getdents64, the nc crate, and the
|
||||
// upcoming version of the walkdir crate, plus random examples here
|
||||
// and there..
|
||||
|
||||
// This should probably be replaced with walkdir when it gets a proper
|
||||
// release with the new additions. nc itself is already too high level
|
||||
// to meet the performance target, unfortunately.
|
||||
#[cfg(target_os = "linux")]
|
||||
pub fn from_getdents(fd: i32, path: &Path, nothidden: &AtomicUsize) -> Vec<File>
|
||||
{
|
||||
use libc::SYS_getdents64;
|
||||
|
||||
// Nice big 4MB buffer
|
||||
const BUFFER_SIZE: usize = 1024 * 1024 * 4;
|
||||
|
||||
let mut buf: Vec<u8> = vec![0; BUFFER_SIZE];
|
||||
let bufptr = buf.as_mut_ptr();
|
||||
let files = std::sync::Mutex::new(vec![]);
|
||||
let files = &files;
|
||||
|
||||
crossbeam::scope(|s| {
|
||||
loop {
|
||||
let nread = unsafe { libc::syscall(SYS_getdents64, fd, bufptr, BUFFER_SIZE) };
|
||||
|
||||
if nread <= 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
// Clone buffer for processing in another thread and fetch more entries
|
||||
let mut buf: Vec<u8> = buf.clone();
|
||||
|
||||
s.spawn(move |_| {
|
||||
let cap = nread as usize / std::mem::size_of::<linux_dirent>();
|
||||
let mut localfiles = Vec::with_capacity(cap);
|
||||
let bufptr = buf.as_mut_ptr();
|
||||
let mut bpos: usize = 0;
|
||||
|
||||
while bpos < nread as usize {
|
||||
let d: &linux_dirent = unsafe { std::mem::transmute(bufptr as usize + bpos as usize) };
|
||||
|
||||
// Name lenegth is overallocated, true length can be found by checking with strlen
|
||||
let name_len = d.d_reclen as usize -
|
||||
std::mem::size_of::<u64>() -
|
||||
std::mem::size_of::<u64>() -
|
||||
std::mem::size_of::<u16>() -
|
||||
std::mem::size_of::<u8>();
|
||||
|
||||
// OOB!!!
|
||||
if bpos + name_len > BUFFER_SIZE {
|
||||
HError::log::<()>(&format!("WARNING: Name for file was out of bounds in: {}",
|
||||
path.to_string_lossy())).ok();
|
||||
return;
|
||||
}
|
||||
|
||||
// Doing it here simplifies skipping
|
||||
bpos = bpos + d.d_reclen as usize;
|
||||
|
||||
let name: &OsStr = {
|
||||
let true_len = unsafe { libc::strlen(d.d_name.as_ptr() as *const i8) };
|
||||
let bytes: &[u8] = unsafe { std::slice::from_raw_parts(d.d_name.as_ptr() as *const u8,
|
||||
true_len) };
|
||||
|
||||
// Don't want this
|
||||
if bytes.len() == 0 || bytes == b"." || bytes == b".." {
|
||||
continue;
|
||||
}
|
||||
|
||||
unsafe { std::mem::transmute(bytes) }
|
||||
};
|
||||
|
||||
|
||||
// Avoid reallocation on push
|
||||
let mut pathstr = std::ffi::OsString::with_capacity(path.as_os_str().len() +
|
||||
name.len() +
|
||||
2);
|
||||
pathstr.push(path.as_os_str());
|
||||
pathstr.push("/");
|
||||
pathstr.push(name);
|
||||
|
||||
let path = PathBuf::from(pathstr);
|
||||
|
||||
let name = name.to_str()
|
||||
.map(|n| String::from(n))
|
||||
.unwrap_or_else(|| name.to_string_lossy().to_string());
|
||||
|
||||
|
||||
let hidden = name.as_bytes()[0] == b'.';
|
||||
|
||||
if !hidden {
|
||||
nothidden.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
let kind = match d.d_type {
|
||||
4 => Kind::Directory,
|
||||
_ => Kind::File,
|
||||
};
|
||||
|
||||
let file = File {
|
||||
name: name,
|
||||
hidden: hidden,
|
||||
kind: kind,
|
||||
path: path,
|
||||
dirsize: None,
|
||||
target: None,
|
||||
meta: None,
|
||||
selected: false,
|
||||
tag: None,
|
||||
};
|
||||
|
||||
localfiles.push(file);
|
||||
|
||||
|
||||
}
|
||||
|
||||
files.lock().unwrap().append(&mut localfiles);
|
||||
});
|
||||
}
|
||||
}).unwrap();
|
||||
|
||||
return std::mem::take(&mut *files.lock().unwrap());
|
||||
}
|
||||
|
||||
|
||||
impl Files {
|
||||
// Use getdents64 on Linux
|
||||
#[cfg(target_os = "linux")]
|
||||
pub fn new_from_path_cancellable(path: &Path, stale: Stale) -> HResult<Files> {
|
||||
use std::os::unix::io::AsRawFd;
|
||||
|
||||
let nonhidden = AtomicUsize::default();
|
||||
|
||||
let dir = Dir::open(path.clone(),
|
||||
OFlag::O_DIRECTORY,
|
||||
Mode::empty())
|
||||
.map_err(|e| FileError::OpenDir(e))?;
|
||||
|
||||
let direntries = from_getdents(dir.as_raw_fd(), path, &nonhidden);
|
||||
|
||||
if stale.is_stale()? {
|
||||
HError::stale()?;
|
||||
}
|
||||
|
||||
let mut files = Files::default();
|
||||
files.directory = File::new_from_path(&path)?;
|
||||
|
||||
|
||||
files.files = direntries;
|
||||
files.len = nonhidden.load(Ordering::Relaxed);
|
||||
files.stale = Some(stale);
|
||||
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
pub fn new_from_path_cancellable(path: &Path, stale: Stale) -> HResult<Files> {
|
||||
let nonhidden = AtomicUsize::default();
|
||||
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ extern crate strum_macros;
|
|||
extern crate derivative;
|
||||
extern crate nix;
|
||||
extern crate strip_ansi_escapes;
|
||||
extern crate crossbeam;
|
||||
|
||||
extern crate osstrtools;
|
||||
extern crate pathbuftools;
|
||||
|
|
|
|||
Loading…
Reference in New Issue