// === crates/m1nd-ingest/src/walker.rs === use crate::path_policy::{is_noise_dir_name, is_noise_path}; use m1nd_core::error::{M1ndError, M1ndResult}; use std::path::{Path, PathBuf}; // --------------------------------------------------------------------------- // DirectoryWalker — file discovery // FM-ING-004 fix: binary detection (NUL byte in first 8KB). // Replaces: ingest.py CodebaseIngestor._walk_directory() // --------------------------------------------------------------------------- /// A discovered file with metadata. #[derive(Clone, Debug)] pub struct DiscoveredFile { pub path: PathBuf, pub relative_path: String, pub extension: Option, pub size_bytes: u64, pub last_modified: f64, /// Number of git commits touching this file (0 if not in a git repo). pub commit_count: u32, /// Timestamp of most recent git commit for this file (0.0 if unavailable). pub last_commit_time: f64, } /// Result of directory walking including co-change commit groups. #[derive(Clone, Debug, Default)] pub struct WalkResult { pub files: Vec, /// Each inner Vec is a group of relative_paths that changed together in one commit. pub commit_groups: Vec>, } /// Directory walker with skip rules and binary file detection. /// Replaces: ingest.py SKIP_DIRS, SKIP_FILES, and walk logic pub struct DirectoryWalker { skip_dirs: Vec, skip_files: Vec, include_dotfiles: bool, dotfile_patterns: Vec, } impl DirectoryWalker { pub fn new( skip_dirs: Vec, skip_files: Vec, include_dotfiles: bool, dotfile_patterns: Vec, ) -> Self { Self { skip_dirs, skip_files, include_dotfiles, dotfile_patterns, } } fn normalize_rel_path(path: &Path, root: &Path) -> String { path.strip_prefix(root) .unwrap_or(path) .to_string_lossy() .replace('\\', "/") .trim_start_matches("./") .trim_matches('/') .to_string() } fn dotfile_pattern_matches(pattern: &str, rel_path: &str) -> bool { let pattern = pattern.trim().trim_start_matches("./"); let rel_path = rel_path.trim().trim_start_matches("./").trim_matches('/'); if pattern.is_empty() { return false; } if let Some(prefix) = pattern.strip_suffix("/**") { return rel_path == prefix || rel_path.starts_with(&format!("{}/", prefix)); } if let Some(prefix) = pattern.strip_suffix("/*") { return rel_path == prefix || rel_path.starts_with(&format!("{}/", prefix)); } rel_path == pattern } fn allow_hidden_path(&self, rel_path: &str) -> bool { self.include_dotfiles && self .dotfile_patterns .iter() .any(|pattern| Self::dotfile_pattern_matches(pattern, rel_path)) } /// Walk directory and return all non-binary, non-skipped files. /// FM-ING-004 fix: checks first 8KB for NUL bytes to detect binary files. /// Replaces: ingest.py directory walking logic pub fn walk(&self, root: &Path) -> M1ndResult { use ignore::WalkBuilder; if !root.exists() { return Err(M1ndError::Io(std::io::Error::new( std::io::ErrorKind::NotFound, format!("Root directory not found: {}", root.display()), ))); } let mut files = Vec::new(); let root_canonical = root.canonicalize().map_err(M1ndError::Io)?; if root_canonical.is_file() { let rel_path = root_canonical .file_name() .and_then(|name| name.to_str()) .map(|name| name.to_string()) .unwrap_or_else(|| root_canonical.to_string_lossy().replace('\\', "/")); if !rel_path.is_empty() { match Self::is_binary(&root_canonical) { Ok(true) => {} Ok(false) => { if let Ok(metadata) = root_canonical.metadata() { files.push(DiscoveredFile { path: root_canonical.clone(), relative_path: rel_path, extension: root_canonical .extension() .map(|ext| ext.to_string_lossy().to_string()), size_bytes: metadata.len(), last_modified: metadata .modified() .ok() .and_then(|time| { time.duration_since(std::time::UNIX_EPOCH).ok() }) .map(|duration| duration.as_secs_f64()) .unwrap_or(0.0), commit_count: 0, last_commit_time: 0.0, }); } } Err(_) => {} } } return Ok(WalkResult { files, commit_groups: Vec::new(), }); } // Gitignore-aware walk (ignore crate — same walker ripgrep uses). // standard_filters default keeps git_ignore/git_global/git_exclude/ignore/parents ON. // hidden(!include_dotfiles) mirrors the old hidden-dir/file pruning while honoring the // include_dotfiles escape hatch. The filter_entry below preserves the hardcoded // skip_dirs + is_noise_dir_name pruning so noise dirs are dropped even in a repo with // NO .gitignore. let filter_root = root_canonical.clone(); let skip_dirs = self.skip_dirs.clone(); let include_dotfiles = self.include_dotfiles; let dotfile_patterns = self.dotfile_patterns.clone(); let walk = WalkBuilder::new(&root_canonical) .follow_links(false) .hidden(!self.include_dotfiles) .filter_entry(move |e| { if e.file_type().is_some_and(|ft| ft.is_dir()) { let name = e.file_name().to_string_lossy(); let rel_path = Self::normalize_rel_path(e.path(), &filter_root); if is_noise_dir_name(&name) { return false; } // Skip hidden dirs and configured skip dirs (mirrors old filter_entry). let allow_hidden = include_dotfiles && dotfile_patterns .iter() .any(|pattern| Self::dotfile_pattern_matches(pattern, &rel_path)); if name.starts_with('.') && name != "." && !allow_hidden { return false; } return !skip_dirs.iter().any(|s| name == s.as_str()); } true }) .build(); for entry in walk { let entry = match entry { Ok(e) => e, Err(_) => continue, // permission error, broken symlink, etc. }; // ignore::DirEntry::file_type is Option (None for stdin/errors) — skip non-files. if !entry.file_type().is_some_and(|ft| ft.is_file()) { continue; } let file_name = entry.file_name().to_string_lossy(); if is_noise_path(entry.path()) { continue; } // Skip configured file patterns if self.skip_files.iter().any(|s| file_name == s.as_str()) { continue; } let rel_path = Self::normalize_rel_path(entry.path(), &root_canonical); // Skip hidden files unless explicitly allowed. if file_name.starts_with('.') && !self.allow_hidden_path(&rel_path) { continue; } let path = entry.path().to_path_buf(); // FM-ING-004: skip binary files match Self::is_binary(&path) { Ok(true) => continue, Ok(false) => {} Err(_) => continue, // can't read -> skip } let metadata = match entry.metadata() { Ok(m) => m, Err(_) => continue, }; let relative_path = rel_path; let extension = path.extension().map(|e| e.to_string_lossy().to_string()); let last_modified = metadata .modified() .ok() .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) .map(|d| d.as_secs_f64()) .unwrap_or(0.0); files.push(DiscoveredFile { path, relative_path, extension, size_bytes: metadata.len(), last_modified, commit_count: 0, last_commit_time: 0.0, }); } // Enrich with git history if available, and collect commit groups let commit_groups = Self::enrich_with_git(&root_canonical, &mut files); Ok(WalkResult { files, commit_groups, }) } /// Enrich discovered files with git history (commit count + last commit time). /// Runs `git log --format='%at' --name-only` once and distributes to files. /// Also collects commit groups: files that changed together in the same commit. /// Gracefully returns empty groups if not in a git repo. fn enrich_with_git(root: &Path, files: &mut [DiscoveredFile]) -> Vec> { use std::collections::HashMap; use std::process::Command; // Run git log to get all commits with timestamps and affected files let output = match Command::new("git") .args(["log", "--format=%at", "--name-only", "--diff-filter=ACDMR"]) .current_dir(root) .output() { Ok(o) if o.status.success() => o, _ => return Vec::new(), // Not a git repo or git not available }; let stdout = String::from_utf8_lossy(&output.stdout); // Parse: alternating timestamp lines and file path lines let mut file_stats: HashMap = HashMap::new(); // path -> (count, last_time) let mut current_timestamp = 0.0f64; let mut commit_groups: Vec> = Vec::new(); let mut current_group: Vec = Vec::new(); for line in stdout.lines() { let line = line.trim(); if line.is_empty() { // Empty line separates commits; flush current group if current_group.len() >= 2 { commit_groups.push(std::mem::take(&mut current_group)); } else { current_group.clear(); } continue; } // Is this a timestamp line? (all digits) if line.chars().all(|c| c.is_ascii_digit()) && line.len() >= 8 { // New commit: flush previous group if any if current_group.len() >= 2 { commit_groups.push(std::mem::take(&mut current_group)); } else { current_group.clear(); } current_timestamp = line.parse::().unwrap_or(0.0); } else if current_timestamp > 0.0 { // File path line let entry = file_stats.entry(line.to_string()).or_insert((0, 0.0)); entry.0 += 1; // commit count if current_timestamp > entry.1 { entry.1 = current_timestamp; // latest commit time } current_group.push(line.to_string()); } } // Flush final group if current_group.len() >= 2 { commit_groups.push(current_group); } // Apply to discovered files for file in files.iter_mut() { if let Some(&(count, last_time)) = file_stats.get(&file.relative_path) { file.commit_count = count; file.last_commit_time = last_time; // Override last_modified with git time if available if last_time > 0.0 { file.last_modified = last_time; } } } commit_groups } /// Check if a file is binary (NUL byte in first 8KB). /// FM-ING-004 fix. pub fn is_binary(path: &Path) -> M1ndResult { use std::io::Read; let mut file = std::fs::File::open(path).map_err(M1ndError::Io)?; let mut buf = [0u8; 8192]; let n = file.read(&mut buf).map_err(M1ndError::Io)?; Ok(buf[..n].contains(&0)) } } #[cfg(test)] mod tests { use super::DirectoryWalker; #[test] fn walk_single_file_uses_filename_as_relative_path() { let temp = std::env::temp_dir().join(format!( "m1nd-walker-single-file-{}-{}", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .expect("clock") .as_nanos() )); std::fs::create_dir_all(&temp).expect("tempdir"); let file = temp.join("core.rs"); std::fs::write(&file, "pub fn core() {}\n").expect("write file"); let walker = DirectoryWalker::new(Vec::new(), Vec::new(), false, Vec::new()); let result = walker.walk(&file).expect("walk single file"); assert_eq!(result.files.len(), 1); assert_eq!(result.files[0].relative_path, "core.rs"); assert_eq!( result.files[0].path, file.canonicalize().expect("canonical file") ); std::fs::remove_dir_all(temp).expect("cleanup tempdir"); } #[test] fn walk_respects_gitignore() { let temp = std::env::temp_dir().join(format!( "m1nd-walker-gitignore-{}-{}", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .expect("clock") .as_nanos() )); std::fs::create_dir_all(temp.join("vendored")).expect("vendored dir"); std::fs::create_dir_all(temp.join("src")).expect("src dir"); std::fs::write(temp.join(".gitignore"), "vendored/\n").expect("gitignore"); std::fs::write(temp.join("vendored").join("junk.rs"), "pub fn junk() {}\n") .expect("junk file"); std::fs::write(temp.join("src").join("real.rs"), "pub fn real() {}\n").expect("real file"); // `git init` makes the .gitignore deterministically honored by the ignore crate. let status = std::process::Command::new("git") .args(["init", "-q"]) .current_dir(&temp) .status() .expect("git init"); assert!(status.success(), "git init failed"); let walker = DirectoryWalker::new(Vec::new(), Vec::new(), false, Vec::new()); let result = walker.walk(&temp).expect("walk gitignore tree"); let rels: Vec<&str> = result .files .iter() .map(|f| f.relative_path.as_str()) .collect(); assert!( rels.contains(&"src/real.rs"), "expected src/real.rs to be discovered, got {rels:?}" ); assert!( !rels.contains(&"vendored/junk.rs"), "expected vendored/junk.rs to be gitignored, got {rels:?}" ); std::fs::remove_dir_all(temp).expect("cleanup tempdir"); } }