Shared memory and context tools for agentic work.
Code Rooms
// === crates/m1nd-ingest/src/extract/mod.rs ===
use m1nd_core::error::M1ndResult;
use m1nd_core::types::NodeType;
pub mod generic;
pub mod go;
pub mod java;
pub mod python;
pub mod rust_lang;
pub mod typescript;
#[cfg(feature = "tier1")]
pub mod tree_sitter_ext;
// ---------------------------------------------------------------------------
// Comment/string stripping — shared preprocessing for all extractors
// FM-ING-010: strip comments and string literals before regex extraction
// so that e.g. `"fn main()"` in a string literal is not extracted as a function.
/// Language-specific comment syntax for the pre-processor.
#[derive(Clone, Copy)]
pub struct CommentSyntax {
/// Single-line comment prefix (e.g., "//", "#", "--").
pub line_comment: &'static str,
/// Block comment open (e.g., "/*"). Empty string means none.
pub block_open: &'static str,
/// Block comment close (e.g., "*/"). Empty string means none.
pub block_close: &'static str,
/// Triple-quote doc comment (e.g., `"""` for Python). Empty string means none.
pub triple_quote: &'static str,
/// Rust uses apostrophe-prefixed lifetime/label tokens (`'a`, `'static`)
/// that are not character literals and must remain visible to brace/scope
/// tracking. Other syntaxes keep the ordinary single-quoted-string rule.
pub rust_lifetimes: bool,
}
impl CommentSyntax {
pub const RUST: Self = Self {
line_comment: "//",
block_open: "/*",
block_close: "*/",
triple_quote: "",
rust_lifetimes: true,
};
pub const PYTHON: Self = Self {
line_comment: "#",
block_open: "",
block_close: "",
triple_quote: "\"\"\"",
rust_lifetimes: false,
pub const C_STYLE: Self = Self {
pub const GO: Self = Self {
pub const GENERIC: Self = Self {
// --- Tier 1 tree-sitter languages ---
pub const CPP: Self = Self {
pub const CSHARP: Self = Self {
pub const RUBY: Self = Self {
block_open: "=begin",
block_close: "=end",
pub const PHP: Self = Self {
pub const SWIFT: Self = Self {
pub const KOTLIN: Self = Self {
pub const SCALA: Self = Self {
pub const BASH: Self = Self {
pub const LUA: Self = Self {
line_comment: "--",
block_open: "--[[",
block_close: "]]",
pub const R: Self = Self {
pub const HTML: Self = Self {
line_comment: "",
block_open: "<!--",
block_close: "-->",
pub const CSS: Self = Self {
/// Strips comments and string literals from source text, line by line.
/// Returns a Vec of cleaned lines (one per input line).
/// Block comment / triple-quote state is tracked across lines.
///
/// Import lines (e.g., `import "fmt"`, `from 'react'`, `use crate::foo`)
/// have their string content preserved so that module names are not lost.
/// Only comments are stripped on import lines.
pub fn strip_comments_and_strings(text: &str, syntax: CommentSyntax) -> Vec<String> {
let mut result = Vec::new();
let mut in_block_comment = false;
let mut in_triple_quote = false;
let mut rust_raw_string_hashes = None;
for line in text.lines() {
// If we are NOT inside a block comment / triple-quote **and** the
// line looks like an import statement, preserve string content —
// only strip comments.
let preserve_strings = !in_block_comment && !in_triple_quote && is_import_line(line);
let cleaned = strip_line(
line,
&syntax,
&mut in_block_comment,
&mut in_triple_quote,
&mut rust_raw_string_hashes,
preserve_strings,
);
result.push(cleaned);
result
/// Returns true if `line` looks like an import/use statement in any
/// supported language, meaning string literals on this line contain
/// module names that must not be stripped.
fn is_import_line(line: &str) -> bool {
let trimmed = line.trim_start();
// Go / TS / JS / Java / Python: `import ...`
if trimmed.starts_with("import ") || trimmed.starts_with("import\t") {
return true;
// Python: `from foo import bar`
if trimmed.starts_with("from ") || trimmed.starts_with("from\t") {
// Rust: `use crate::...` or `pub use ...`
if trimmed.starts_with("use ") || trimmed.starts_with("use\t") {
if trimmed.starts_with("pub use ") || trimmed.starts_with("pub use\t") {
// Go grouped import: line inside `import ( ... )` block — these are
// typically just `"package/path"`, detected by leading quote after
// optional whitespace.
if trimmed.starts_with('"') || trimmed.starts_with('\'') {
// Could be inside an import block; preserve conservatively.
// Callers that are NOT in an import block still benefit because
// a bare string-only line has no function/class defs to confuse.
false
/// Strip a single line, mutating block-comment/triple-quote tracking state.
/// When `preserve_strings` is true, string literal content is kept intact
/// (only comments are stripped). This is used for import lines where module
/// names live inside quotes.
fn strip_line(
line: &str,
syntax: &CommentSyntax,
in_block_comment: &mut bool,
in_triple_quote: &mut bool,
rust_raw_string_hashes: &mut Option<usize>,
preserve_strings: bool,
) -> String {
let mut out = String::with_capacity(line.len());
let chars: Vec<char> = line.chars().collect();
let len = chars.len();
let mut i = 0;
while i < len {
// Rust raw strings may span lines and legitimately contain arbitrary
// braces, quotes, comments, and source-looking declarations. Keep a
// hash-counted state so regex/scope extraction never sees their body.
if let Some(hashes) = *rust_raw_string_hashes {
if rust_raw_string_closes_at(&chars, i, hashes) {
i += 1 + hashes;
*rust_raw_string_hashes = None;
} else {
i += 1;
continue;
// --- Inside a block comment: scan for close ---
if *in_block_comment {
if !syntax.block_close.is_empty() {
let close_chars: Vec<char> = syntax.block_close.chars().collect();
if i + close_chars.len() <= len
&& chars[i..i + close_chars.len()] == close_chars[..]
{
*in_block_comment = false;
i += close_chars.len();
// --- Inside a triple-quote string: scan for closing triple-quote ---
if *in_triple_quote {
if !syntax.triple_quote.is_empty() {
let tq_chars: Vec<char> = syntax.triple_quote.chars().collect();
if i + tq_chars.len() <= len && chars[i..i + tq_chars.len()] == tq_chars[..] {
*in_triple_quote = false;
i += tq_chars.len();
// --- Check for triple-quote open ---
*in_triple_quote = true;
// --- Check for block comment open ---
if !syntax.block_open.is_empty() {
let bo_chars: Vec<char> = syntax.block_open.chars().collect();
if i + bo_chars.len() <= len && chars[i..i + bo_chars.len()] == bo_chars[..] {
*in_block_comment = true;
i += bo_chars.len();
if syntax.rust_lifetimes {
if let Some((opener_len, hashes)) = rust_raw_string_opens_at(&chars, i) {
// Retain an empty string token as a separator while hiding the
// complete raw body from every downstream regex.
out.push_str("\"\"");
i += opener_len;
*rust_raw_string_hashes = Some(hashes);
// --- Check for line comment ---
if !syntax.line_comment.is_empty() {
let lc_chars: Vec<char> = syntax.line_comment.chars().collect();
if i + lc_chars.len() <= len && chars[i..i + lc_chars.len()] == lc_chars[..] {
// Rest of line is comment; stop processing this line
break;
// --- Check for string literals: "..." or '...' ---
if chars[i] == '\'' && syntax.rust_lifetimes && is_rust_lifetime_or_label_start(&chars, i) {
out.push(chars[i]);
if chars[i] == '"' || chars[i] == '\'' {
let quote = chars[i];
out.push(quote); // keep the quote delimiters
// Skip content until matching close quote (handle escapes)
if chars[i] == '\\' {
if preserve_strings {
if i + 1 < len {
out.push(chars[i + 1]);
i += 2; // skip escaped char
if chars[i] == quote {
out.push(quote);
// Otherwise: strip string content (original behavior)
out
fn is_rust_lifetime_or_label_start(chars: &[char], apostrophe: usize) -> bool {
let Some(next) = chars.get(apostrophe + 1) else {
return false;
if !(*next == '_' || next.is_ascii_alphabetic()) {
// `'a'` is a character literal; `'a`, `'static`, `'_`, and `'loop:` are
// lifetime/label tokens. Escaped character literals start with `\` and are
// rejected by the identifier check above.
chars.get(apostrophe + 2) != Some(&'\'')
fn rust_raw_string_opens_at(chars: &[char], start: usize) -> Option<(usize, usize)> {
if start > 0 && (chars[start - 1].is_ascii_alphanumeric() || chars[start - 1] == '_') {
return None;
let r = match (chars.get(start), chars.get(start + 1)) {
(Some('r'), _) => start,
(Some('b' | 'c'), Some('r')) => start + 1,
_ => return None,
let mut cursor = r + 1;
while chars.get(cursor) == Some(&'#') {
cursor += 1;
if chars.get(cursor) != Some(&'"') {
let hashes = cursor - (r + 1);
Some((cursor - start + 1, hashes))
fn rust_raw_string_closes_at(chars: &[char], quote: usize, hashes: usize) -> bool {
chars.get(quote) == Some(&'"')
&& (0..hashes).all(|offset| chars.get(quote + 1 + offset) == Some(&'#'))
#[cfg(test)]
mod preprocessing_tests {
use super::*;
#[test]
fn rust_lifetimes_do_not_hide_scope_braces_but_char_literals_do() {
let source = "enum E {\n Named { field: &'static str },\n}\nfn f<'a>(v: &'a str) {\n let brace = '{';\n}\n";
let cleaned = strip_comments_and_strings(source, CommentSyntax::RUST);
assert!(cleaned[1].contains("&'static str"));
assert!(cleaned[3].contains("fn f<'a>(v: &'a str)"));
assert_eq!(cleaned[4].matches('{').count(), 0);
let opens = cleaned
.iter()
.map(|line| line.matches('{').count())
.sum::<usize>();
let closes = cleaned
.map(|line| line.matches('}').count())
assert_eq!(opens, closes);
fn rust_raw_strings_hide_embedded_source_across_lines() {
let source = "fn outer() {\n let fixture = br##\"\n pub struct Engine;\n } // not a real scope\n \"##;\n}\n";
assert!(!cleaned.iter().any(|line| line.contains("struct Engine")));
// ExtractedNode / ExtractedEdge — extraction output
// Replaces: ingest.py per-extractor output tuples
/// A node extracted from source code.
#[derive(Clone, Debug)]
pub struct ExtractedNode {
/// Unique ID within the file (e.g., "file::src/main.rs::fn::main").
pub id: String,
/// Human-readable label.
pub label: String,
/// Node type (function, class, struct, etc.).
pub node_type: NodeType,
/// Tags (e.g., ["async", "public", "test"]).
pub tags: Vec<String>,
/// Line number in source file.
pub line: u32,
/// End line number.
pub end_line: u32,
/// An edge extracted from source code.
pub struct ExtractedEdge {
/// Source node ID.
pub source: String,
/// Target node ID (may be unresolved reference).
pub target: String,
/// Relation type (e.g., "contains", "calls", "imports", "ref::").
pub relation: String,
/// Edge weight (default 1.0).
pub weight: f32,
/// Result of extracting a single file.
pub struct ExtractionResult {
pub nodes: Vec<ExtractedNode>,
pub edges: Vec<ExtractedEdge>,
/// Unresolved references (target IDs that need resolution).
pub unresolved_refs: Vec<String>,
/// Make a node id unique within a single file's extraction. Function/method ids
/// are `file::…::fn::<name>` with NO line, so two same-named definitions in ONE
/// file (TS overloads / two class methods named `process`, Java overloads, Go
/// methods named `Run` on two types, Python methods named `save` in two classes)
/// otherwise collide on one id. `Graph::add_node` keys on the id and returns
/// `Err(DuplicateNode)`, and the ingest loader drops the duplicate — so the
/// later same-named sibling silently vanishes from the graph (invisible to
/// impact/why/seek). The FIRST occurrence keeps the clean id (back-compat:
/// line-less `…::fn::name` queries still resolve to it); later siblings get a
/// `#2`, `#3`, … suffix so every distinct definition exists and is addressable.
/// The node `label` stays `name`, so search/seek still match by label. Mirrors
/// `RustExtractor::unique_fn_id` but takes a node slice so the regex extractors
/// (which build a bare `Vec<ExtractedNode>`) can share it.
pub fn unique_node_id(nodes: &[ExtractedNode], base_id: &str) -> String {
if !nodes.iter().any(|n| n.id == base_id) {
return base_id.to_string();
let mut n = 2u32;
loop {
let candidate = format!("{base_id}#{n}");
if !nodes.iter().any(|node| node.id == candidate) {
return candidate;
n += 1;
/// Max source lines folded into a code symbol's excerpt (signature + a few body
/// lines) and the hard character budget on the joined result.
pub const EXCERPT_MAX_LINES: usize = 4;
pub const EXCERPT_MAX_CHARS: usize = 320;
/// Max preceding comment lines folded into a symbol's excerpt as doc-intent context.
pub const EXCERPT_MAX_DOC_LINES: usize = 6;
/// If `trimmed` is a comment line, return its prose payload (text after the
/// marker); else None. Handles `///` `//!` `//` (Rust/JS/Go/C), `#`
/// (Python/Ruby/shell), and `/**` `/*` `*/` `*` block-comment bodies. Used to
/// fold a symbol's preceding doc comment into its embedding text.
fn comment_prose(trimmed: &str) -> Option<&str> {
for marker in ["///", "//!", "//", "/**", "/*", "*/", "*", "#"] {
if let Some(rest) = trimmed.strip_prefix(marker) {
return Some(rest.trim());
None
/// Derive a short behavioral excerpt for each non-File symbol from its own source
/// span (signature + first body lines), so downstream embeddings capture what a
/// symbol DOES — not just its name (without this, code symbols embed label-only).
/// Returns `(node_id, excerpt)` pairs; nodes with no usable span are omitted.
/// Language-agnostic: it slices the file text by the node's `[line, end_line]`
/// range. Non-UTF-8 content and File nodes are skipped.
pub fn compute_excerpts(result: &ExtractionResult, content: &[u8]) -> Vec<(String, String)> {
let text = match std::str::from_utf8(content) {
Ok(t) => t,
Err(_) => return Vec::new(),
let lines: Vec<&str> = text.lines().collect();
let mut out = Vec::new();
for node in &result.nodes {
if node.node_type == NodeType::File || node.line == 0 {
let start = (node.line as usize) - 1;
if start >= lines.len() {
// Cap by the symbol's real end ONLY when the extractor tracked it
// (end_line > line). Regex-based extractors leave end_line == line, so
// fall back to the line budget to still fold in the first body lines —
// the behavioral signal — rather than the signature alone.
let upper = if (node.end_line as usize) > (node.line as usize) {
(node.end_line as usize).min(lines.len())
lines.len()
let stop = (start + EXCERPT_MAX_LINES)
.min(upper)
.max(start + 1)
.min(lines.len());
// Fold in the symbol's PRECEDING doc comment (rustdoc/JSDoc/Go/`#`) — the
// purest statement of INTENT — scanning upward, skipping Rust attributes,
// stopping at a blank line (so file-top license headers are excluded).
let mut doc: Vec<&str> = Vec::new();
let mut i = start;
let mut scanned = 0;
while i > 0 && scanned < EXCERPT_MAX_DOC_LINES {
i -= 1;
let t = lines[i].trim();
if t.is_empty() {
if t.starts_with("#[") || t.starts_with("#![") {
scanned += 1; // Rust attribute between doc and item — skip, keep scanning
match comment_prose(t) {
Some(prose) => {
if !prose.is_empty() {
doc.push(prose);
scanned += 1;
None => break, // reached real code
doc.reverse(); // back into source order
// excerpt = doc prose (intent) + signature/body, joined and char-bounded.
let mut excerpt = String::new();
let body = lines[start..stop].iter().map(|l| l.trim());
for piece in doc.into_iter().chain(body) {
if piece.is_empty() {
if !excerpt.is_empty() {
excerpt.push(' ');
excerpt.push_str(piece);
if excerpt.chars().count() >= EXCERPT_MAX_CHARS {
let excerpt: String = excerpt.chars().take(EXCERPT_MAX_CHARS).collect();
out.push((node.id.clone(), excerpt));
// Extractor — trait for language-specific extraction
// Replaces: ingest.py PythonExtractor, TypeScriptExtractor, etc.
/// Language-specific code structure extractor.
/// All impls use tree-sitter (not regex) for correct extraction.
/// FM-ING-009: tree-sitter captures indented defs.
/// FM-ING-010: tree-sitter AST excludes strings/comments.
pub trait Extractor: Send + Sync {
/// Extract nodes and edges from file content.
/// `file_id` is the canonical file identifier (e.g., "file::src/main.rs").
fn extract(&self, content: &[u8], file_id: &str) -> M1ndResult<ExtractionResult>;
/// File extensions this extractor handles.
fn extensions(&self) -> &[&str];
mod excerpt_tests {
fn compute_excerpts_slices_signature_and_body_and_skips_files() {
let src = "/// Drains in-flight work on cancellation.\npub fn drain_inflight(ctx: &Ctx) -> Result<()> {\n abort_running_work();\n stop_accepting_tasks();\n}\n";
let result = ExtractionResult {
nodes: vec![
ExtractedNode {
id: "file::x.rs::fn::drain_inflight".into(),
label: "drain_inflight".into(),
node_type: NodeType::Function,
tags: vec![],
line: 2, // the `pub fn ...` line (1-based)
end_line: 5,
},
id: "file::x.rs".into(),
label: "x.rs".into(),
node_type: NodeType::File,
line: 1,
],
edges: vec![],
unresolved_refs: vec![],
let ex = compute_excerpts(&result, src.as_bytes());
// File node is skipped; the function node gets a behavioral excerpt.
assert_eq!(ex.len(), 1, "only the non-file symbol gets an excerpt");
let (id, text) = &ex[0];
assert_eq!(id, "file::x.rs::fn::drain_inflight");
assert!(
text.contains("drain_inflight"),
"excerpt carries the signature: {text}"
text.contains("abort_running_work"),
"excerpt folds in the first body line(s): {text}"
text.contains("cancellation"),
"excerpt folds in the preceding doc comment (intent): {text}"
assert!(text.chars().count() <= EXCERPT_MAX_CHARS);
