Shared memory and context tools for agentic work.
Code Rooms
use crate::canonical::{
short_hash, short_hash_bytes, source_kind_from_extension, CanonicalDocument, ClaimModality,
ConfidenceLevel, DocumentBlock, DocumentBlockKind, DocumentCitation, DocumentClaimCandidate,
DocumentClaimKind, DocumentCodeCandidate, DocumentEntityCandidate, DocumentEntityKind,
DocumentLink, DocumentMetadata, DocumentSection, DocumentSectionKind, DocumentSpan,
DocumentTable, DocumentTableCell, DocumentTableRow, ProvenanceSpan, SourceKind,
};
use crate::{extension_of, relative_source_path};
use crate::{
BibTexAdapter, CrossRefAdapter, IngestAdapter, IngestStats, JatsArticleAdapter,
L1ghtIngestAdapter, PatentIngestAdapter, RfcAdapter,
use m1nd_core::error::{M1ndError, M1ndResult};
use m1nd_core::graph::{Graph, NodeProvenanceInput};
use m1nd_core::types::{EdgeDirection, FiniteF32, NodeType};
use regex::Regex;
use serde::{Deserialize, Serialize};
use std::collections::HashSet;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;
use walkdir::WalkDir;
#[derive(Clone, Debug, Serialize, Deserialize, Default)]
pub struct ProviderAvailability {
pub magika: bool,
pub docling: bool,
pub markitdown: bool,
pub trafilatura: bool,
pub grobid: bool,
pub marker: bool,
pub mineru: bool,
}
pub struct UniversalIngestBundle {
pub graph: Graph,
pub stats: IngestStats,
pub documents: Vec<CanonicalDocument>,
pub struct UniversalIngestAdapter {
namespace: String,
impl UniversalIngestAdapter {
pub fn new(namespace: Option<String>) -> Self {
Self {
namespace: namespace.unwrap_or_else(|| "universal".to_string()),
pub fn provider_availability() -> ProviderAvailability {
ProviderAvailability {
magika: python_module_available("magika"),
docling: python_module_available("docling"),
markitdown: python_module_available("markitdown"),
trafilatura: python_module_available("trafilatura"),
grobid: grobid_configured(),
marker: command_available("marker"),
mineru: command_available("mineru"),
pub fn provider_python_command() -> String {
provider_python()
pub fn can_handle_path(path: &Path) -> bool {
let ext = extension_of(path);
matches!(
ext.as_str(),
"md" | "markdown"
| "txt"
| "rst"
| "adoc"
| "html"
| "htm"
| "pdf"
| "docx"
| "pptx"
| "xlsx"
| "xml"
| "nxml"
| "json"
| "bib"
| "bibtex"
)
pub fn ingest_bundle(&self, root: &Path) -> M1ndResult<UniversalIngestBundle> {
let start = std::time::Instant::now();
let mut stats = IngestStats::default();
let files = collect_candidate_files(root);
stats.files_scanned = files.len() as u64;
let mut documents = Vec::new();
for path in files {
if let Some(document) = self.canonicalize_path(root, &path)? {
stats.files_parsed += 1;
documents.push(document);
let graph = graphify_documents(&documents, &self.namespace)?;
stats.nodes_created = graph.num_nodes() as u64;
stats.edges_created = graph.num_edges() as u64;
stats.elapsed_ms = start.elapsed().as_secs_f64() * 1000.0;
Ok(UniversalIngestBundle {
graph,
stats,
documents,
})
fn canonicalize_path(&self, root: &Path, path: &Path) -> M1ndResult<Option<CanonicalDocument>> {
let rel_path = relative_source_path(root, path);
let source_kind = source_kind_from_extension(path);
let availability = Self::provider_availability();
let bytes = fs::read(path).map_err(|error| M1ndError::InvalidParams {
tool: "universal_ingest".into(),
detail: format!("failed to read {}: {}", path.display(), error),
})?;
let source_text = String::from_utf8_lossy(&bytes).to_string();
let mut document = match source_kind {
SourceKind::Markdown | SourceKind::Text => {
canonicalize_plain_text(&rel_path, source_kind, "universal:internal", &source_text)
SourceKind::Html => {
if availability.trafilatura {
canonicalize_html_with_fallback(
&rel_path,
"universal:trafilatura",
"universal:internal-html",
trafilatura_extract(path).as_deref().unwrap_or(&source_text),
} else if availability.docling {
let extracted = docling_extract(path);
"universal:docling",
extracted.as_deref().unwrap_or(&source_text),
} else {
&source_text,
SourceKind::Pdf | SourceKind::Docx | SourceKind::Pptx | SourceKind::Xlsx
if availability.docling || availability.markitdown =>
{
let extracted = if matches!(source_kind, SourceKind::Pdf) && availability.grobid {
grobid_extract(path)
docling_extract(path)
} else if availability.markitdown {
markitdown_extract(path)
None
canonicalize_binary_placeholder(
source_kind.clone(),
if matches!(source_kind, SourceKind::Pdf) && availability.grobid {
"universal:grobid"
"universal:docling"
"universal:markitdown"
},
SourceKind::Pdf | SourceKind::Docx | SourceKind::Pptx | SourceKind::Xlsx => {
return Ok(None);
SourceKind::Unknown => {
if let Some(native) =
self.wrap_native_document(root, path, &rel_path, &source_text)?
native
_ => return Ok(None),
document.content_hash = short_hash_bytes(&bytes);
Ok(Some(document))
fn wrap_native_document(
&self,
_root: &Path,
path: &Path,
rel_path: &str,
source_text: &str,
) -> M1ndResult<Option<CanonicalDocument>> {
let ext = path
.extension()
.and_then(|value| value.to_str())
.map(|value| value.to_ascii_lowercase())
.unwrap_or_default();
let native_kind = if matches!(ext.as_str(), "md" | "markdown")
&& L1ghtIngestAdapter::looks_like_l1ght(source_text)
Some(SourceKind::NativeLight)
} else if matches!(ext.as_str(), "bib" | "bibtex") {
Some(SourceKind::NativeBibtex)
} else if matches!(ext.as_str(), "xml" | "nxml") {
if source_text.contains("<PubmedArticle")
|| source_text.contains("<PubmedArticleSet")
|| source_text.contains("NLM//DTD")
|| (source_text.contains("<article") && source_text.contains("dtd-version"))
Some(SourceKind::NativeArticle)
} else if source_text.contains("<rfc ") || source_text.contains("<rfc>") {
Some(SourceKind::NativeRfc)
} else if source_text.contains("<us-patent-grant")
|| source_text.contains("<us-patent-application")
|| source_text.contains("<ep-patent-document")
Some(SourceKind::NativePatent)
} else if ext == "json"
&& source_text.contains("\"DOI\"")
&& source_text.contains("\"publisher\"")
&& source_text.contains("\"type\"")
Some(SourceKind::NativeCrossref)
let Some(native_kind) = native_kind else {
Ok(Some(canonicalize_plain_text(
rel_path,
native_kind,
"universal:native-wrap",
source_text,
)))
impl IngestAdapter for UniversalIngestAdapter {
fn domain(&self) -> &str {
"universal"
fn ingest(&self, root: &Path) -> M1ndResult<(Graph, IngestStats)> {
let bundle = self.ingest_bundle(root)?;
Ok((bundle.graph, bundle.stats))
fn python_module_available(module_name: &str) -> bool {
let output = Command::new(provider_python())
.arg("-c")
.arg(format!("import importlib.util; print('1' if importlib.util.find_spec('{module_name}') else '0')"))
.output();
matches!(output, Ok(result) if String::from_utf8_lossy(&result.stdout).trim() == "1")
fn command_available(name: &str) -> bool {
Command::new("sh")
.arg(format!("command -v {} >/dev/null 2>&1", name))
.status()
.map(|status| status.success())
.unwrap_or(false)
fn grobid_configured() -> bool {
std::env::var("M1ND_GROBID_URL")
.map(|value| !value.trim().is_empty())
fn python_inline(script: &str, arg: &Path) -> Option<String> {
.arg(script)
.arg(arg)
.output()
.ok()?;
if !output.status.success() {
return None;
let stdout = String::from_utf8(output.stdout).ok()?;
let trimmed = stdout.trim();
(!trimmed.is_empty()).then(|| trimmed.to_string())
fn provider_python() -> String {
std::env::var("M1ND_PROVIDER_PYTHON").unwrap_or_else(|_| "python3".to_string())
fn docling_extract(path: &Path) -> Option<String> {
python_inline(
r#"
import sys
from docling.document_converter import DocumentConverter
converter = DocumentConverter()
result = converter.convert(sys.argv[1])
doc = getattr(result, 'document', None)
if doc is not None and hasattr(doc, 'export_to_markdown'):
text = doc.export_to_markdown()
if text:
print(text)
"#,
path,
fn trafilatura_extract(path: &Path) -> Option<String> {
import sys, pathlib, trafilatura
path = pathlib.Path(sys.argv[1])
text = path.read_text(encoding='utf-8', errors='ignore')
extracted = trafilatura.extract(text, output_format='markdown', include_links=True, include_tables=True)
if extracted:
print(extracted)
fn markitdown_extract(path: &Path) -> Option<String> {
from markitdown import MarkItDown
md = MarkItDown()
result = md.convert(sys.argv[1])
text = getattr(result, 'text_content', None) or getattr(result, 'markdown', None) or str(result)
fn grobid_extract(path: &Path) -> Option<String> {
let url = std::env::var("M1ND_GROBID_URL").ok()?;
let script = format!(
import sys, requests
path = sys.argv[1]
url = {url_repr}.rstrip('/') + '/api/processFulltextDocument'
with open(path, 'rb') as fh:
resp = requests.post(url, files={{'input': fh}}, timeout=30)
resp.raise_for_status()
text = resp.text.strip()
url_repr = serde_json::to_string(&url).ok()?
);
python_inline(&script, path)
fn collect_candidate_files(root: &Path) -> Vec<PathBuf> {
if root.is_file() {
return vec![root.to_path_buf()];
WalkDir::new(root)
.follow_links(true)
.into_iter()
.filter_map(Result::ok)
.filter(|entry| entry.file_type().is_file())
.map(|entry| entry.into_path())
.collect()
fn slugify(raw: &str) -> String {
let mut out = String::with_capacity(raw.len());
let mut prev_dash = false;
for ch in raw.chars() {
if ch.is_ascii_alphanumeric() {
out.push(ch.to_ascii_lowercase());
prev_dash = false;
} else if !prev_dash {
out.push('-');
prev_dash = true;
out.trim_matches('-').to_string()
fn content_hash(input: &str) -> String {
short_hash(input)
fn default_title(source_path: &str, plain_text: &str) -> String {
plain_text
.lines()
.map(str::trim)
.find(|line| !line.is_empty())
.map(|line| line.trim_start_matches('#').trim().to_string())
.filter(|line| !line.is_empty())
.unwrap_or_else(|| {
Path::new(source_path)
.file_name()
.unwrap_or(source_path)
.to_string()
fn section_id(source_path: &str, heading: &str, index: usize) -> String {
format!(
"section::{}::{}-{}",
short_hash(source_path),
slugify(heading),
index
fn block_id(source_path: &str, heading: &str, index: usize) -> String {
"block::{}::{}-{}",
fn claim_id(source_path: &str, label: &str, index: usize) -> String {
"claim::{}::{}-{}",
slugify(label),
fn classify_section_kind(heading: &str) -> DocumentSectionKind {
let lower = heading.to_ascii_lowercase();
if lower.contains("api") || lower.contains("contract") || lower.contains("interface") {
DocumentSectionKind::Api
} else if lower.contains("constraint") || lower.contains("invariant") || lower.contains("guard")
DocumentSectionKind::Constraints
} else if lower.contains("test") || lower.contains("verification") {
DocumentSectionKind::Tests
} else if lower.contains("rollout") || lower.contains("migration") {
DocumentSectionKind::Rollout
} else if lower.contains("reference") || lower.contains("bibliography") {
DocumentSectionKind::Reference
} else if lower.contains("appendix") {
DocumentSectionKind::Appendix
} else if lower.contains("overview")
|| lower.contains("introduction")
|| lower.contains("summary")
DocumentSectionKind::Overview
DocumentSectionKind::Unknown
fn classify_entity_kind(label: &str) -> DocumentEntityKind {
if label.contains("m1nd.") {
DocumentEntityKind::ToolId
} else if label.contains('/')
|| label.contains(".rs")
|| label.contains(".py")
|| label.contains(".ts")
|| label.contains(".md")
DocumentEntityKind::FilePath
} else if label.contains("::") || label.contains('.') {
DocumentEntityKind::Symbol
} else if label.to_ascii_lowercase().contains("test") {
DocumentEntityKind::TestName
DocumentEntityKind::NamedTerm
fn classify_claim(line: &str) -> (DocumentClaimKind, ClaimModality, bool) {
let lower = line.to_ascii_lowercase();
let modality = if lower.contains(" must ") || lower.starts_with("must ") {
ClaimModality::Must
} else if lower.contains(" should ") || lower.starts_with("should ") {
ClaimModality::Should
} else if lower.contains(" may ") || lower.starts_with("may ") {
ClaimModality::May
} else if lower.contains(" is ") || lower.starts_with("is ") {
ClaimModality::Is
ClaimModality::Unknown
let negated = lower.contains(" not ") || lower.contains(" no ");
let kind = if lower.contains("warning") || lower.contains("danger") || lower.contains("risk") {
DocumentClaimKind::Warning
} else if lower.contains("test") || lower.contains("assert") || lower.contains("expect") {
DocumentClaimKind::TestExpectation
} else if matches!(modality, ClaimModality::Must | ClaimModality::Should) {
DocumentClaimKind::Requirement
} else if lower.contains("decision") || lower.contains("we choose") || lower.contains("chosen")
DocumentClaimKind::Decision
} else if lower.contains("always") || lower.contains("never") || lower.contains("invariant") {
DocumentClaimKind::Invariant
} else if lower.contains("can ") || lower.contains("supports") || lower.contains("capability") {
DocumentClaimKind::Capability
DocumentClaimKind::Unknown
(kind, modality, negated)
fn canonicalize_plain_text(
source_path: &str,
source_kind: SourceKind,
producer: &str,
raw_text: &str,
) -> CanonicalDocument {
let heading_re = Regex::new(r"^(#{1,6})\s+(.+?)\s*$").unwrap();
let link_re = Regex::new(r"\[([^\]]+)\]\(([^)]+)\)").unwrap();
let doi_re = Regex::new(r"(10\.\d{4,9}/[-._;()/:A-Za-z0-9]+)").unwrap();
let code_ref_re = Regex::new(r"`([^`]+)`").unwrap();
let entity_re = Regex::new(r"\b([A-Z][A-Za-z0-9_.:-]{3,})\b").unwrap();
let mut sections = Vec::new();
let mut links = Vec::new();
let mut citations = Vec::new();
let mut entities = Vec::new();
let mut claims = Vec::new();
let mut code_candidates = Vec::new();
let mut tables = Vec::new();
let mut current_heading = "Document".to_string();
let mut current_level = 1u8;
let mut current_parent_section_id: Option<String> = None;
let mut current_blocks = Vec::new();
let mut section_index = 0usize;
let mut block_index = 0usize;
let mut claim_index = 0usize;
let mut seen_entities = HashSet::new();
let mut seen_links = HashSet::new();
let mut seen_citations = HashSet::new();
let mut seen_code_candidates = HashSet::new();
let mut section_stack: Vec<(u8, String)> = Vec::new();
let mut in_code_block = false;
let mut code_lang: Option<String> = None;
let mut flush_section = |sections: &mut Vec<DocumentSection>,
current_heading: &mut String,
current_level: &mut u8,
current_parent_section_id: &mut Option<String>,
current_blocks: &mut Vec<DocumentBlock>,
section_index: &mut usize| {
if current_blocks.is_empty() {
return;
*section_index += 1;
sections.push(DocumentSection {
section_id: section_id(source_path, current_heading, *section_index),
heading: current_heading.clone(),
level: *current_level,
kind: classify_section_kind(current_heading),
parent_section_id: current_parent_section_id.clone(),
blocks: std::mem::take(current_blocks),
provenance: ProvenanceSpan {
line_start: None,
line_end: None,
excerpt: Some(current_heading.clone()),
});
for (idx, line) in raw_text.lines().enumerate() {
let line_no = idx as u32 + 1;
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
if let Some(caps) = heading_re.captures(trimmed) {
flush_section(
&mut sections,
&mut current_heading,
&mut current_level,
&mut current_parent_section_id,
&mut current_blocks,
&mut section_index,
current_heading = caps.get(2).unwrap().as_str().trim().to_string();
current_level = caps.get(1).unwrap().as_str().len() as u8;
while section_stack
.last()
.is_some_and(|(level, _)| *level >= current_level)
section_stack.pop();
current_parent_section_id = section_stack.last().map(|(_, id)| id.clone());
section_stack.push((
current_level,
section_id(source_path, ¤t_heading, section_index + 1),
));
if trimmed.starts_with("```") {
in_code_block = !in_code_block;
code_lang = trimmed
.strip_prefix("```")
.filter(|value| !value.is_empty())
.map(|value| value.to_string());
let is_table_line = trimmed.contains('|')
&& trimmed.matches('|').count() >= 2
&& !trimmed.starts_with("http");
if is_table_line {
let cells = trimmed
.trim_matches('|')
.split('|')
.map(|cell| DocumentTableCell {
text: cell.trim().to_string(),
line_start: Some(line_no),
line_end: Some(line_no),
excerpt: Some(trimmed.to_string()),
.collect::<Vec<_>>();
let table_id = format!("table::{}::{}", short_hash(source_path), tables.len() + 1);
let headers = if tables.last().is_none() {
cells.iter().map(|cell| cell.text.clone()).collect()
Vec::new()
tables.push(DocumentTable {
table_id,
headers,
rows: vec![DocumentTableRow { cells }],
confidence: ConfidenceLevel::Parsed,
let kind = if in_code_block {
DocumentBlockKind::Code
} else if trimmed.starts_with("- ") || trimmed.starts_with("* ") {
DocumentBlockKind::ListItem
} else if trimmed.starts_with('>') {
DocumentBlockKind::Quote
} else if is_table_line {
DocumentBlockKind::Table
DocumentBlockKind::Paragraph
let mut spans = Vec::new();
block_index += 1;
current_blocks.push(DocumentBlock {
block_id: block_id(source_path, ¤t_heading, block_index),
kind,
text: trimmed.to_string(),
excerpt: Some(trimmed.chars().take(200).collect()),
language: code_lang.clone().filter(|_| in_code_block),
spans: Vec::new(),
for caps in link_re.captures_iter(trimmed) {
let label = caps.get(1).unwrap().as_str().to_string();
let target = caps.get(2).unwrap().as_str().to_string();
let key = format!("{label}|{target}");
if seen_links.insert(key) {
links.push(DocumentLink {
label,
target: target.clone(),
spans.push(DocumentSpan {
text: target,
kind: "link".into(),
for caps in doi_re.captures_iter(trimmed) {
let target = caps.get(1).unwrap().as_str().to_string();
if seen_citations.insert(target.clone()) {
citations.push(DocumentCitation {
label: target.clone(),
citation_kind: "paper".into(),
title: None,
authors: Vec::new(),
venue: None,
year: None,
kind: "doi".into(),
for caps in code_ref_re.captures_iter(trimmed) {
let label = caps.get(1).unwrap().as_str().trim().to_string();
if !label.is_empty() && seen_entities.insert(label.clone()) {
entities.push(DocumentEntityCandidate {
label: label.clone(),
kind: DocumentEntityKind::CodeRef,
aliases: Vec::new(),
if !label.is_empty() && seen_code_candidates.insert(label.clone()) {
code_candidates.push(DocumentCodeCandidate {
candidate_kind: classify_entity_kind(&label),
text: label,
kind: "code_ref".into(),
for caps in entity_re.captures_iter(trimmed) {
if seen_entities.insert(label.clone()) {
kind: classify_entity_kind(caps.get(1).unwrap().as_str().trim()),
confidence: ConfidenceLevel::Inferred,
if trimmed.ends_with('.') && trimmed.len() > 20 {
claim_index += 1;
let (kind, modality, negated) = classify_claim(trimmed);
let subject = code_ref_re
.captures(trimmed)
.and_then(|caps| caps.get(1).map(|m| m.as_str().trim().to_string()))
.or_else(|| {
entity_re
let predicate = match modality {
ClaimModality::Must => Some("must".to_string()),
ClaimModality::Should => Some("should".to_string()),
ClaimModality::May => Some("may".to_string()),
ClaimModality::Is => Some("is".to_string()),
ClaimModality::Unknown => None,
let object = predicate.as_ref().and_then(|verb| {
let needle = format!(" {} ", verb);
trimmed
.to_ascii_lowercase()
.find(&needle)
.map(|idx| {
trimmed[idx + needle.len()..]
.trim_end_matches('.')
.trim()
claims.push(DocumentClaimCandidate {
claim_id: claim_id(source_path, trimmed, claim_index),
label: trimmed.to_string(),
modality,
subject,
predicate,
object,
negated,
if let Some(last) = current_blocks.last_mut() {
last.spans = spans;
let title = default_title(source_path, raw_text);
let mut metadata = DocumentMetadata {
title: Some(title.clone()),
..Default::default()
if let Some(first) = citations.first() {
metadata.doi = Some(first.target.clone());
CanonicalDocument {
doc_id: format!("canon::{}", short_hash(source_path)),
source_path: source_path.to_string(),
source_kind: source_kind.clone(),
detected_type: match source_kind {
SourceKind::Markdown => "markdown".into(),
SourceKind::Text => "text".into(),
SourceKind::Html => "html".into(),
SourceKind::NativeLight => "light".into(),
SourceKind::NativeArticle => "article".into(),
SourceKind::NativeBibtex => "bibtex".into(),
SourceKind::NativeCrossref => "crossref".into(),
SourceKind::NativeRfc => "rfc".into(),
SourceKind::NativePatent => "patent".into(),
other => format!("{:?}", other).to_lowercase(),
producer: producer.to_string(),
content_hash: content_hash(raw_text),
title,
plain_text: raw_text.to_string(),
metadata,
sections,
tables,
links,
citations,
entities,
claims,
code_candidates,
structured_origin: serde_json::json!({ "producer": producer }),
fn strip_html_tags(input: &str) -> String {
let tag_re = Regex::new(r"<[^>]+>").unwrap();
let heading_re = Regex::new(r"(?i)<h([1-6])[^>]*>(.*?)</h[1-6]>").unwrap();
let mut text = input.to_string();
for caps in heading_re.captures_iter(input) {
let level = caps.get(1).unwrap().as_str();
let body = tag_re.replace_all(caps.get(2).unwrap().as_str(), "");
text = text.replace(
caps.get(0).unwrap().as_str(),
&format!(
"\n{} {}\n",
"#".repeat(level.parse::<usize>().unwrap_or(1)),
body.trim()
),
tag_re.replace_all(&text, " ").to_string()
fn canonicalize_html_with_fallback(
fallback_producer: &str,
let text = strip_html_tags(raw_text);
let producer = if text.trim().is_empty() {
fallback_producer
producer
canonicalize_plain_text(source_path, SourceKind::Html, producer, &text)
fn canonicalize_binary_placeholder(
let fallback = format!(
"# Imported Document\n\nSource: {}\n\nThis document was detected and reserved for optional provider-based canonicalization.\n",
source_path
let text = if source_text.trim().is_empty() {
fallback
source_text.to_string()
canonicalize_plain_text(source_path, source_kind, producer, &text)
pub fn graphify_documents(documents: &[CanonicalDocument], namespace: &str) -> M1ndResult<Graph> {
let mut graph = Graph::with_capacity(documents.len() * 24, documents.len() * 48);
let mut entity_nodes = HashSet::new();
let mut citation_nodes = HashSet::new();
let mut binding_nodes = HashSet::new();
for document in documents {
let doc_id = format!("universal::{}::doc::{}", namespace, document.doc_id);
let doc_tags = [
"universal".to_string(),
format!("universal:type:{}", document.detected_type),
format!("namespace:{}", namespace),
format!("producer:{}", document.producer),
];
let doc_tag_refs: Vec<&str> = doc_tags.iter().map(String::as_str).collect();
let doc_node = graph.add_node(
&doc_id,
&document.title,
NodeType::File,
&doc_tag_refs,
0.0,
0.5,
)?;
graph.set_node_provenance(
doc_node,
NodeProvenanceInput {
source_path: Some(&document.source_path),
excerpt: Some(&document.title),
namespace: Some(namespace),
canonical: true,
for section in &document.sections {
let section_id = format!("universal::{}::{}", namespace, section.section_id);
let section_tags = [
"universal:section".to_string(),
format!("section:kind:{:?}", section.kind).to_lowercase(),
let section_refs: Vec<&str> = section_tags.iter().map(String::as_str).collect();
let section_node = graph.add_node(
§ion_id,
§ion.heading,
NodeType::Module,
§ion_refs,
0.4,
section_node,
line_start: section.provenance.line_start,
line_end: section.provenance.line_end,
excerpt: section.provenance.excerpt.as_deref(),
graph.add_edge(
"contains_section",
FiniteF32::ONE,
EdgeDirection::Forward,
false,
FiniteF32::new(0.8),
if let Some(parent_id) = §ion.parent_section_id {
let graph_parent_id = format!("universal::{}::{}", namespace, parent_id);
if let Some(parent_node) = graph.resolve_id(&graph_parent_id) {
parent_node,
"subsection_of",
FiniteF32::new(0.7),
FiniteF32::new(0.5),
for block in §ion.blocks {
let block_node_id = format!("universal::{}::{}", namespace, block.block_id);
let (node_type, relation, kind_tag) = match block.kind {
DocumentBlockKind::Code => {
(NodeType::Module, "contains_code", "universal:code")
DocumentBlockKind::Table => {
(NodeType::System, "contains_table", "universal:table")
_ => (NodeType::Concept, "contains_block", "universal:block"),
let block_tags = [
kind_tag.to_string(),
format!("confidence:{:?}", block.confidence).to_lowercase(),
let block_refs: Vec<&str> = block_tags.iter().map(String::as_str).collect();
let block_node = graph.add_node(
&block_node_id,
&block.text.chars().take(80).collect::<String>(),
node_type,
&block_refs,
0.3,
block_node,
line_start: block.provenance.line_start,
line_end: block.provenance.line_end,
excerpt: block.provenance.excerpt.as_deref(),
relation,
FiniteF32::new(0.9),
FiniteF32::new(0.6),
for table in &document.tables {
let table_id = format!("universal::{}::{}", namespace, table.table_id);
let table_tags = ["universal", "universal:table"];
let table_node = graph.add_node(
&table_id,
&format!("Table {}", table.table_id),
NodeType::System,
&table_tags,
0.35,
table_node,
line_start: table.provenance.line_start,
line_end: table.provenance.line_end,
excerpt: table.provenance.excerpt.as_deref(),
"contains_table",
FiniteF32::new(0.85),
for entity in &document.entities {
let entity_id = format!(
"universal::{}::entity::{}",
namespace,
slugify(&entity.label)
if entity_nodes.insert(entity_id.clone()) {
let tags = [
format!("entity:kind:{:?}", entity.kind).to_lowercase(),
format!("confidence:{:?}", entity.confidence).to_lowercase(),
let tag_refs: Vec<&str> = tags.iter().map(String::as_str).collect();
let entity_node = graph.add_node(
&entity_id,
&entity.label,
NodeType::Concept,
&tag_refs,
entity_node,
line_start: entity.provenance.line_start,
line_end: entity.provenance.line_end,
excerpt: entity.provenance.excerpt.as_deref(),
if let Some(entity_node) = graph.resolve_id(&entity_id) {
"declares_entity",
for citation in &document.citations {
let citation_id = format!(
"universal::{}::citation::{}",
slugify(&citation.target)
if citation_nodes.insert(citation_id.clone()) {
"universal:citation".to_string(),
format!("target:{}", citation.target),
let citation_node = graph.add_node(
&citation_id,
&citation.label,
NodeType::Reference,
0.25,
citation_node,
line_start: citation.provenance.line_start,
line_end: citation.provenance.line_end,
excerpt: citation.provenance.excerpt.as_deref(),
if let Some(citation_node) = graph.resolve_id(&citation_id) {
"references",
for link in &document.links {
let link_id = format!("universal::{}::link::{}", namespace, slugify(&link.target));
if graph.resolve_id(&link_id).is_none() {
let tags = ["universal".to_string(), "universal:link".to_string()];
let link_node = graph.add_node(
&link_id,
&link.label,
0.2,
link_node,
line_start: link.provenance.line_start,
line_end: link.provenance.line_end,
excerpt: link.provenance.excerpt.as_deref(),
if let Some(link_node) = graph.resolve_id(&link_id) {
"binds_to",
for claim in &document.claims {
let claim_id = format!("universal::{}::{}", namespace, claim.claim_id);
if graph.resolve_id(&claim_id).is_none() {
format!("confidence:{:?}", claim.confidence).to_lowercase(),
"universal:claim".to_string(),
format!("claim:kind:{:?}", claim.kind).to_lowercase(),
format!("claim:modality:{:?}", claim.modality).to_lowercase(),
let claim_node = graph.add_node(
&claim_id,
&claim.label.chars().take(80).collect::<String>(),
claim_node,
line_start: claim.provenance.line_start,
line_end: claim.provenance.line_end,
excerpt: claim.provenance.excerpt.as_deref(),
if let Some(claim_node) = graph.resolve_id(&claim_id) {
"declares_claim",
let excerpt = claim.provenance.excerpt.as_deref().unwrap_or(&claim.label);
if !excerpt.contains(&citation.target) && !excerpt.contains(&citation.label) {
"supports",
FiniteF32::new(0.4),
for candidate in &document.code_candidates {
let binding_id = format!(
"universal::{}::binding::{}",
slugify(&candidate.label)
if binding_nodes.insert(binding_id.clone()) {
"universal:binding".to_string(),
format!("candidate:{:?}", candidate.candidate_kind).to_lowercase(),
let binding_node = graph.add_node(
&binding_id,
&candidate.label,
binding_node,
line_start: candidate.provenance.line_start,
line_end: candidate.provenance.line_end,
excerpt: candidate.provenance.excerpt.as_deref(),
if let Some(binding_node) = graph.resolve_id(&binding_id) {
"mentions_symbol",
FiniteF32::new(0.75),
if graph.num_nodes() > 0 {
graph.finalize()?;
Ok(graph)
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn canonicalizes_markdown_without_l1ght() {
let doc = canonicalize_plain_text(
"docs/example.md",
SourceKind::Markdown,
"test",
"# API\n\n`TokenValidator` must validate requests.\n\nSee [Docs](https://example.com).\n\n`TokenValidator`\n",
assert_eq!(doc.detected_type, "markdown");
assert!(!doc.sections.is_empty());
assert!(matches!(doc.sections[0].kind, DocumentSectionKind::Api));
assert!(!doc.links.is_empty());
assert!(!doc.entities.is_empty());
assert!(!doc.claims.is_empty());
assert!(!doc.code_candidates.is_empty());
assert!(matches!(doc.claims[0].kind, DocumentClaimKind::Requirement));
assert!(matches!(doc.claims[0].modality, ClaimModality::Must));
fn graphify_documents_creates_sections_entities_and_refs() {
"# Overview\n\nHello World.\n\nSee [Docs](https://example.com).\n\n`TokenValidator`\n10.1000/test\n",
let graph = graphify_documents(&[doc], "universal").unwrap();
assert!(graph.num_nodes() >= 6);
assert!(graph.num_edges() >= 5);
fn canonicalize_extracts_simple_tables() {
"docs/table.md",
"# Overview\n\n| Name | Value |\n| A | B |\n",
assert!(!doc.tables.is_empty());
assert_eq!(doc.tables[0].rows.len(), 1);
fn graphify_documents_emits_code_table_and_subsection_edges() {
"docs/semantic.md",
"# API\n\n```rust\nfn validate() {}\n```\n\n## Tables\n\n| Name | Value |\n| A | B |\n",
let mut contains_code = 0;
let mut contains_table = 0;
let mut subsection_of = 0;
for src in 0..graph.num_nodes() as usize {
for edge_idx in graph
.csr
.out_range(m1nd_core::types::NodeId::new(src as u32))
let rel = graph.strings.resolve(graph.csr.relations[edge_idx]);
match rel {
"contains_code" => contains_code += 1,
"contains_table" => contains_table += 1,
"subsection_of" => subsection_of += 1,
_ => {}
assert!(contains_code >= 1);
assert!(contains_table >= 1);
assert!(subsection_of >= 1);
fn claim_support_edges_require_local_citation_evidence() {
"docs/evidence.md",
"# API\n\nTokenValidator must validate requests.\n\n10.1000/alpha\n10.1000/beta\n",
let mut support_edges = 0;
if rel == "supports" {
support_edges += 1;
assert_eq!(support_edges, 0);
fn provider_probe_is_stable() {
let providers = UniversalIngestAdapter::provider_availability();
let _ = serde_json::to_string(&providers).unwrap();
fn content_hash_tracks_original_source_bytes_for_html_documents() {
let temp = std::env::temp_dir().join(format!(
"m1nd-universal-hash-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
std::fs::create_dir_all(&temp).unwrap();
let file = temp.join("page.html");
let raw = "<html><body><h1>Hash Title</h1><p>TokenValidator must validate requests.</p></body></html>";
std::fs::write(&file, raw).unwrap();
let adapter = UniversalIngestAdapter::new(Some("universal".into()));
let bundle = adapter.ingest_bundle(&file).unwrap();
let document = bundle.documents.first().unwrap();
assert_eq!(document.content_hash, short_hash_bytes(raw.as_bytes()));
assert_ne!(document.content_hash, short_hash(&document.plain_text));
let _ = std::fs::remove_dir_all(&temp);
