use serde::{Deserialize, Serialize}; use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; use std::path::Path; #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum SourceKind { NativeLight, NativeArticle, NativeBibtex, NativeCrossref, NativeRfc, NativePatent, Markdown, Text, Html, Pdf, Docx, Pptx, Xlsx, Unknown, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Default)] #[serde(rename_all = "snake_case")] pub enum ConfidenceLevel { Explicit, #[default] Parsed, Inferred, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum DocumentBlockKind { Heading, Paragraph, ListItem, Code, Quote, Table, Link, Text, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Default)] #[serde(rename_all = "snake_case")] pub enum DocumentSectionKind { Overview, Api, Constraints, Tests, Rollout, Reference, Appendix, #[default] Unknown, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Default)] #[serde(rename_all = "snake_case")] pub enum DocumentEntityKind { Symbol, FilePath, ToolId, CitationId, CodeRef, NamedTerm, TestName, #[default] Unknown, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Default)] #[serde(rename_all = "snake_case")] pub enum DocumentClaimKind { Requirement, Invariant, Warning, Decision, TestExpectation, Capability, #[default] Unknown, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Default)] #[serde(rename_all = "snake_case")] pub enum ClaimModality { Must, Should, May, Is, #[default] Unknown, } #[derive(Clone, Debug, Serialize, Deserialize, Default)] pub struct DocumentMetadata { pub title: Option, pub subtitle: Option, pub authors: Vec, pub doi: Option, pub published_at: Option, pub language: Option, pub extra: std::collections::HashMap, } #[derive(Clone, Debug, Serialize, Deserialize, Default)] pub struct ProvenanceSpan { pub line_start: Option, pub line_end: Option, pub excerpt: Option, } #[derive(Clone, Debug, Serialize, Deserialize, Default)] pub struct DocumentSpan { pub text: String, pub kind: String, pub confidence: ConfidenceLevel, pub provenance: ProvenanceSpan, } #[derive(Clone, Debug, Serialize, Deserialize)] pub struct DocumentBlock { pub block_id: String, pub kind: DocumentBlockKind, pub text: String, pub confidence: ConfidenceLevel, pub provenance: ProvenanceSpan, #[serde(default)] pub language: Option, #[serde(default)] pub spans: Vec, } #[derive(Clone, Debug, Serialize, Deserialize)] pub struct DocumentSection { pub section_id: String, pub heading: String, pub level: u8, #[serde(default)] pub kind: DocumentSectionKind, #[serde(default)] pub parent_section_id: Option, pub blocks: Vec, pub provenance: ProvenanceSpan, } #[derive(Clone, Debug, Serialize, Deserialize, Default)] pub struct DocumentTableCell { pub text: String, pub provenance: ProvenanceSpan, } #[derive(Clone, Debug, Serialize, Deserialize, Default)] pub struct DocumentTableRow { pub cells: Vec, } #[derive(Clone, Debug, Serialize, Deserialize, Default)] pub struct DocumentTable { pub table_id: String, pub headers: Vec, pub rows: Vec, pub confidence: ConfidenceLevel, pub provenance: ProvenanceSpan, } #[derive(Clone, Debug, Serialize, Deserialize)] pub struct DocumentLink { pub label: String, pub target: String, pub confidence: ConfidenceLevel, pub provenance: ProvenanceSpan, } #[derive(Clone, Debug, Serialize, Deserialize)] pub struct DocumentCitation { pub label: String, pub target: String, #[serde(default)] pub citation_kind: String, #[serde(default)] pub title: Option, #[serde(default)] pub authors: Vec, #[serde(default)] pub venue: Option, #[serde(default)] pub year: Option, pub confidence: ConfidenceLevel, pub provenance: ProvenanceSpan, } #[derive(Clone, Debug, Serialize, Deserialize)] pub struct DocumentEntityCandidate { pub label: String, pub kind: DocumentEntityKind, #[serde(default)] pub aliases: Vec, pub confidence: ConfidenceLevel, pub provenance: ProvenanceSpan, } #[derive(Clone, Debug, Serialize, Deserialize)] pub struct DocumentClaimCandidate { pub claim_id: String, pub label: String, #[serde(default)] pub kind: DocumentClaimKind, #[serde(default)] pub modality: ClaimModality, #[serde(default)] pub subject: Option, #[serde(default)] pub predicate: Option, #[serde(default)] pub object: Option, #[serde(default)] pub negated: bool, pub confidence: ConfidenceLevel, pub provenance: ProvenanceSpan, } #[derive(Clone, Debug, Serialize, Deserialize, Default)] pub struct DocumentCodeCandidate { pub label: String, pub candidate_kind: DocumentEntityKind, pub confidence: ConfidenceLevel, pub provenance: ProvenanceSpan, } #[derive(Clone, Debug, Serialize, Deserialize)] pub struct CanonicalDocument { pub doc_id: String, pub source_path: String, pub source_kind: SourceKind, pub detected_type: String, pub producer: String, pub content_hash: String, pub title: String, pub plain_text: String, pub metadata: DocumentMetadata, pub sections: Vec, #[serde(default)] pub tables: Vec, pub links: Vec, pub citations: Vec, pub entities: Vec, pub claims: Vec, #[serde(default)] pub code_candidates: Vec, pub confidence: ConfidenceLevel, #[serde(default)] pub structured_origin: serde_json::Value, } pub fn short_hash(input: &str) -> String { let mut hasher = DefaultHasher::new(); input.hash(&mut hasher); format!("{:016x}", hasher.finish()) } pub fn short_hash_bytes(input: &[u8]) -> String { let mut hasher = DefaultHasher::new(); input.hash(&mut hasher); format!("{:016x}", hasher.finish()) } pub fn source_key(source_path: &str) -> String { short_hash(source_path) } pub fn source_kind_from_extension(path: &Path) -> SourceKind { match path .extension() .and_then(|value| value.to_str()) .map(|value| value.to_ascii_lowercase()) .as_deref() { Some("md" | "markdown") => SourceKind::Markdown, Some("txt" | "rst" | "adoc") => SourceKind::Text, Some("html" | "htm") => SourceKind::Html, Some("pdf") => SourceKind::Pdf, Some("docx") => SourceKind::Docx, Some("pptx") => SourceKind::Pptx, Some("xlsx") => SourceKind::Xlsx, _ => SourceKind::Unknown, } }

m1nd-ingest/src/canonical.rs