mos_eval/
dependency.rs

1//! External files a lowering read, with the fingerprint each had at the
2//! time, so a cached [`LowerResult`](crate::LowerResult) can be checked
3//! against the filesystem before it is reused (issue #125).
4
5use std::collections::BTreeMap;
6use std::fs::Metadata;
7use std::io;
8use std::path::{Path, PathBuf};
9use std::sync::{Arc, OnceLock};
10use std::time::{Duration, SystemTime};
11
12use mos_core::{ContentHash, ContentHasher};
13
14const DOMAIN_TAG: &[u8] = b"mos-eval/external-dependency/v1";
15
16/// A modification time this close to the moment the fingerprint was taken is
17/// not trusted by [`ExternalDependency::is_current`]: a second write inside
18/// the same filesystem timestamp tick would leave `stat` unchanged. Two
19/// seconds covers FAT's timestamp resolution.
20pub const RACY_WINDOW: Duration = Duration::from_secs(2);
21
22/// What a regular file looked like when it was read: its `stat` fields, the
23/// moment that `stat` was taken, and a [`fingerprint_bytes`] hash of its
24/// contents.
25#[derive(Clone, Copy, Debug, PartialEq, Eq)]
26pub struct Fingerprint {
27    /// File size in bytes.
28    pub len: u64,
29    /// Modification time, when the platform reports one.
30    pub modified: Option<SystemTime>,
31    /// Inode-level identity; all fields are `None` off Unix.
32    pub identity: FileIdentity,
33    /// When the `stat` was taken.
34    pub observed: SystemTime,
35    /// Hash of the bytes that were read.
36    pub content: ContentHash,
37}
38
39/// The device and inode numbers and the status-change time of a file on
40/// Unix. Unlike the modification time, none of these can be set from
41/// userspace, so a rewrite that restores the old mtime still moves one of
42/// them. Every field is `None` on platforms that do not expose them.
43#[derive(Clone, Copy, Debug, PartialEq, Eq)]
44pub struct FileIdentity {
45    /// Device number.
46    pub device: Option<u64>,
47    /// Inode number.
48    pub inode: Option<u64>,
49    /// Status-change time (`ctime`), when it is representable.
50    pub changed: Option<SystemTime>,
51}
52
53impl FileIdentity {
54    /// The identity of a file on a platform that exposes none of the fields.
55    pub const NONE: Self = Self {
56        device: None,
57        inode: None,
58        changed: None,
59    };
60
61    #[cfg(unix)]
62    fn of(metadata: &Metadata) -> Self {
63        use std::os::unix::fs::MetadataExt;
64
65        let changed = u64::try_from(metadata.ctime())
66            .ok()
67            .zip(u32::try_from(metadata.ctime_nsec()).ok())
68            .and_then(|(secs, nanos)| {
69                SystemTime::UNIX_EPOCH.checked_add(Duration::new(secs, nanos))
70            });
71        Self {
72            device: Some(metadata.dev()),
73            inode: Some(metadata.ino()),
74            changed,
75        }
76    }
77
78    #[cfg(not(unix))]
79    fn of(_metadata: &Metadata) -> Self {
80        Self::NONE
81    }
82}
83
84impl Fingerprint {
85    fn stat_matches(&self, metadata: &Metadata) -> bool {
86        metadata.len() == self.len
87            && metadata.modified().ok() == self.modified
88            && FileIdentity::of(metadata) == self.identity
89    }
90
91    /// Whether the file was modified within [`RACY_WINDOW`] of the moment the
92    /// fingerprint was taken, or has no modification time at all, so a
93    /// matching `stat` is not proof that the contents are unchanged.
94    #[must_use]
95    pub fn is_racy(&self) -> bool {
96        let Some(modified) = self.modified else {
97            return true;
98        };
99        !self
100            .observed
101            .duration_since(modified)
102            .is_ok_and(|age| age >= RACY_WINDOW)
103    }
104}
105
106/// A resource content hash, optionally accompanied by filesystem freshness metadata.
107#[derive(Clone, Copy, Debug, PartialEq, Eq)]
108pub enum ResourceFingerprint {
109    /// A regular file read by [`FileSystemReader`].
110    File(Fingerprint),
111    /// Caller-supplied bytes with no filesystem identity or timestamps.
112    Content(ContentHash),
113}
114
115impl ResourceFingerprint {
116    /// Hash of the exact bytes consumed by lowering.
117    #[must_use]
118    pub const fn content(self) -> ContentHash {
119        match self {
120            Self::File(file) => file.content,
121            Self::Content(content) => content,
122        }
123    }
124}
125
126/// Bytes returned by a [`ResourceReader`].
127#[derive(Clone, Debug)]
128pub struct ResourceData {
129    bytes: Arc<[u8]>,
130    text_index: OnceLock<Option<mos_core::LineIndex>>,
131    fingerprint: ResourceFingerprint,
132}
133
134impl ResourceData {
135    /// Capture supplied bytes without consulting the filesystem.
136    #[must_use]
137    pub fn new(bytes: impl Into<Arc<[u8]>>) -> Self {
138        let bytes = bytes.into();
139        let fingerprint = ResourceFingerprint::Content(fingerprint_bytes(&bytes));
140        Self {
141            bytes,
142            fingerprint,
143            text_index: OnceLock::new(),
144        }
145    }
146}
147
148/// Supplies image and bibliography bytes at already-resolved source paths.
149///
150/// Each path is read at most once per lowering, including failed reads. Implementors
151/// may use memory, archives, or editor overlays; they need no filesystem metadata.
152/// The reader is synchronous and need not implement `Send` or `Sync`.
153pub trait ResourceReader {
154    /// Read a resolved path, or return the underlying resource error.
155    ///
156    /// # Errors
157    ///
158    /// Return `NotFound` for absent resources, `InvalidInput` for paths that do
159    /// not identify readable resources (such as directories), and an appropriate
160    /// I/O error for other failures. Lowering preserves the error in its snapshot.
161    fn read(&self, path: &Path) -> io::Result<ResourceData>;
162}
163
164impl<F> ResourceReader for F
165where
166    F: Fn(&Path) -> io::Result<ResourceData>,
167{
168    fn read(&self, path: &Path) -> io::Result<ResourceData> {
169        self(path)
170    }
171}
172
173/// Default resource reader, restricted to regular files.
174#[derive(Clone, Copy, Debug, Default)]
175pub struct FileSystemReader;
176
177impl ResourceReader for FileSystemReader {
178    fn read(&self, path: &Path) -> io::Result<ResourceData> {
179        let (bytes, fingerprint) = read_fingerprinted(path)?;
180        Ok(ResourceData {
181            bytes: bytes.into(),
182            text_index: OnceLock::new(),
183            fingerprint: ResourceFingerprint::File(fingerprint),
184        })
185    }
186}
187
188/// Captured external inputs belonging to one lowering result.
189///
190/// Bytes and failures are retained so consumers interpret external source spans
191/// against the same version used by the compiler. This is a per-path snapshot,
192/// not an atomic snapshot of the whole filesystem.
193#[derive(Debug, Default)]
194pub struct ResourceSnapshot {
195    entries: BTreeMap<PathBuf, io::Result<ResourceData>>,
196}
197
198impl ResourceSnapshot {
199    /// Lazily index a captured UTF-8 resource. Repeated lookups reuse the index.
200    /// Returns `None` for absent, failed, or non-UTF-8 reads; [`Self::get`] retains
201    /// the original bytes or I/O error. Never rereads the filesystem.
202    #[must_use]
203    pub fn text_index(&self, path: &Path) -> Option<&mos_core::LineIndex> {
204        let data = self.entries.get(path)?.as_ref().ok()?;
205        data.text_index
206            .get_or_init(|| {
207                std::str::from_utf8(&data.bytes)
208                    .ok()
209                    .map(mos_core::LineIndex::new)
210            })
211            .as_ref()
212    }
213
214    /// Look up a captured read. `None` means the path was never requested.
215    #[must_use]
216    pub fn get(&self, path: &Path) -> Option<Result<&[u8], &io::Error>> {
217        self.entries
218            .get(path)
219            .map(|result| result.as_ref().map(|data| data.bytes.as_ref()))
220    }
221
222    pub(crate) fn read(
223        &mut self,
224        reader: &dyn ResourceReader,
225        path: &Path,
226    ) -> Result<&[u8], &io::Error> {
227        self.entries
228            .entry(path.to_path_buf())
229            .or_insert_with(|| reader.read(path))
230            .as_ref()
231            .map(|data| data.bytes.as_ref())
232    }
233
234    pub(crate) fn dependencies(&self) -> Vec<ExternalDependency> {
235        self.entries
236            .iter()
237            .map(|(path, result)| ExternalDependency {
238                path: path.clone(),
239                fingerprint: result.as_ref().ok().map(|data| data.fingerprint),
240            })
241            .collect()
242    }
243}
244
245/// One external file a lowering depended on: an `#image` / `#figure` raster or
246/// a `#bibliography` source.
247///
248/// `fingerprint` is `None` when the reader failed. Successful reads carry
249/// a content hash and, for the filesystem adapter, file freshness metadata.
250///
251/// # Examples
252///
253/// ```
254/// use std::path::Path;
255///
256/// use mos_eval::ExternalDependency;
257///
258/// let dep = ExternalDependency::observe(Path::new("/nonexistent/x.png"));
259/// assert_eq!(dep.fingerprint, None);
260/// assert!(dep.is_current());
261/// ```
262#[derive(Clone, Debug, PartialEq, Eq)]
263pub struct ExternalDependency {
264    /// The resolved filesystem path that was read.
265    pub path: PathBuf,
266    /// The resource fingerprint, or `None` when it could not be read.
267    pub fingerprint: Option<ResourceFingerprint>,
268}
269
270impl ExternalDependency {
271    /// Record `path` with the fingerprint it has right now.
272    #[must_use]
273    pub fn observe(path: impl Into<PathBuf>) -> Self {
274        let path = path.into();
275        let fingerprint = fingerprint_file(&path).map(ResourceFingerprint::File);
276        Self { path, fingerprint }
277    }
278
279    /// Whether the file on disk still matches the recorded fingerprint.
280    /// Caller-supplied content fingerprints conservatively return false; this
281    /// method checks the filesystem, not a custom reader or overlay.
282    ///
283    /// A `stat` settles the common cases: a file whose size, modification
284    /// time, and [`FileIdentity`] are unchanged is current, and a file that
285    /// appeared or vanished is not. The bytes are re-read and hashed only when
286    /// the `stat` differs or the fingerprint [is racy](Fingerprint::is_racy),
287    /// so a `touch` that leaves the contents alone still counts as current,
288    /// while a same-size rewrite that restores the old mtime is caught by the
289    /// inode identity on Unix and by the racy window everywhere.
290    #[must_use]
291    pub fn is_current(&self) -> bool {
292        let Some(recorded) = &self.fingerprint else {
293            return regular_file_metadata(&self.path).is_none();
294        };
295        let ResourceFingerprint::File(recorded) = recorded else {
296            return false;
297        };
298        let Some(metadata) = regular_file_metadata(&self.path) else {
299            return false;
300        };
301        if recorded.stat_matches(&metadata) && !recorded.is_racy() {
302            return true;
303        }
304        fingerprint_file(&self.path).is_some_and(|now| now.content == recorded.content)
305    }
306}
307
308/// Fingerprint raw file bytes for dependency comparison.
309///
310/// # Examples
311///
312/// ```
313/// use mos_eval::fingerprint_bytes;
314///
315/// assert_eq!(fingerprint_bytes(b"a"), fingerprint_bytes(b"a"));
316/// assert_ne!(fingerprint_bytes(b"a"), fingerprint_bytes(b"b"));
317/// ```
318#[must_use]
319pub fn fingerprint_bytes(bytes: &[u8]) -> ContentHash {
320    let mut hasher = ContentHasher::new();
321    hasher.field(DOMAIN_TAG).field(bytes);
322    hasher.finish()
323}
324
325/// The [`Fingerprint`] of the regular file at `path`, or `None` when there is
326/// no readable regular file there. Directories, devices, and pipes are never
327/// opened.
328#[must_use]
329pub fn fingerprint_file(path: &Path) -> Option<Fingerprint> {
330    read_fingerprinted(path)
331        .ok()
332        .map(|(_, fingerprint)| fingerprint)
333}
334
335/// Read the regular file at `path` and fingerprint it in one pass. The
336/// `stat` is taken before the read, so a write that lands between the two
337/// shows up as a changed `stat` on the next [`ExternalDependency::is_current`]
338/// and forces a hash comparison.
339pub(crate) fn read_fingerprinted(path: &Path) -> io::Result<(Vec<u8>, Fingerprint)> {
340    let metadata = std::fs::metadata(path)?;
341    if !metadata.is_file() {
342        return Err(io::Error::new(
343            io::ErrorKind::InvalidInput,
344            "not a regular file",
345        ));
346    }
347    let observed = SystemTime::now();
348    let bytes = std::fs::read(path)?;
349    let fingerprint = Fingerprint {
350        len: metadata.len(),
351        modified: metadata.modified().ok(),
352        identity: FileIdentity::of(&metadata),
353        observed,
354        content: fingerprint_bytes(&bytes),
355    };
356    Ok((bytes, fingerprint))
357}
358
359fn regular_file_metadata(path: &Path) -> Option<Metadata> {
360    std::fs::metadata(path).ok().filter(Metadata::is_file)
361}
362
363/// Resource access for a directive, using the enclosing lowering's snapshot.
364pub(crate) struct ExternalInputs<'a> {
365    pub(crate) source_file: &'a Path,
366    pub(crate) reader: &'a dyn ResourceReader,
367    pub(crate) resources: &'a mut ResourceSnapshot,
368}
369
370#[cfg(test)]
371mod tests {
372    use super::*;
373
374    fn unique_temp_file(name: &str) -> PathBuf {
375        let dir = std::env::temp_dir().join(format!(
376            "mos-eval-dependency-{name}-{}-{}",
377            std::process::id(),
378            SystemTime::now()
379                .duration_since(std::time::UNIX_EPOCH)
380                .map_or(0, |d| d.as_nanos())
381        ));
382        std::fs::create_dir_all(&dir).expect("temp dir");
383        dir.join("file.bin")
384    }
385
386    fn cleanup(path: &Path) {
387        std::fs::remove_dir_all(path.parent().expect("parent")).ok();
388    }
389
390    #[test]
391    fn captured_text_index_is_lazy_reused_and_revision_bound() {
392        let path = Path::new("/virtual/refs.bib");
393        let mut snapshot = ResourceSnapshot::default();
394        let reader = |_: &Path| Ok(ResourceData::new("😀\nold".as_bytes()));
395        assert!(snapshot.read(&reader, path).is_ok());
396        let first = snapshot.text_index(path).expect("UTF-8 resource");
397        let second = snapshot.text_index(path).expect("same captured resource");
398        assert!(std::ptr::eq(first, second));
399        assert_eq!(first.utf16_position(5), (1, 0));
400        let mut next = ResourceSnapshot::default();
401        let changed = |_: &Path| Ok(ResourceData::new(b"new".as_slice()));
402        assert!(next.read(&changed, path).is_ok());
403        assert_eq!(
404            next.text_index(path).map(mos_core::LineIndex::text),
405            Some("new")
406        );
407        assert_eq!(
408            snapshot.text_index(path).map(mos_core::LineIndex::text),
409            Some("😀\nold")
410        );
411        let invalid = Path::new("/virtual/image");
412        assert!(
413            snapshot
414                .read(
415                    &|_: &Path| Ok(ResourceData::new([0xff].as_slice())),
416                    invalid
417                )
418                .is_ok()
419        );
420        assert!(snapshot.text_index(invalid).is_none());
421        assert!(snapshot.text_index(Path::new("/virtual/absent")).is_none());
422    }
423
424    #[test]
425    fn missing_file_has_no_fingerprint_and_is_current_while_still_missing() {
426        let path = unique_temp_file("missing");
427        assert_eq!(fingerprint_file(&path), None);
428        let dep = ExternalDependency::observe(&path);
429        assert!(dep.is_current());
430        std::fs::write(&path, b"x").expect("write");
431        assert!(!dep.is_current(), "appearing counts as a change");
432        cleanup(&path);
433    }
434
435    #[test]
436    fn directory_is_never_read() {
437        let path = unique_temp_file("dir");
438        let dir = path.parent().expect("parent");
439        assert_eq!(fingerprint_file(dir), None);
440        assert!(ExternalDependency::observe(dir).is_current());
441        cleanup(&path);
442    }
443
444    #[test]
445    fn same_bytes_fingerprint_equal_and_different_bytes_diverge() {
446        let path = unique_temp_file("bytes");
447        std::fs::write(&path, b"one").expect("write");
448        assert_eq!(
449            fingerprint_file(&path).map(|f| f.content),
450            Some(fingerprint_bytes(b"one"))
451        );
452        let dep = ExternalDependency::observe(&path);
453        std::fs::write(&path, b"two").expect("write");
454        assert!(!dep.is_current());
455        std::fs::remove_file(&path).expect("remove");
456        assert!(!dep.is_current(), "disappearing counts as a change");
457        cleanup(&path);
458    }
459
460    #[test]
461    fn unchanged_contents_stay_current_when_only_the_stat_moved() {
462        let path = unique_temp_file("touch");
463        std::fs::write(&path, b"same").expect("write");
464        let recorded = fingerprint_file(&path).expect("fingerprint");
465        let moved = ExternalDependency {
466            path: path.clone(),
467            fingerprint: Some(ResourceFingerprint::File(Fingerprint {
468                modified: Some(SystemTime::UNIX_EPOCH),
469                ..recorded
470            })),
471        };
472        assert!(moved.is_current(), "a stat mismatch falls back to the hash");
473        let stale = ExternalDependency {
474            path: path.clone(),
475            fingerprint: Some(ResourceFingerprint::File(Fingerprint {
476                modified: Some(SystemTime::UNIX_EPOCH),
477                content: fingerprint_bytes(b"other"),
478                ..recorded
479            })),
480        };
481        assert!(!stale.is_current());
482        cleanup(&path);
483    }
484
485    #[test]
486    fn a_fingerprint_taken_right_after_the_write_is_racy_and_always_hashed() {
487        let path = unique_temp_file("racy");
488        std::fs::write(&path, b"same").expect("write");
489        let recorded = fingerprint_file(&path).expect("fingerprint");
490        assert!(recorded.is_racy());
491
492        let lying = ExternalDependency {
493            path: path.clone(),
494            fingerprint: Some(ResourceFingerprint::File(Fingerprint {
495                content: fingerprint_bytes(b"other"),
496                ..recorded
497            })),
498        };
499        assert!(
500            !lying.is_current(),
501            "a matching stat inside the racy window is not trusted"
502        );
503
504        let settled = Fingerprint {
505            observed: recorded.observed + RACY_WINDOW * 2,
506            ..recorded
507        };
508        assert!(!settled.is_racy());
509        let trusted = ExternalDependency {
510            path: path.clone(),
511            fingerprint: Some(ResourceFingerprint::File(Fingerprint {
512                content: fingerprint_bytes(b"other"),
513                ..settled
514            })),
515        };
516        assert!(
517            trusted.is_current(),
518            "outside the racy window a matching stat is proof enough"
519        );
520        cleanup(&path);
521    }
522
523    #[test]
524    fn a_fingerprint_without_mtime_is_racy() {
525        let fingerprint = Fingerprint {
526            len: 0,
527            modified: None,
528            identity: FileIdentity::NONE,
529            observed: SystemTime::UNIX_EPOCH,
530            content: ContentHash(0),
531        };
532        assert!(fingerprint.is_racy());
533    }
534
535    #[cfg(unix)]
536    #[test]
537    fn same_size_replacement_that_restores_the_mtime_moves_the_identity() {
538        let path = unique_temp_file("identity");
539        std::fs::write(&path, b"aaaa").expect("write");
540        let recorded = fingerprint_file(&path).expect("fingerprint");
541        let settled = ExternalDependency {
542            path: path.clone(),
543            fingerprint: Some(ResourceFingerprint::File(Fingerprint {
544                observed: recorded.observed + RACY_WINDOW * 2,
545                ..recorded
546            })),
547        };
548        assert!(settled.is_current());
549
550        let staging = path.with_extension("new");
551        std::fs::write(&staging, b"bbbb").expect("write replacement");
552        std::fs::File::options()
553            .write(true)
554            .open(&staging)
555            .expect("open replacement")
556            .set_modified(recorded.modified.expect("mtime"))
557            .expect("restore mtime");
558        std::fs::rename(&staging, &path).expect("swap in");
559        let metadata = std::fs::metadata(&path).expect("stat");
560        assert_eq!(metadata.len(), recorded.len);
561        assert_eq!(metadata.modified().ok(), recorded.modified);
562        assert_ne!(FileIdentity::of(&metadata), recorded.identity);
563        assert!(
564            !settled.is_current(),
565            "a new inode falls through the stat gate to the hash"
566        );
567        cleanup(&path);
568    }
569}
mos_eval/dependency.rs

mos_eval/
dependency.rs
