Merge branch 'buffer-implementation'HEADmain

Establishes the first working baseline: moji <file> opens a file into a
ropey rope and edits it with Helix selection-first modal editing, under a
DO-178C DAL-C requirements and traceability process.

Prior to this, main tracked four files and src/main.rs was still
println!("Hello, world!") — there was no buildable state to build on.

Verified on a fresh clone of the branch with no untracked files:
cargo build; clippy --all-targets -D warnings clean; 294 tests passing;
scripts/check-trace.sh reports 98/98 requirements traced in both
directions.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

1 files changed, 272 insertions, 0 deletions

diff --git a/src/buffer/encoding.rs b/src/buffer/encoding.rs
new file mode 100644
index 0000000..5cd852f
--- /dev/null
+++ b/src/buffer/encoding.rs
@@ -0,0 +1,272 @@
+//! Character encoding detection and transcoding (MJB-HLR-002).
+//!
+//! Two rules, the second an exception to the first:
+//!
+//! 1. **A file whose encoding a byte order mark declares is transcoded.** The
+//!    encoding is known, so its bytes round-trip through load and save.
+//! 2. **UTF-8 is strict.** A file assumed or declared to be UTF-8 that holds an
+//!    invalid byte sequence is *rejected*, not repaired.
+//!
+//! Rule 2 exists because substitution is lossy in a way the user cannot see:
+//! replacing a bad byte with U+FFFD and then saving writes the replacement
+//! character over their data. For a declared encoding we can at least reproduce
+//! what we read; for malformed UTF-8 we cannot, so refusing to open is the only
+//! non-destructive answer. See MJB-DR-001.
+
+use encoding_rs::{Encoding, UTF_8, UTF_16BE, UTF_16LE};
+
+/// MJB-LLR-118: why a file could not be decoded.
+#[derive(Debug, thiserror::Error, PartialEq, Eq)]
+pub enum DecodeError {
+    #[error(
+        "not valid UTF-8 (invalid byte sequence at offset {valid_up_to}); \
+         mojibake edits text, and repairing the bytes would destroy them on save"
+    )]
+    InvalidUtf8 { valid_up_to: usize },
+}
+
+/// The encoding a document was loaded with, plus whether it carried a BOM.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct EncodingInfo {
+    pub encoding: &'static Encoding,
+    pub has_bom: bool,
+}
+
+impl Default for EncodingInfo {
+    fn default() -> Self {
+        Self {
+            encoding: UTF_8,
+            has_bom: false,
+        }
+    }
+}
+
+/// MJB-LLR-110: recognise a byte order mark, returning the encoding it implies
+/// and the mark's length in bytes.
+pub fn detect_bom(bytes: &[u8]) -> Option<(&'static Encoding, usize)> {
+    if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) {
+        Some((UTF_8, 3))
+    } else if bytes.starts_with(&[0xFF, 0xFE]) {
+        Some((UTF_16LE, 2))
+    } else if bytes.starts_with(&[0xFE, 0xFF]) {
+        Some((UTF_16BE, 2))
+    } else {
+        None
+    }
+}
+
+/// The byte order mark for `encoding`, if it has one.
+pub fn bom_bytes(encoding: &'static Encoding) -> &'static [u8] {
+    if encoding == UTF_8 {
+        &[0xEF, 0xBB, 0xBF]
+    } else if encoding == UTF_16LE {
+        &[0xFF, 0xFE]
+    } else if encoding == UTF_16BE {
+        &[0xFE, 0xFF]
+    } else {
+        &[]
+    }
+}
+
+/// MJB-LLR-111, MJB-LLR-113, MJB-LLR-118: decode `bytes` to a `String`.
+///
+/// UTF-8 — whether declared by a BOM or merely assumed — is validated strictly
+/// and rejected when malformed. Other BOM-declared encodings are transcoded.
+pub fn decode(bytes: &[u8]) -> Result<(String, EncodingInfo), DecodeError> {
+    let (encoding, has_bom, body) = match detect_bom(bytes) {
+        Some((encoding, bom_len)) => (encoding, true, &bytes[bom_len..]),
+        None => (UTF_8, false, bytes),
+    };
+
+    let text = if encoding == UTF_8 {
+        // MJB-LLR-118: the exception. `from_utf8` reports exactly how far the
+        // input was valid, which makes the diagnostic actionable.
+        std::str::from_utf8(body)
+            .map_err(|e| DecodeError::InvalidUtf8 {
+                valid_up_to: e.valid_up_to(),
+            })?
+            .to_owned()
+    } else {
+        // MJB-LLR-113: a declared non-UTF-8 encoding is transcoded. Its bytes
+        // round-trip on save, so any substitution here is reproducible.
+        encoding.decode_without_bom_handling(body).0.into_owned()
+    };
+
+    Ok((text, EncodingInfo { encoding, has_bom }))
+}
+
+/// MJB-LLR-115: encode `text` back to bytes, re-emitting the BOM when the
+/// document was loaded with one.
+///
+/// Total by construction: `text` is a `str`, and a document can only hold text
+/// that [`decode`] accepted, so there is nothing here that can fail.
+///
+/// UTF-16 is encoded here by hand rather than through `encoding_rs`.
+/// `Encoding::encode` is deliberately asymmetric: it decodes UTF-16 but will
+/// not *encode* to it, silently substituting UTF-8 instead. Delegating would
+/// therefore write UTF-8 bytes beneath a UTF-16 BOM and corrupt the file.
+/// See MJB-DR-007.
+pub fn encode(text: &str, info: EncodingInfo) -> Vec<u8> {
+    let mut out = Vec::with_capacity(text.len() + 3);
+    if info.has_bom {
+        out.extend_from_slice(bom_bytes(info.encoding));
+    }
+
+    if info.encoding == UTF_16LE {
+        for unit in text.encode_utf16() {
+            out.extend_from_slice(&unit.to_le_bytes());
+        }
+    } else if info.encoding == UTF_16BE {
+        for unit in text.encode_utf16() {
+            out.extend_from_slice(&unit.to_be_bytes());
+        }
+    } else {
+        let (bytes, _, _) = info.encoding.encode(text);
+        out.extend_from_slice(&bytes);
+    }
+
+    out
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn mjb_llr_110_detects_each_bom() {
+        assert_eq!(detect_bom(&[0xEF, 0xBB, 0xBF, b'a']), Some((UTF_8, 3)));
+        assert_eq!(detect_bom(&[0xFF, 0xFE, b'a', 0]), Some((UTF_16LE, 2)));
+        assert_eq!(detect_bom(&[0xFE, 0xFF, 0, b'a']), Some((UTF_16BE, 2)));
+        assert_eq!(detect_bom(b"plain"), None);
+        assert_eq!(detect_bom(b""), None, "empty input must not index past end");
+    }
+
+    #[test]
+    fn mjb_llr_111_plain_utf8_round_trips() {
+        let (text, info) = decode("hello 文字化け".as_bytes()).unwrap();
+        assert_eq!(text, "hello 文字化け");
+        assert!(!info.has_bom);
+        assert_eq!(encode(&text, info), "hello 文字化け".as_bytes());
+    }
+
+    /// MJB-LLR-115, MJB-DR-001: a BOM present on load reappears on save.
+    #[test]
+    fn mjb_llr_115_bom_round_trips() {
+        let mut input = vec![0xEF, 0xBB, 0xBF];
+        input.extend_from_slice(b"hi");
+
+        let (text, info) = decode(&input).unwrap();
+        assert_eq!(text, "hi", "BOM is stripped from the buffer contents");
+        assert!(info.has_bom);
+        assert_eq!(encode(&text, info), input, "and re-emitted on write");
+    }
+
+    /// MJB-LLR-115, MJB-DR-007: UTF-16 must survive a load/save cycle.
+    ///
+    /// Regression guard: `encoding_rs::Encoding::encode` substitutes UTF-8 for
+    /// UTF-16 rather than failing, so delegating to it here would write UTF-8
+    /// bytes under a UTF-16 BOM and corrupt the file.
+    #[test]
+    fn mjb_llr_115_utf16le_round_trips() {
+        // UTF-16LE BOM followed by "hi".
+        let input = vec![0xFF, 0xFE, b'h', 0x00, b'i', 0x00];
+        let (text, info) = decode(&input).unwrap();
+        assert_eq!(text, "hi");
+        assert_eq!(info.encoding, UTF_16LE);
+        assert_eq!(encode(&text, info), input, "must not degrade to UTF-8");
+    }
+
+    #[test]
+    fn mjb_llr_115_utf16be_round_trips() {
+        let input = vec![0xFE, 0xFF, 0x00, b'h', 0x00, b'i'];
+        let (text, info) = decode(&input).unwrap();
+        assert_eq!(text, "hi");
+        assert_eq!(info.encoding, UTF_16BE);
+        assert_eq!(encode(&text, info), input);
+    }
+
+    #[test]
+    fn mjb_llr_115_utf16_handles_non_ascii_and_surrogates() {
+        // 文 is BMP; 𝄞 (U+1D11E) needs a surrogate pair in UTF-16.
+        let original = "文𝄞";
+        let info = EncodingInfo {
+            encoding: UTF_16LE,
+            has_bom: true,
+        };
+        let bytes = encode(original, info);
+        let (back, _) = decode(&bytes).unwrap();
+        assert_eq!(back, original);
+    }
+
+    /// MJB-LLR-113: a *declared* non-UTF-8 encoding is transcoded, not
+    /// rejected. An unpaired surrogate is repaired, and that repair is
+    /// reproducible because the encoding is known.
+    #[test]
+    fn mjb_llr_113_declared_utf16_is_transcoded_not_rejected() {
+        // UTF-16LE BOM, then a lone high surrogate (0xD800) — not valid UTF-16.
+        let input = vec![0xFF, 0xFE, 0x00, 0xD8, b'a', 0x00];
+        let (text, info) = decode(&input).expect("a declared encoding must not be rejected");
+        assert_eq!(info.encoding, UTF_16LE);
+        assert!(
+            text.contains('\u{FFFD}'),
+            "the unpaired surrogate becomes U+FFFD, got {text:?}"
+        );
+    }
+
+    /// MJB-LLR-118: the UTF-8 exception. Invalid UTF-8 is rejected outright
+    /// rather than repaired, because a repair would be written back over the
+    /// user's data on save.
+    #[test]
+    fn mjb_llr_118_invalid_utf8_is_rejected() {
+        // 0xFF is never valid in UTF-8.
+        let err = decode(&[b'a', 0xFF, b'b']).unwrap_err();
+        assert_eq!(err, DecodeError::InvalidUtf8 { valid_up_to: 1 });
+    }
+
+    #[test]
+    fn mjb_llr_118_rejection_names_the_offset() {
+        let err = decode(&[b'h', b'i', 0x80]).unwrap_err();
+        assert_eq!(err, DecodeError::InvalidUtf8 { valid_up_to: 2 });
+        assert!(
+            err.to_string().contains('2'),
+            "the message must locate the bad byte, got: {err}"
+        );
+    }
+
+    /// A truncated multi-byte character is invalid UTF-8 too.
+    #[test]
+    fn mjb_llr_118_truncated_multibyte_char_is_rejected() {
+        // 文 is E6 96 87; drop the last byte.
+        let err = decode(&[0xE6, 0x96]).unwrap_err();
+        assert_eq!(err, DecodeError::InvalidUtf8 { valid_up_to: 0 });
+    }
+
+    /// MJB-LLR-118: a UTF-8 *BOM* does not license invalid bytes after it.
+    #[test]
+    fn mjb_llr_118_declared_utf8_is_strict_too() {
+        let input = vec![0xEF, 0xBB, 0xBF, b'a', 0xFF];
+        let err = decode(&input).unwrap_err();
+        assert_eq!(
+            err,
+            DecodeError::InvalidUtf8 { valid_up_to: 1 },
+            "offset is measured past the BOM"
+        );
+    }
+
+    /// Valid multi-byte UTF-8 must not be mistaken for invalid.
+    #[test]
+    fn mjb_llr_118_valid_multibyte_utf8_is_accepted() {
+        for s in ["文字化け", "e\u{0301}", "𝄞", "café", "", "\u{FFFD}"] {
+            let (text, _) = decode(s.as_bytes())
+                .unwrap_or_else(|e| panic!("{s:?} must decode, got {e}"));
+            assert_eq!(text, s);
+        }
+    }
+
+    #[test]
+    fn mjb_llr_113_empty_input_decodes_to_empty() {
+        let (text, info) = decode(b"").unwrap();
+        assert_eq!(text, "");
+        assert!(!info.has_bom);
+    }
+}