path: root/src/buffer/encoding.rs

//! Character encoding detection and transcoding (MJB-HLR-002).
//!
//! Two rules, the second an exception to the first:
//!
//! 1. **A file whose encoding a byte order mark declares is transcoded.** The
//!    encoding is known, so its bytes round-trip through load and save.
//! 2. **UTF-8 is strict.** A file assumed or declared to be UTF-8 that holds an
//!    invalid byte sequence is *rejected*, not repaired.
//!
//! Rule 2 exists because substitution is lossy in a way the user cannot see:
//! replacing a bad byte with U+FFFD and then saving writes the replacement
//! character over their data. For a declared encoding we can at least reproduce
//! what we read; for malformed UTF-8 we cannot, so refusing to open is the only
//! non-destructive answer. See MJB-DR-001.

use encoding_rs::{Encoding, UTF_8, UTF_16BE, UTF_16LE};

/// MJB-LLR-118: why a file could not be decoded.
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
pub enum DecodeError {
    #[error(
        "not valid UTF-8 (invalid byte sequence at offset {valid_up_to}); \
         mojibake edits text, and repairing the bytes would destroy them on save"
    )]
    InvalidUtf8 { valid_up_to: usize },
}

/// The encoding a document was loaded with, plus whether it carried a BOM.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct EncodingInfo {
    pub encoding: &'static Encoding,
    pub has_bom: bool,
}

impl Default for EncodingInfo {
    fn default() -> Self {
        Self {
            encoding: UTF_8,
            has_bom: false,
        }
    }
}

/// MJB-LLR-110: recognise a byte order mark, returning the encoding it implies
/// and the mark's length in bytes.
pub fn detect_bom(bytes: &[u8]) -> Option<(&'static Encoding, usize)> {
    if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) {
        Some((UTF_8, 3))
    } else if bytes.starts_with(&[0xFF, 0xFE]) {
        Some((UTF_16LE, 2))
    } else if bytes.starts_with(&[0xFE, 0xFF]) {
        Some((UTF_16BE, 2))
    } else {
        None
    }
}

/// The byte order mark for `encoding`, if it has one.
pub fn bom_bytes(encoding: &'static Encoding) -> &'static [u8] {
    if encoding == UTF_8 {
        &[0xEF, 0xBB, 0xBF]
    } else if encoding == UTF_16LE {
        &[0xFF, 0xFE]
    } else if encoding == UTF_16BE {
        &[0xFE, 0xFF]
    } else {
        &[]
    }
}

/// MJB-LLR-111, MJB-LLR-113, MJB-LLR-118: decode `bytes` to a `String`.
///
/// UTF-8 — whether declared by a BOM or merely assumed — is validated strictly
/// and rejected when malformed. Other BOM-declared encodings are transcoded.
pub fn decode(bytes: &[u8]) -> Result<(String, EncodingInfo), DecodeError> {
    let (encoding, has_bom, body) = match detect_bom(bytes) {
        Some((encoding, bom_len)) => (encoding, true, &bytes[bom_len..]),
        None => (UTF_8, false, bytes),
    };

    let text = if encoding == UTF_8 {
        // MJB-LLR-118: the exception. `from_utf8` reports exactly how far the
        // input was valid, which makes the diagnostic actionable.
        std::str::from_utf8(body)
            .map_err(|e| DecodeError::InvalidUtf8 {
                valid_up_to: e.valid_up_to(),
            })?
            .to_owned()
    } else {
        // MJB-LLR-113: a declared non-UTF-8 encoding is transcoded. Its bytes
        // round-trip on save, so any substitution here is reproducible.
        encoding.decode_without_bom_handling(body).0.into_owned()
    };

    Ok((text, EncodingInfo { encoding, has_bom }))
}

/// MJB-LLR-115: encode `text` back to bytes, re-emitting the BOM when the
/// document was loaded with one.
///
/// Total by construction: `text` is a `str`, and a document can only hold text
/// that [`decode`] accepted, so there is nothing here that can fail.
///
/// UTF-16 is encoded here by hand rather than through `encoding_rs`.
/// `Encoding::encode` is deliberately asymmetric: it decodes UTF-16 but will
/// not *encode* to it, silently substituting UTF-8 instead. Delegating would
/// therefore write UTF-8 bytes beneath a UTF-16 BOM and corrupt the file.
/// See MJB-DR-007.
pub fn encode(text: &str, info: EncodingInfo) -> Vec<u8> {
    let mut out = Vec::with_capacity(text.len() + 3);
    if info.has_bom {
        out.extend_from_slice(bom_bytes(info.encoding));
    }

    if info.encoding == UTF_16LE {
        for unit in text.encode_utf16() {
            out.extend_from_slice(&unit.to_le_bytes());
        }
    } else if info.encoding == UTF_16BE {
        for unit in text.encode_utf16() {
            out.extend_from_slice(&unit.to_be_bytes());
        }
    } else {
        let (bytes, _, _) = info.encoding.encode(text);
        out.extend_from_slice(&bytes);
    }

    out
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn mjb_llr_110_detects_each_bom() {
        assert_eq!(detect_bom(&[0xEF, 0xBB, 0xBF, b'a']), Some((UTF_8, 3)));
        assert_eq!(detect_bom(&[0xFF, 0xFE, b'a', 0]), Some((UTF_16LE, 2)));
        assert_eq!(detect_bom(&[0xFE, 0xFF, 0, b'a']), Some((UTF_16BE, 2)));
        assert_eq!(detect_bom(b"plain"), None);
        assert_eq!(detect_bom(b""), None, "empty input must not index past end");
    }

    #[test]
    fn mjb_llr_111_plain_utf8_round_trips() {
        let (text, info) = decode("hello 文字化け".as_bytes()).unwrap();
        assert_eq!(text, "hello 文字化け");
        assert!(!info.has_bom);
        assert_eq!(encode(&text, info), "hello 文字化け".as_bytes());
    }

    /// MJB-LLR-115, MJB-DR-001: a BOM present on load reappears on save.
    #[test]
    fn mjb_llr_115_bom_round_trips() {
        let mut input = vec![0xEF, 0xBB, 0xBF];
        input.extend_from_slice(b"hi");

        let (text, info) = decode(&input).unwrap();
        assert_eq!(text, "hi", "BOM is stripped from the buffer contents");
        assert!(info.has_bom);
        assert_eq!(encode(&text, info), input, "and re-emitted on write");
    }

    /// MJB-LLR-115, MJB-DR-007: UTF-16 must survive a load/save cycle.
    ///
    /// Regression guard: `encoding_rs::Encoding::encode` substitutes UTF-8 for
    /// UTF-16 rather than failing, so delegating to it here would write UTF-8
    /// bytes under a UTF-16 BOM and corrupt the file.
    #[test]
    fn mjb_llr_115_utf16le_round_trips() {
        // UTF-16LE BOM followed by "hi".
        let input = vec![0xFF, 0xFE, b'h', 0x00, b'i', 0x00];
        let (text, info) = decode(&input).unwrap();
        assert_eq!(text, "hi");
        assert_eq!(info.encoding, UTF_16LE);
        assert_eq!(encode(&text, info), input, "must not degrade to UTF-8");
    }

    #[test]
    fn mjb_llr_115_utf16be_round_trips() {
        let input = vec![0xFE, 0xFF, 0x00, b'h', 0x00, b'i'];
        let (text, info) = decode(&input).unwrap();
        assert_eq!(text, "hi");
        assert_eq!(info.encoding, UTF_16BE);
        assert_eq!(encode(&text, info), input);
    }

    #[test]
    fn mjb_llr_115_utf16_handles_non_ascii_and_surrogates() {
        // 文 is BMP; 𝄞 (U+1D11E) needs a surrogate pair in UTF-16.
        let original = "文𝄞";
        let info = EncodingInfo {
            encoding: UTF_16LE,
            has_bom: true,
        };
        let bytes = encode(original, info);
        let (back, _) = decode(&bytes).unwrap();
        assert_eq!(back, original);
    }

    /// MJB-LLR-113: a *declared* non-UTF-8 encoding is transcoded, not
    /// rejected. An unpaired surrogate is repaired, and that repair is
    /// reproducible because the encoding is known.
    #[test]
    fn mjb_llr_113_declared_utf16_is_transcoded_not_rejected() {
        // UTF-16LE BOM, then a lone high surrogate (0xD800) — not valid UTF-16.
        let input = vec![0xFF, 0xFE, 0x00, 0xD8, b'a', 0x00];
        let (text, info) = decode(&input).expect("a declared encoding must not be rejected");
        assert_eq!(info.encoding, UTF_16LE);
        assert!(
            text.contains('\u{FFFD}'),
            "the unpaired surrogate becomes U+FFFD, got {text:?}"
        );
    }

    /// MJB-LLR-118: the UTF-8 exception. Invalid UTF-8 is rejected outright
    /// rather than repaired, because a repair would be written back over the
    /// user's data on save.
    #[test]
    fn mjb_llr_118_invalid_utf8_is_rejected() {
        // 0xFF is never valid in UTF-8.
        let err = decode(&[b'a', 0xFF, b'b']).unwrap_err();
        assert_eq!(err, DecodeError::InvalidUtf8 { valid_up_to: 1 });
    }

    #[test]
    fn mjb_llr_118_rejection_names_the_offset() {
        let err = decode(&[b'h', b'i', 0x80]).unwrap_err();
        assert_eq!(err, DecodeError::InvalidUtf8 { valid_up_to: 2 });
        assert!(
            err.to_string().contains('2'),
            "the message must locate the bad byte, got: {err}"
        );
    }

    /// A truncated multi-byte character is invalid UTF-8 too.
    #[test]
    fn mjb_llr_118_truncated_multibyte_char_is_rejected() {
        // 文 is E6 96 87; drop the last byte.
        let err = decode(&[0xE6, 0x96]).unwrap_err();
        assert_eq!(err, DecodeError::InvalidUtf8 { valid_up_to: 0 });
    }

    /// MJB-LLR-118: a UTF-8 *BOM* does not license invalid bytes after it.
    #[test]
    fn mjb_llr_118_declared_utf8_is_strict_too() {
        let input = vec![0xEF, 0xBB, 0xBF, b'a', 0xFF];
        let err = decode(&input).unwrap_err();
        assert_eq!(
            err,
            DecodeError::InvalidUtf8 { valid_up_to: 1 },
            "offset is measured past the BOM"
        );
    }

    /// Valid multi-byte UTF-8 must not be mistaken for invalid.
    #[test]
    fn mjb_llr_118_valid_multibyte_utf8_is_accepted() {
        for s in ["文字化け", "e\u{0301}", "𝄞", "café", "", "\u{FFFD}"] {
            let (text, _) = decode(s.as_bytes())
                .unwrap_or_else(|e| panic!("{s:?} must decode, got {e}"));
            assert_eq!(text, s);
        }
    }

    #[test]
    fn mjb_llr_113_empty_input_decodes_to_empty() {
        let (text, info) = decode(b"").unwrap();
        assert_eq!(text, "");
        assert!(!info.has_bom);
    }
}