1//! Encoding and decoding utilities using the `encoding_rs` crate.
2use std::{
3 fmt::Debug,
4 sync::{Arc, Mutex},
5};
6
7use anyhow::Result;
8use encoding_rs::Encoding;
9
10/// A wrapper around `encoding_rs::Encoding` to implement `Send` and `Sync`.
11/// Since the reference is static, it is safe to send it across threads.
12pub struct EncodingWrapper(&'static Encoding);
13
14impl Debug for EncodingWrapper {
15 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
16 f.debug_tuple(&format!("EncodingWrapper{:?}", self.0))
17 .field(&self.0.name())
18 .finish()
19 }
20}
21
22pub struct EncodingWrapperVisitor;
23
24impl PartialEq for EncodingWrapper {
25 fn eq(&self, other: &Self) -> bool {
26 self.0.name() == other.0.name()
27 }
28}
29
30unsafe impl Send for EncodingWrapper {}
31unsafe impl Sync for EncodingWrapper {}
32
33impl Clone for EncodingWrapper {
34 fn clone(&self) -> Self {
35 EncodingWrapper(self.0)
36 }
37}
38
39impl EncodingWrapper {
40 pub fn new(encoding: &'static Encoding) -> EncodingWrapper {
41 EncodingWrapper(encoding)
42 }
43
44 pub fn get_encoding(&self) -> &'static Encoding {
45 self.0
46 }
47
48 pub async fn decode(
49 &mut self,
50 input: Vec<u8>,
51 force: bool,
52 buffer_encoding: Option<Arc<Mutex<&'static Encoding>>>,
53 ) -> Result<String> {
54 // Check if the input starts with a BOM for UTF-16 encodings only if not forced to
55 // use the encoding specified.
56 if !force {
57 if let Some(encoding) = match input.get(..2) {
58 Some([0xFF, 0xFE]) => Some(encoding_rs::UTF_16LE),
59 Some([0xFE, 0xFF]) => Some(encoding_rs::UTF_16BE),
60 _ => None,
61 } {
62 self.0 = encoding;
63
64 if let Some(v) = buffer_encoding {
65 if let Ok(mut v) = (*v).lock() {
66 *v = encoding;
67 }
68 }
69 }
70 }
71
72 let (cow, _had_errors) = self.0.decode_with_bom_removal(&input);
73
74 // `encoding_rs` handles invalid bytes by replacing them with replacement characters
75 // in the output string, so we return the result even if there were errors.
76 // This preserves the original behaviour where files with invalid bytes could still be opened.
77 Ok(cow.into_owned())
78 }
79
80 pub async fn encode(&self, input: String) -> Result<Vec<u8>> {
81 if self.0 == encoding_rs::UTF_16BE {
82 let mut data = Vec::<u8>::with_capacity(input.len() * 2);
83
84 // Convert the input string to UTF-16BE bytes
85 let utf16be_bytes = input.encode_utf16().flat_map(|u| u.to_be_bytes());
86
87 data.extend(utf16be_bytes);
88 return Ok(data);
89 } else if self.0 == encoding_rs::UTF_16LE {
90 let mut data = Vec::<u8>::with_capacity(input.len() * 2);
91
92 // Convert the input string to UTF-16LE bytes
93 let utf16le_bytes = input.encode_utf16().flat_map(|u| u.to_le_bytes());
94
95 data.extend(utf16le_bytes);
96 return Ok(data);
97 } else {
98 let (cow, _encoding_used, _had_errors) = self.0.encode(&input);
99 // `encoding_rs` handles unencodable characters by replacing them with
100 // appropriate substitutes in the output, so we return the result even if there were errors.
101 // This maintains consistency with the decode behaviour.
102 Ok(cow.into_owned())
103 }
104 }
105}
106
107/// Convert a byte vector from a specified encoding to a UTF-8 string.
108pub async fn to_utf8(
109 input: Vec<u8>,
110 mut encoding: EncodingWrapper,
111 force: bool,
112 buffer_encoding: Option<Arc<Mutex<&'static Encoding>>>,
113) -> Result<String> {
114 encoding.decode(input, force, buffer_encoding).await
115}
116
117/// Convert a UTF-8 string to a byte vector in a specified encoding.
118pub async fn from_utf8(input: String, target: EncodingWrapper) -> Result<Vec<u8>> {
119 target.encode(input).await
120}