1//! Encoding and decoding utilities using the `encoding_rs` crate.
2use std::{
3 fmt::Debug,
4 sync::{Arc, Mutex},
5};
6
7use anyhow::Result;
8use encoding_rs::Encoding;
9
10/// A wrapper around `encoding_rs::Encoding` to implement `Send` and `Sync`.
11/// Since the reference is static, it is safe to send it across threads.
12pub struct EncodingWrapper(&'static Encoding);
13
14impl Debug for EncodingWrapper {
15 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
16 f.debug_tuple(&format!("EncodingWrapper{:?}", self.0))
17 .field(&self.0.name())
18 .finish()
19 }
20}
21
22pub struct EncodingWrapperVisitor;
23
24impl PartialEq for EncodingWrapper {
25 fn eq(&self, other: &Self) -> bool {
26 self.0.name() == other.0.name()
27 }
28}
29
30unsafe impl Send for EncodingWrapper {}
31unsafe impl Sync for EncodingWrapper {}
32
33impl Clone for EncodingWrapper {
34 fn clone(&self) -> Self {
35 EncodingWrapper(self.0)
36 }
37}
38
39impl EncodingWrapper {
40 pub fn new(encoding: &'static Encoding) -> EncodingWrapper {
41 EncodingWrapper(encoding)
42 }
43
44 pub fn get_encoding(&self) -> &'static Encoding {
45 self.0
46 }
47
48 pub async fn decode(
49 &mut self,
50 input: Vec<u8>,
51 force: bool,
52 buffer_encoding: Option<Arc<Mutex<&'static Encoding>>>,
53 ) -> Result<String> {
54 // Check if the input starts with a BOM for UTF-16 encodings only if not forced to
55 // use the encoding specified.
56 if !force {
57 if (input[0] == 0xFF) & (input[1] == 0xFE) {
58 self.0 = encoding_rs::UTF_16LE;
59
60 if let Some(v) = buffer_encoding {
61 if let Ok(mut v) = (*v).lock() {
62 *v = encoding_rs::UTF_16LE;
63 }
64 }
65 } else if (input[0] == 0xFE) & (input[1] == 0xFF) {
66 self.0 = encoding_rs::UTF_16BE;
67
68 if let Some(v) = buffer_encoding {
69 if let Ok(mut v) = (*v).lock() {
70 *v = encoding_rs::UTF_16BE;
71 }
72 }
73 }
74 }
75
76 let (cow, _had_errors) = self.0.decode_with_bom_removal(&input);
77
78 // `encoding_rs` handles invalid bytes by replacing them with replacement characters
79 // in the output string, so we return the result even if there were errors.
80 // This preserves the original behaviour where files with invalid bytes could still be opened.
81 Ok(cow.into_owned())
82 }
83
84 pub async fn encode(&self, input: String) -> Result<Vec<u8>> {
85 if self.0 == encoding_rs::UTF_16BE {
86 let mut data = Vec::<u8>::with_capacity(input.len() * 2);
87
88 // Convert the input string to UTF-16BE bytes
89 let utf16be_bytes: Vec<u8> =
90 input.encode_utf16().flat_map(|u| u.to_be_bytes()).collect();
91
92 data.extend(utf16be_bytes);
93 return Ok(data);
94 } else if self.0 == encoding_rs::UTF_16LE {
95 let mut data = Vec::<u8>::with_capacity(input.len() * 2);
96
97 // Convert the input string to UTF-16LE bytes
98 let utf16le_bytes: Vec<u8> =
99 input.encode_utf16().flat_map(|u| u.to_le_bytes()).collect();
100
101 data.extend(utf16le_bytes);
102 return Ok(data);
103 } else {
104 let (cow, _encoding_used, _had_errors) = self.0.encode(&input);
105 // `encoding_rs` handles unencodable characters by replacing them with
106 // appropriate substitutes in the output, so we return the result even if there were errors.
107 // This maintains consistency with the decode behaviour.
108 Ok(cow.into_owned())
109 }
110 }
111}
112
113/// Convert a byte vector from a specified encoding to a UTF-8 string.
114pub async fn to_utf8(
115 input: Vec<u8>,
116 mut encoding: EncodingWrapper,
117 force: bool,
118 buffer_encoding: Option<Arc<Mutex<&'static Encoding>>>,
119) -> Result<String> {
120 encoding.decode(input, force, buffer_encoding).await
121}
122
123/// Convert a UTF-8 string to a byte vector in a specified encoding.
124pub async fn from_utf8(input: String, target: EncodingWrapper) -> Result<Vec<u8>> {
125 target.encode(input).await
126}