1// Copyright 2013 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5// Package charset provides common text encodings for HTML documents.
6//
7// The mapping from encoding labels to encodings is defined at
8// https://encoding.spec.whatwg.org/.
9package charset // import "golang.org/x/net/html/charset"
10
11import (
12 "bytes"
13 "fmt"
14 "io"
15 "mime"
16 "strings"
17 "unicode/utf8"
18
19 "golang.org/x/net/html"
20 "golang.org/x/text/encoding"
21 "golang.org/x/text/encoding/charmap"
22 "golang.org/x/text/encoding/htmlindex"
23 "golang.org/x/text/transform"
24)
25
26// Lookup returns the encoding with the specified label, and its canonical
27// name. It returns nil and the empty string if label is not one of the
28// standard encodings for HTML. Matching is case-insensitive and ignores
29// leading and trailing whitespace. Encoders will use HTML escape sequences for
30// runes that are not supported by the character set.
31func Lookup(label string) (e encoding.Encoding, name string) {
32 e, err := htmlindex.Get(label)
33 if err != nil {
34 return nil, ""
35 }
36 name, _ = htmlindex.Name(e)
37 return &htmlEncoding{e}, name
38}
39
40type htmlEncoding struct{ encoding.Encoding }
41
42func (h *htmlEncoding) NewEncoder() *encoding.Encoder {
43 // HTML requires a non-terminating legacy encoder. We use HTML escapes to
44 // substitute unsupported code points.
45 return encoding.HTMLEscapeUnsupported(h.Encoding.NewEncoder())
46}
47
48// DetermineEncoding determines the encoding of an HTML document by examining
49// up to the first 1024 bytes of content and the declared Content-Type.
50//
51// See http://www.whatwg.org/specs/web-apps/current-work/multipage/parsing.html#determining-the-character-encoding
52func DetermineEncoding(content []byte, contentType string) (e encoding.Encoding, name string, certain bool) {
53 if len(content) > 1024 {
54 content = content[:1024]
55 }
56
57 for _, b := range boms {
58 if bytes.HasPrefix(content, b.bom) {
59 e, name = Lookup(b.enc)
60 return e, name, true
61 }
62 }
63
64 if _, params, err := mime.ParseMediaType(contentType); err == nil {
65 if cs, ok := params["charset"]; ok {
66 if e, name = Lookup(cs); e != nil {
67 return e, name, true
68 }
69 }
70 }
71
72 if len(content) > 0 {
73 e, name = prescan(content)
74 if e != nil {
75 return e, name, false
76 }
77 }
78
79 // Try to detect UTF-8.
80 // First eliminate any partial rune at the end.
81 for i := len(content) - 1; i >= 0 && i > len(content)-4; i-- {
82 b := content[i]
83 if b < 0x80 {
84 break
85 }
86 if utf8.RuneStart(b) {
87 content = content[:i]
88 break
89 }
90 }
91 hasHighBit := false
92 for _, c := range content {
93 if c >= 0x80 {
94 hasHighBit = true
95 break
96 }
97 }
98 if hasHighBit && utf8.Valid(content) {
99 return encoding.Nop, "utf-8", false
100 }
101
102 // TODO: change default depending on user's locale?
103 return charmap.Windows1252, "windows-1252", false
104}
105
106// NewReader returns an io.Reader that converts the content of r to UTF-8.
107// It calls DetermineEncoding to find out what r's encoding is.
108func NewReader(r io.Reader, contentType string) (io.Reader, error) {
109 preview := make([]byte, 1024)
110 n, err := io.ReadFull(r, preview)
111 switch {
112 case err == io.ErrUnexpectedEOF:
113 preview = preview[:n]
114 r = bytes.NewReader(preview)
115 case err != nil:
116 return nil, err
117 default:
118 r = io.MultiReader(bytes.NewReader(preview), r)
119 }
120
121 if e, _, _ := DetermineEncoding(preview, contentType); e != encoding.Nop {
122 r = transform.NewReader(r, e.NewDecoder())
123 }
124 return r, nil
125}
126
127// NewReaderLabel returns a reader that converts from the specified charset to
128// UTF-8. It uses Lookup to find the encoding that corresponds to label, and
129// returns an error if Lookup returns nil. It is suitable for use as
130// encoding/xml.Decoder's CharsetReader function.
131func NewReaderLabel(label string, input io.Reader) (io.Reader, error) {
132 e, _ := Lookup(label)
133 if e == nil {
134 return nil, fmt.Errorf("unsupported charset: %q", label)
135 }
136 return transform.NewReader(input, e.NewDecoder()), nil
137}
138
139func prescan(content []byte) (e encoding.Encoding, name string) {
140 z := html.NewTokenizer(bytes.NewReader(content))
141 for {
142 switch z.Next() {
143 case html.ErrorToken:
144 return nil, ""
145
146 case html.StartTagToken, html.SelfClosingTagToken:
147 tagName, hasAttr := z.TagName()
148 if !bytes.Equal(tagName, []byte("meta")) {
149 continue
150 }
151 attrList := make(map[string]bool)
152 gotPragma := false
153
154 const (
155 dontKnow = iota
156 doNeedPragma
157 doNotNeedPragma
158 )
159 needPragma := dontKnow
160
161 name = ""
162 e = nil
163 for hasAttr {
164 var key, val []byte
165 key, val, hasAttr = z.TagAttr()
166 ks := string(key)
167 if attrList[ks] {
168 continue
169 }
170 attrList[ks] = true
171 for i, c := range val {
172 if 'A' <= c && c <= 'Z' {
173 val[i] = c + 0x20
174 }
175 }
176
177 switch ks {
178 case "http-equiv":
179 if bytes.Equal(val, []byte("content-type")) {
180 gotPragma = true
181 }
182
183 case "content":
184 if e == nil {
185 name = fromMetaElement(string(val))
186 if name != "" {
187 e, name = Lookup(name)
188 if e != nil {
189 needPragma = doNeedPragma
190 }
191 }
192 }
193
194 case "charset":
195 e, name = Lookup(string(val))
196 needPragma = doNotNeedPragma
197 }
198 }
199
200 if needPragma == dontKnow || needPragma == doNeedPragma && !gotPragma {
201 continue
202 }
203
204 if strings.HasPrefix(name, "utf-16") {
205 name = "utf-8"
206 e = encoding.Nop
207 }
208
209 if e != nil {
210 return e, name
211 }
212 }
213 }
214}
215
216func fromMetaElement(s string) string {
217 for s != "" {
218 csLoc := strings.Index(s, "charset")
219 if csLoc == -1 {
220 return ""
221 }
222 s = s[csLoc+len("charset"):]
223 s = strings.TrimLeft(s, " \t\n\f\r")
224 if !strings.HasPrefix(s, "=") {
225 continue
226 }
227 s = s[1:]
228 s = strings.TrimLeft(s, " \t\n\f\r")
229 if s == "" {
230 return ""
231 }
232 if q := s[0]; q == '"' || q == '\'' {
233 s = s[1:]
234 closeQuote := strings.IndexRune(s, rune(q))
235 if closeQuote == -1 {
236 return ""
237 }
238 return s[:closeQuote]
239 }
240
241 end := strings.IndexAny(s, "; \t\n\f\r")
242 if end == -1 {
243 end = len(s)
244 }
245 return s[:end]
246 }
247 return ""
248}
249
250var boms = []struct {
251 bom []byte
252 enc string
253}{
254 {[]byte{0xfe, 0xff}, "utf-16be"},
255 {[]byte{0xff, 0xfe}, "utf-16le"},
256 {[]byte{0xef, 0xbb, 0xbf}, "utf-8"},
257}