1// Copyright 2013 The Go Authors. All rights reserved.
  2// Use of this source code is governed by a BSD-style
  3// license that can be found in the LICENSE file.
  4
  5//go:generate go run gen.go -output tables.go
  6
  7package language
  8
  9// TODO: Remove above NOTE after:
 10// - verifying that tables are dropped correctly (most notably matcher tables).
 11
 12import (
 13	"strings"
 14
 15	"golang.org/x/text/internal/language"
 16	"golang.org/x/text/internal/language/compact"
 17)
 18
 19// Tag represents a BCP 47 language tag. It is used to specify an instance of a
 20// specific language or locale. All language tag values are guaranteed to be
 21// well-formed.
 22type Tag compact.Tag
 23
 24func makeTag(t language.Tag) (tag Tag) {
 25	return Tag(compact.Make(t))
 26}
 27
 28func (t *Tag) tag() language.Tag {
 29	return (*compact.Tag)(t).Tag()
 30}
 31
 32func (t *Tag) isCompact() bool {
 33	return (*compact.Tag)(t).IsCompact()
 34}
 35
 36// TODO: improve performance.
 37func (t *Tag) lang() language.Language { return t.tag().LangID }
 38func (t *Tag) region() language.Region { return t.tag().RegionID }
 39func (t *Tag) script() language.Script { return t.tag().ScriptID }
 40
 41// Make is a convenience wrapper for Parse that omits the error.
 42// In case of an error, a sensible default is returned.
 43func Make(s string) Tag {
 44	return Default.Make(s)
 45}
 46
 47// Make is a convenience wrapper for c.Parse that omits the error.
 48// In case of an error, a sensible default is returned.
 49func (c CanonType) Make(s string) Tag {
 50	t, _ := c.Parse(s)
 51	return t
 52}
 53
 54// Raw returns the raw base language, script and region, without making an
 55// attempt to infer their values.
 56func (t Tag) Raw() (b Base, s Script, r Region) {
 57	tt := t.tag()
 58	return Base{tt.LangID}, Script{tt.ScriptID}, Region{tt.RegionID}
 59}
 60
 61// IsRoot returns true if t is equal to language "und".
 62func (t Tag) IsRoot() bool {
 63	return compact.Tag(t).IsRoot()
 64}
 65
 66// CanonType can be used to enable or disable various types of canonicalization.
 67type CanonType int
 68
 69const (
 70	// Replace deprecated base languages with their preferred replacements.
 71	DeprecatedBase CanonType = 1 << iota
 72	// Replace deprecated scripts with their preferred replacements.
 73	DeprecatedScript
 74	// Replace deprecated regions with their preferred replacements.
 75	DeprecatedRegion
 76	// Remove redundant scripts.
 77	SuppressScript
 78	// Normalize legacy encodings. This includes legacy languages defined in
 79	// CLDR as well as bibliographic codes defined in ISO-639.
 80	Legacy
 81	// Map the dominant language of a macro language group to the macro language
 82	// subtag. For example cmn -> zh.
 83	Macro
 84	// The CLDR flag should be used if full compatibility with CLDR is required.
 85	// There are a few cases where language.Tag may differ from CLDR. To follow all
 86	// of CLDR's suggestions, use All|CLDR.
 87	CLDR
 88
 89	// Raw can be used to Compose or Parse without Canonicalization.
 90	Raw CanonType = 0
 91
 92	// Replace all deprecated tags with their preferred replacements.
 93	Deprecated = DeprecatedBase | DeprecatedScript | DeprecatedRegion
 94
 95	// All canonicalizations recommended by BCP 47.
 96	BCP47 = Deprecated | SuppressScript
 97
 98	// All canonicalizations.
 99	All = BCP47 | Legacy | Macro
100
101	// Default is the canonicalization used by Parse, Make and Compose. To
102	// preserve as much information as possible, canonicalizations that remove
103	// potentially valuable information are not included. The Matcher is
104	// designed to recognize similar tags that would be the same if
105	// they were canonicalized using All.
106	Default = Deprecated | Legacy
107
108	canonLang = DeprecatedBase | Legacy | Macro
109
110	// TODO: LikelyScript, LikelyRegion: suppress similar to ICU.
111)
112
113// canonicalize returns the canonicalized equivalent of the tag and
114// whether there was any change.
115func canonicalize(c CanonType, t language.Tag) (language.Tag, bool) {
116	if c == Raw {
117		return t, false
118	}
119	changed := false
120	if c&SuppressScript != 0 {
121		if t.LangID.SuppressScript() == t.ScriptID {
122			t.ScriptID = 0
123			changed = true
124		}
125	}
126	if c&canonLang != 0 {
127		for {
128			if l, aliasType := t.LangID.Canonicalize(); l != t.LangID {
129				switch aliasType {
130				case language.Legacy:
131					if c&Legacy != 0 {
132						if t.LangID == _sh && t.ScriptID == 0 {
133							t.ScriptID = _Latn
134						}
135						t.LangID = l
136						changed = true
137					}
138				case language.Macro:
139					if c&Macro != 0 {
140						// We deviate here from CLDR. The mapping "nb" -> "no"
141						// qualifies as a typical Macro language mapping.  However,
142						// for legacy reasons, CLDR maps "no", the macro language
143						// code for Norwegian, to the dominant variant "nb". This
144						// change is currently under consideration for CLDR as well.
145						// See https://unicode.org/cldr/trac/ticket/2698 and also
146						// https://unicode.org/cldr/trac/ticket/1790 for some of the
147						// practical implications. TODO: this check could be removed
148						// if CLDR adopts this change.
149						if c&CLDR == 0 || t.LangID != _nb {
150							changed = true
151							t.LangID = l
152						}
153					}
154				case language.Deprecated:
155					if c&DeprecatedBase != 0 {
156						if t.LangID == _mo && t.RegionID == 0 {
157							t.RegionID = _MD
158						}
159						t.LangID = l
160						changed = true
161						// Other canonicalization types may still apply.
162						continue
163					}
164				}
165			} else if c&Legacy != 0 && t.LangID == _no && c&CLDR != 0 {
166				t.LangID = _nb
167				changed = true
168			}
169			break
170		}
171	}
172	if c&DeprecatedScript != 0 {
173		if t.ScriptID == _Qaai {
174			changed = true
175			t.ScriptID = _Zinh
176		}
177	}
178	if c&DeprecatedRegion != 0 {
179		if r := t.RegionID.Canonicalize(); r != t.RegionID {
180			changed = true
181			t.RegionID = r
182		}
183	}
184	return t, changed
185}
186
187// Canonicalize returns the canonicalized equivalent of the tag.
188func (c CanonType) Canonicalize(t Tag) (Tag, error) {
189	// First try fast path.
190	if t.isCompact() {
191		if _, changed := canonicalize(c, compact.Tag(t).Tag()); !changed {
192			return t, nil
193		}
194	}
195	// It is unlikely that one will canonicalize a tag after matching. So do
196	// a slow but simple approach here.
197	if tag, changed := canonicalize(c, t.tag()); changed {
198		tag.RemakeString()
199		return makeTag(tag), nil
200	}
201	return t, nil
202
203}
204
205// Confidence indicates the level of certainty for a given return value.
206// For example, Serbian may be written in Cyrillic or Latin script.
207// The confidence level indicates whether a value was explicitly specified,
208// whether it is typically the only possible value, or whether there is
209// an ambiguity.
210type Confidence int
211
212const (
213	No    Confidence = iota // full confidence that there was no match
214	Low                     // most likely value picked out of a set of alternatives
215	High                    // value is generally assumed to be the correct match
216	Exact                   // exact match or explicitly specified value
217)
218
219var confName = []string{"No", "Low", "High", "Exact"}
220
221func (c Confidence) String() string {
222	return confName[c]
223}
224
225// String returns the canonical string representation of the language tag.
226func (t Tag) String() string {
227	return t.tag().String()
228}
229
230// MarshalText implements encoding.TextMarshaler.
231func (t Tag) MarshalText() (text []byte, err error) {
232	return t.tag().MarshalText()
233}
234
235// UnmarshalText implements encoding.TextUnmarshaler.
236func (t *Tag) UnmarshalText(text []byte) error {
237	var tag language.Tag
238	err := tag.UnmarshalText(text)
239	*t = makeTag(tag)
240	return err
241}
242
243// Base returns the base language of the language tag. If the base language is
244// unspecified, an attempt will be made to infer it from the context.
245// It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
246func (t Tag) Base() (Base, Confidence) {
247	if b := t.lang(); b != 0 {
248		return Base{b}, Exact
249	}
250	tt := t.tag()
251	c := High
252	if tt.ScriptID == 0 && !tt.RegionID.IsCountry() {
253		c = Low
254	}
255	if tag, err := tt.Maximize(); err == nil && tag.LangID != 0 {
256		return Base{tag.LangID}, c
257	}
258	return Base{0}, No
259}
260
261// Script infers the script for the language tag. If it was not explicitly given, it will infer
262// a most likely candidate.
263// If more than one script is commonly used for a language, the most likely one
264// is returned with a low confidence indication. For example, it returns (Cyrl, Low)
265// for Serbian.
266// If a script cannot be inferred (Zzzz, No) is returned. We do not use Zyyy (undetermined)
267// as one would suspect from the IANA registry for BCP 47. In a Unicode context Zyyy marks
268// common characters (like 1, 2, 3, '.', etc.) and is therefore more like multiple scripts.
269// See https://www.unicode.org/reports/tr24/#Values for more details. Zzzz is also used for
270// unknown value in CLDR.  (Zzzz, Exact) is returned if Zzzz was explicitly specified.
271// Note that an inferred script is never guaranteed to be the correct one. Latin is
272// almost exclusively used for Afrikaans, but Arabic has been used for some texts
273// in the past.  Also, the script that is commonly used may change over time.
274// It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
275func (t Tag) Script() (Script, Confidence) {
276	if scr := t.script(); scr != 0 {
277		return Script{scr}, Exact
278	}
279	tt := t.tag()
280	sc, c := language.Script(_Zzzz), No
281	if scr := tt.LangID.SuppressScript(); scr != 0 {
282		// Note: it is not always the case that a language with a suppress
283		// script value is only written in one script (e.g. kk, ms, pa).
284		if tt.RegionID == 0 {
285			return Script{scr}, High
286		}
287		sc, c = scr, High
288	}
289	if tag, err := tt.Maximize(); err == nil {
290		if tag.ScriptID != sc {
291			sc, c = tag.ScriptID, Low
292		}
293	} else {
294		tt, _ = canonicalize(Deprecated|Macro, tt)
295		if tag, err := tt.Maximize(); err == nil && tag.ScriptID != sc {
296			sc, c = tag.ScriptID, Low
297		}
298	}
299	return Script{sc}, c
300}
301
302// Region returns the region for the language tag. If it was not explicitly given, it will
303// infer a most likely candidate from the context.
304// It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
305func (t Tag) Region() (Region, Confidence) {
306	if r := t.region(); r != 0 {
307		return Region{r}, Exact
308	}
309	tt := t.tag()
310	if tt, err := tt.Maximize(); err == nil {
311		return Region{tt.RegionID}, Low // TODO: differentiate between high and low.
312	}
313	tt, _ = canonicalize(Deprecated|Macro, tt)
314	if tag, err := tt.Maximize(); err == nil {
315		return Region{tag.RegionID}, Low
316	}
317	return Region{_ZZ}, No // TODO: return world instead of undetermined?
318}
319
320// Variants returns the variants specified explicitly for this language tag.
321// or nil if no variant was specified.
322func (t Tag) Variants() []Variant {
323	if !compact.Tag(t).MayHaveVariants() {
324		return nil
325	}
326	v := []Variant{}
327	x, str := "", t.tag().Variants()
328	for str != "" {
329		x, str = nextToken(str)
330		v = append(v, Variant{x})
331	}
332	return v
333}
334
335// Parent returns the CLDR parent of t. In CLDR, missing fields in data for a
336// specific language are substituted with fields from the parent language.
337// The parent for a language may change for newer versions of CLDR.
338//
339// Parent returns a tag for a less specific language that is mutually
340// intelligible or Und if there is no such language. This may not be the same as
341// simply stripping the last BCP 47 subtag. For instance, the parent of "zh-TW"
342// is "zh-Hant", and the parent of "zh-Hant" is "und".
343func (t Tag) Parent() Tag {
344	return Tag(compact.Tag(t).Parent())
345}
346
347// nextToken returns token t and the rest of the string.
348func nextToken(s string) (t, tail string) {
349	p := strings.Index(s[1:], "-")
350	if p == -1 {
351		return s[1:], ""
352	}
353	p++
354	return s[1:p], s[p:]
355}
356
357// Extension is a single BCP 47 extension.
358type Extension struct {
359	s string
360}
361
362// String returns the string representation of the extension, including the
363// type tag.
364func (e Extension) String() string {
365	return e.s
366}
367
368// ParseExtension parses s as an extension and returns it on success.
369func ParseExtension(s string) (e Extension, err error) {
370	ext, err := language.ParseExtension(s)
371	return Extension{ext}, err
372}
373
374// Type returns the one-byte extension type of e. It returns 0 for the zero
375// exception.
376func (e Extension) Type() byte {
377	if e.s == "" {
378		return 0
379	}
380	return e.s[0]
381}
382
383// Tokens returns the list of tokens of e.
384func (e Extension) Tokens() []string {
385	return strings.Split(e.s, "-")
386}
387
388// Extension returns the extension of type x for tag t. It will return
389// false for ok if t does not have the requested extension. The returned
390// extension will be invalid in this case.
391func (t Tag) Extension(x byte) (ext Extension, ok bool) {
392	if !compact.Tag(t).MayHaveExtensions() {
393		return Extension{}, false
394	}
395	e, ok := t.tag().Extension(x)
396	return Extension{e}, ok
397}
398
399// Extensions returns all extensions of t.
400func (t Tag) Extensions() []Extension {
401	if !compact.Tag(t).MayHaveExtensions() {
402		return nil
403	}
404	e := []Extension{}
405	for _, ext := range t.tag().Extensions() {
406		e = append(e, Extension{ext})
407	}
408	return e
409}
410
411// TypeForKey returns the type associated with the given key, where key and type
412// are of the allowed values defined for the Unicode locale extension ('u') in
413// https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
414// TypeForKey will traverse the inheritance chain to get the correct value.
415//
416// If there are multiple types associated with a key, only the first will be
417// returned. If there is no type associated with a key, it returns the empty
418// string.
419func (t Tag) TypeForKey(key string) string {
420	if !compact.Tag(t).MayHaveExtensions() {
421		if key != "rg" && key != "va" {
422			return ""
423		}
424	}
425	return t.tag().TypeForKey(key)
426}
427
428// SetTypeForKey returns a new Tag with the key set to type, where key and type
429// are of the allowed values defined for the Unicode locale extension ('u') in
430// https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
431// An empty value removes an existing pair with the same key.
432func (t Tag) SetTypeForKey(key, value string) (Tag, error) {
433	tt, err := t.tag().SetTypeForKey(key, value)
434	return makeTag(tt), err
435}
436
437// NumCompactTags is the number of compact tags. The maximum tag is
438// NumCompactTags-1.
439const NumCompactTags = compact.NumCompactTags
440
441// CompactIndex returns an index, where 0 <= index < NumCompactTags, for tags
442// for which data exists in the text repository.The index will change over time
443// and should not be stored in persistent storage. If t does not match a compact
444// index, exact will be false and the compact index will be returned for the
445// first match after repeatedly taking the Parent of t.
446func CompactIndex(t Tag) (index int, exact bool) {
447	id, exact := compact.LanguageID(compact.Tag(t))
448	return int(id), exact
449}
450
451var root = language.Tag{}
452
453// Base is an ISO 639 language code, used for encoding the base language
454// of a language tag.
455type Base struct {
456	langID language.Language
457}
458
459// ParseBase parses a 2- or 3-letter ISO 639 code.
460// It returns a ValueError if s is a well-formed but unknown language identifier
461// or another error if another error occurred.
462func ParseBase(s string) (Base, error) {
463	l, err := language.ParseBase(s)
464	return Base{l}, err
465}
466
467// String returns the BCP 47 representation of the base language.
468func (b Base) String() string {
469	return b.langID.String()
470}
471
472// ISO3 returns the ISO 639-3 language code.
473func (b Base) ISO3() string {
474	return b.langID.ISO3()
475}
476
477// IsPrivateUse reports whether this language code is reserved for private use.
478func (b Base) IsPrivateUse() bool {
479	return b.langID.IsPrivateUse()
480}
481
482// Script is a 4-letter ISO 15924 code for representing scripts.
483// It is idiomatically represented in title case.
484type Script struct {
485	scriptID language.Script
486}
487
488// ParseScript parses a 4-letter ISO 15924 code.
489// It returns a ValueError if s is a well-formed but unknown script identifier
490// or another error if another error occurred.
491func ParseScript(s string) (Script, error) {
492	sc, err := language.ParseScript(s)
493	return Script{sc}, err
494}
495
496// String returns the script code in title case.
497// It returns "Zzzz" for an unspecified script.
498func (s Script) String() string {
499	return s.scriptID.String()
500}
501
502// IsPrivateUse reports whether this script code is reserved for private use.
503func (s Script) IsPrivateUse() bool {
504	return s.scriptID.IsPrivateUse()
505}
506
507// Region is an ISO 3166-1 or UN M.49 code for representing countries and regions.
508type Region struct {
509	regionID language.Region
510}
511
512// EncodeM49 returns the Region for the given UN M.49 code.
513// It returns an error if r is not a valid code.
514func EncodeM49(r int) (Region, error) {
515	rid, err := language.EncodeM49(r)
516	return Region{rid}, err
517}
518
519// ParseRegion parses a 2- or 3-letter ISO 3166-1 or a UN M.49 code.
520// It returns a ValueError if s is a well-formed but unknown region identifier
521// or another error if another error occurred.
522func ParseRegion(s string) (Region, error) {
523	r, err := language.ParseRegion(s)
524	return Region{r}, err
525}
526
527// String returns the BCP 47 representation for the region.
528// It returns "ZZ" for an unspecified region.
529func (r Region) String() string {
530	return r.regionID.String()
531}
532
533// ISO3 returns the 3-letter ISO code of r.
534// Note that not all regions have a 3-letter ISO code.
535// In such cases this method returns "ZZZ".
536func (r Region) ISO3() string {
537	return r.regionID.ISO3()
538}
539
540// M49 returns the UN M.49 encoding of r, or 0 if this encoding
541// is not defined for r.
542func (r Region) M49() int {
543	return r.regionID.M49()
544}
545
546// IsPrivateUse reports whether r has the ISO 3166 User-assigned status. This
547// may include private-use tags that are assigned by CLDR and used in this
548// implementation. So IsPrivateUse and IsCountry can be simultaneously true.
549func (r Region) IsPrivateUse() bool {
550	return r.regionID.IsPrivateUse()
551}
552
553// IsCountry returns whether this region is a country or autonomous area. This
554// includes non-standard definitions from CLDR.
555func (r Region) IsCountry() bool {
556	return r.regionID.IsCountry()
557}
558
559// IsGroup returns whether this region defines a collection of regions. This
560// includes non-standard definitions from CLDR.
561func (r Region) IsGroup() bool {
562	return r.regionID.IsGroup()
563}
564
565// Contains returns whether Region c is contained by Region r. It returns true
566// if c == r.
567func (r Region) Contains(c Region) bool {
568	return r.regionID.Contains(c.regionID)
569}
570
571// TLD returns the country code top-level domain (ccTLD). UK is returned for GB.
572// In all other cases it returns either the region itself or an error.
573//
574// This method may return an error for a region for which there exists a
575// canonical form with a ccTLD. To get that ccTLD canonicalize r first. The
576// region will already be canonicalized it was obtained from a Tag that was
577// obtained using any of the default methods.
578func (r Region) TLD() (Region, error) {
579	tld, err := r.regionID.TLD()
580	return Region{tld}, err
581}
582
583// Canonicalize returns the region or a possible replacement if the region is
584// deprecated. It will not return a replacement for deprecated regions that
585// are split into multiple regions.
586func (r Region) Canonicalize() Region {
587	return Region{r.regionID.Canonicalize()}
588}
589
590// Variant represents a registered variant of a language as defined by BCP 47.
591type Variant struct {
592	variant string
593}
594
595// ParseVariant parses and returns a Variant. An error is returned if s is not
596// a valid variant.
597func ParseVariant(s string) (Variant, error) {
598	v, err := language.ParseVariant(s)
599	return Variant{v.String()}, err
600}
601
602// String returns the string representation of the variant.
603func (v Variant) String() string {
604	return v.variant
605}