From edc78b963361eb27bb013c9d74e095c3954dc8b0 Mon Sep 17 00:00:00 2001 From: Antoine GIRARD Date: Sun, 20 May 2018 12:09:00 +0200 Subject: [PATCH] Update to last common x/text (#3994) --- .../x/text/encoding/charmap/charmap.go | 84 +- .../x/text/encoding/charmap/maketables.go | 524 -- .../x/text/encoding/charmap/tables.go | 886 +++- .../x/text/encoding/htmlindex/gen.go | 167 - .../x/text/encoding/htmlindex/tables.go | 2 +- .../text/encoding/internal/identifier/gen.go | 137 - .../internal/identifier/identifier.go | 2 +- .../text/encoding/internal/identifier/mib.go | 2 +- .../x/text/encoding/japanese/eucjp.go | 72 +- .../x/text/encoding/japanese/iso2022jp.go | 59 +- .../x/text/encoding/japanese/maketables.go | 161 - .../x/text/encoding/japanese/shiftjis.go | 36 +- .../x/text/encoding/korean/euckr.go | 39 +- .../x/text/encoding/korean/maketables.go | 143 - .../x/text/encoding/simplifiedchinese/gbk.go | 60 +- .../encoding/simplifiedchinese/hzgb2312.go | 49 +- .../encoding/simplifiedchinese/maketables.go | 161 - .../text/encoding/traditionalchinese/big5.go | 25 +- .../encoding/traditionalchinese/maketables.go | 140 - vendor/golang.org/x/text/language/common.go | 2 +- .../golang.org/x/text/language/gen_common.go | 20 - .../golang.org/x/text/language/gen_index.go | 162 - vendor/golang.org/x/text/language/index.go | 1510 +++--- .../golang.org/x/text/language/maketables.go | 1648 ------ vendor/golang.org/x/text/language/match.go | 20 +- vendor/golang.org/x/text/language/tables.go | 4461 +++++++++-------- .../x/text/unicode/norm/maketables.go | 976 ---- .../golang.org/x/text/unicode/norm/triegen.go | 117 - vendor/vendor.json | 78 +- 29 files changed, 4131 insertions(+), 7612 deletions(-) delete mode 100644 vendor/golang.org/x/text/encoding/charmap/maketables.go delete mode 100644 vendor/golang.org/x/text/encoding/htmlindex/gen.go delete mode 100644 vendor/golang.org/x/text/encoding/internal/identifier/gen.go delete mode 100644 vendor/golang.org/x/text/encoding/japanese/maketables.go delete mode 100644 vendor/golang.org/x/text/encoding/korean/maketables.go delete mode 100644 vendor/golang.org/x/text/encoding/simplifiedchinese/maketables.go delete mode 100644 vendor/golang.org/x/text/encoding/traditionalchinese/maketables.go delete mode 100644 vendor/golang.org/x/text/language/gen_common.go delete mode 100644 vendor/golang.org/x/text/language/gen_index.go delete mode 100644 vendor/golang.org/x/text/language/maketables.go delete mode 100644 vendor/golang.org/x/text/unicode/norm/maketables.go delete mode 100644 vendor/golang.org/x/text/unicode/norm/triegen.go diff --git a/vendor/golang.org/x/text/encoding/charmap/charmap.go b/vendor/golang.org/x/text/encoding/charmap/charmap.go index 6e62a8374..e89ff0734 100644 --- a/vendor/golang.org/x/text/encoding/charmap/charmap.go +++ b/vendor/golang.org/x/text/encoding/charmap/charmap.go @@ -33,32 +33,32 @@ var ( ISO8859_8I encoding.Encoding = &iso8859_8I iso8859_6E = internal.Encoding{ - ISO8859_6, - "ISO-8859-6E", - identifier.ISO88596E, + Encoding: ISO8859_6, + Name: "ISO-8859-6E", + MIB: identifier.ISO88596E, } iso8859_6I = internal.Encoding{ - ISO8859_6, - "ISO-8859-6I", - identifier.ISO88596I, + Encoding: ISO8859_6, + Name: "ISO-8859-6I", + MIB: identifier.ISO88596I, } iso8859_8E = internal.Encoding{ - ISO8859_8, - "ISO-8859-8E", - identifier.ISO88598E, + Encoding: ISO8859_8, + Name: "ISO-8859-8E", + MIB: identifier.ISO88598E, } iso8859_8I = internal.Encoding{ - ISO8859_8, - "ISO-8859-8I", - identifier.ISO88598I, + Encoding: ISO8859_8, + Name: "ISO-8859-8I", + MIB: identifier.ISO88598I, } ) // All is a list of all defined encodings in this package. -var All = listAll +var All []encoding.Encoding = listAll // TODO: implement these encodings, in order of importance. // ASCII, ISO8859_1: Rather common. Close to Windows 1252. @@ -70,8 +70,8 @@ type utf8Enc struct { data [3]byte } -// charmap describes an 8-bit character set encoding. -type charmap struct { +// Charmap is an 8-bit character set encoding. +type Charmap struct { // name is the encoding's name. name string // mib is the encoding type of this encoder. @@ -79,7 +79,7 @@ type charmap struct { // asciiSuperset states whether the encoding is a superset of ASCII. asciiSuperset bool // low is the lower bound of the encoded byte for a non-ASCII rune. If - // charmap.asciiSuperset is true then this will be 0x80, otherwise 0x00. + // Charmap.asciiSuperset is true then this will be 0x80, otherwise 0x00. low uint8 // replacement is the encoded replacement character. replacement byte @@ -91,26 +91,30 @@ type charmap struct { encode [256]uint32 } -func (m *charmap) NewDecoder() *encoding.Decoder { +// NewDecoder implements the encoding.Encoding interface. +func (m *Charmap) NewDecoder() *encoding.Decoder { return &encoding.Decoder{Transformer: charmapDecoder{charmap: m}} } -func (m *charmap) NewEncoder() *encoding.Encoder { +// NewEncoder implements the encoding.Encoding interface. +func (m *Charmap) NewEncoder() *encoding.Encoder { return &encoding.Encoder{Transformer: charmapEncoder{charmap: m}} } -func (m *charmap) String() string { +// String returns the Charmap's name. +func (m *Charmap) String() string { return m.name } -func (m *charmap) ID() (mib identifier.MIB, other string) { +// ID implements an internal interface. +func (m *Charmap) ID() (mib identifier.MIB, other string) { return m.mib, "" } // charmapDecoder implements transform.Transformer by decoding to UTF-8. type charmapDecoder struct { transform.NopResetter - charmap *charmap + charmap *Charmap } func (m charmapDecoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { @@ -142,10 +146,22 @@ func (m charmapDecoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, return nDst, nSrc, err } +// DecodeByte returns the Charmap's rune decoding of the byte b. +func (m *Charmap) DecodeByte(b byte) rune { + switch x := &m.decode[b]; x.len { + case 1: + return rune(x.data[0]) + case 2: + return rune(x.data[0]&0x1f)<<6 | rune(x.data[1]&0x3f) + default: + return rune(x.data[0]&0x0f)<<12 | rune(x.data[1]&0x3f)<<6 | rune(x.data[2]&0x3f) + } +} + // charmapEncoder implements transform.Transformer by encoding from UTF-8. type charmapEncoder struct { transform.NopResetter - charmap *charmap + charmap *Charmap } func (m charmapEncoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { @@ -207,3 +223,27 @@ loop: } return nDst, nSrc, err } + +// EncodeRune returns the Charmap's byte encoding of the rune r. ok is whether +// r is in the Charmap's repertoire. If not, b is set to the Charmap's +// replacement byte. This is often the ASCII substitute character '\x1a'. +func (m *Charmap) EncodeRune(r rune) (b byte, ok bool) { + if r < utf8.RuneSelf && m.asciiSuperset { + return byte(r), true + } + for low, high := int(m.low), 0x100; ; { + if low >= high { + return m.replacement, false + } + mid := (low + high) / 2 + got := m.encode[mid] + gotRune := rune(got & (1<<24 - 1)) + if gotRune < r { + low = mid + 1 + } else if gotRune > r { + high = mid + } else { + return byte(got >> 24), true + } + } +} diff --git a/vendor/golang.org/x/text/encoding/charmap/maketables.go b/vendor/golang.org/x/text/encoding/charmap/maketables.go deleted file mode 100644 index 9672c552d..000000000 --- a/vendor/golang.org/x/text/encoding/charmap/maketables.go +++ /dev/null @@ -1,524 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -package main - -import ( - "bufio" - "fmt" - "log" - "net/http" - "sort" - "strings" - "unicode/utf8" - - "golang.org/x/text/encoding" - "golang.org/x/text/internal/gen" -) - -const ascii = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + - "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + - ` !"#$%&'()*+,-./0123456789:;<=>?` + - `@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_` + - "`abcdefghijklmnopqrstuvwxyz{|}~\u007f" - -var encodings = []struct { - name string - mib string - comment string - varName string - replacement byte - mapping string -}{ - { - "IBM Code Page 437", - "PC8CodePage437", - "", - "CodePage437", - encoding.ASCIISub, - "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM437-2.1.2.ucm", - }, - { - "IBM Code Page 850", - "PC850Multilingual", - "", - "CodePage850", - encoding.ASCIISub, - "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM850-2.1.2.ucm", - }, - { - "IBM Code Page 852", - "PCp852", - "", - "CodePage852", - encoding.ASCIISub, - "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM852-2.1.2.ucm", - }, - { - "IBM Code Page 855", - "IBM855", - "", - "CodePage855", - encoding.ASCIISub, - "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM855-2.1.2.ucm", - }, - { - "Windows Code Page 858", // PC latin1 with Euro - "IBM00858", - "", - "CodePage858", - encoding.ASCIISub, - "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/windows-858-2000.ucm", - }, - { - "IBM Code Page 860", - "IBM860", - "", - "CodePage860", - encoding.ASCIISub, - "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM860-2.1.2.ucm", - }, - { - "IBM Code Page 862", - "PC862LatinHebrew", - "", - "CodePage862", - encoding.ASCIISub, - "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM862-2.1.2.ucm", - }, - { - "IBM Code Page 863", - "IBM863", - "", - "CodePage863", - encoding.ASCIISub, - "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM863-2.1.2.ucm", - }, - { - "IBM Code Page 865", - "IBM865", - "", - "CodePage865", - encoding.ASCIISub, - "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM865-2.1.2.ucm", - }, - { - "IBM Code Page 866", - "IBM866", - "", - "CodePage866", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-ibm866.txt", - }, - { - "ISO 8859-1", - "ISOLatin1", - "", - "ISO8859_1", - encoding.ASCIISub, - "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/iso-8859_1-1998.ucm", - }, - { - "ISO 8859-2", - "ISOLatin2", - "", - "ISO8859_2", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-iso-8859-2.txt", - }, - { - "ISO 8859-3", - "ISOLatin3", - "", - "ISO8859_3", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-iso-8859-3.txt", - }, - { - "ISO 8859-4", - "ISOLatin4", - "", - "ISO8859_4", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-iso-8859-4.txt", - }, - { - "ISO 8859-5", - "ISOLatinCyrillic", - "", - "ISO8859_5", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-iso-8859-5.txt", - }, - { - "ISO 8859-6", - "ISOLatinArabic", - "", - "ISO8859_6,ISO8859_6E,ISO8859_6I", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-iso-8859-6.txt", - }, - { - "ISO 8859-7", - "ISOLatinGreek", - "", - "ISO8859_7", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-iso-8859-7.txt", - }, - { - "ISO 8859-8", - "ISOLatinHebrew", - "", - "ISO8859_8,ISO8859_8E,ISO8859_8I", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-iso-8859-8.txt", - }, - { - "ISO 8859-10", - "ISOLatin6", - "", - "ISO8859_10", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-iso-8859-10.txt", - }, - { - "ISO 8859-13", - "ISO885913", - "", - "ISO8859_13", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-iso-8859-13.txt", - }, - { - "ISO 8859-14", - "ISO885914", - "", - "ISO8859_14", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-iso-8859-14.txt", - }, - { - "ISO 8859-15", - "ISO885915", - "", - "ISO8859_15", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-iso-8859-15.txt", - }, - { - "ISO 8859-16", - "ISO885916", - "", - "ISO8859_16", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-iso-8859-16.txt", - }, - { - "KOI8-R", - "KOI8R", - "", - "KOI8R", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-koi8-r.txt", - }, - { - "KOI8-U", - "KOI8U", - "", - "KOI8U", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-koi8-u.txt", - }, - { - "Macintosh", - "Macintosh", - "", - "Macintosh", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-macintosh.txt", - }, - { - "Macintosh Cyrillic", - "MacintoshCyrillic", - "", - "MacintoshCyrillic", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-x-mac-cyrillic.txt", - }, - { - "Windows 874", - "Windows874", - "", - "Windows874", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-windows-874.txt", - }, - { - "Windows 1250", - "Windows1250", - "", - "Windows1250", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-windows-1250.txt", - }, - { - "Windows 1251", - "Windows1251", - "", - "Windows1251", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-windows-1251.txt", - }, - { - "Windows 1252", - "Windows1252", - "", - "Windows1252", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-windows-1252.txt", - }, - { - "Windows 1253", - "Windows1253", - "", - "Windows1253", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-windows-1253.txt", - }, - { - "Windows 1254", - "Windows1254", - "", - "Windows1254", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-windows-1254.txt", - }, - { - "Windows 1255", - "Windows1255", - "", - "Windows1255", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-windows-1255.txt", - }, - { - "Windows 1256", - "Windows1256", - "", - "Windows1256", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-windows-1256.txt", - }, - { - "Windows 1257", - "Windows1257", - "", - "Windows1257", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-windows-1257.txt", - }, - { - "Windows 1258", - "Windows1258", - "", - "Windows1258", - encoding.ASCIISub, - "http://encoding.spec.whatwg.org/index-windows-1258.txt", - }, - { - "X-User-Defined", - "XUserDefined", - "It is defined at http://encoding.spec.whatwg.org/#x-user-defined", - "XUserDefined", - encoding.ASCIISub, - ascii + - "\uf780\uf781\uf782\uf783\uf784\uf785\uf786\uf787" + - "\uf788\uf789\uf78a\uf78b\uf78c\uf78d\uf78e\uf78f" + - "\uf790\uf791\uf792\uf793\uf794\uf795\uf796\uf797" + - "\uf798\uf799\uf79a\uf79b\uf79c\uf79d\uf79e\uf79f" + - "\uf7a0\uf7a1\uf7a2\uf7a3\uf7a4\uf7a5\uf7a6\uf7a7" + - "\uf7a8\uf7a9\uf7aa\uf7ab\uf7ac\uf7ad\uf7ae\uf7af" + - "\uf7b0\uf7b1\uf7b2\uf7b3\uf7b4\uf7b5\uf7b6\uf7b7" + - "\uf7b8\uf7b9\uf7ba\uf7bb\uf7bc\uf7bd\uf7be\uf7bf" + - "\uf7c0\uf7c1\uf7c2\uf7c3\uf7c4\uf7c5\uf7c6\uf7c7" + - "\uf7c8\uf7c9\uf7ca\uf7cb\uf7cc\uf7cd\uf7ce\uf7cf" + - "\uf7d0\uf7d1\uf7d2\uf7d3\uf7d4\uf7d5\uf7d6\uf7d7" + - "\uf7d8\uf7d9\uf7da\uf7db\uf7dc\uf7dd\uf7de\uf7df" + - "\uf7e0\uf7e1\uf7e2\uf7e3\uf7e4\uf7e5\uf7e6\uf7e7" + - "\uf7e8\uf7e9\uf7ea\uf7eb\uf7ec\uf7ed\uf7ee\uf7ef" + - "\uf7f0\uf7f1\uf7f2\uf7f3\uf7f4\uf7f5\uf7f6\uf7f7" + - "\uf7f8\uf7f9\uf7fa\uf7fb\uf7fc\uf7fd\uf7fe\uf7ff", - }, -} - -func getWHATWG(url string) string { - res, err := http.Get(url) - if err != nil { - log.Fatalf("%q: Get: %v", url, err) - } - defer res.Body.Close() - - mapping := make([]rune, 128) - for i := range mapping { - mapping[i] = '\ufffd' - } - - scanner := bufio.NewScanner(res.Body) - for scanner.Scan() { - s := strings.TrimSpace(scanner.Text()) - if s == "" || s[0] == '#' { - continue - } - x, y := 0, 0 - if _, err := fmt.Sscanf(s, "%d\t0x%x", &x, &y); err != nil { - log.Fatalf("could not parse %q", s) - } - if x < 0 || 128 <= x { - log.Fatalf("code %d is out of range", x) - } - if 0x80 <= y && y < 0xa0 { - // We diverge from the WHATWG spec by mapping control characters - // in the range [0x80, 0xa0) to U+FFFD. - continue - } - mapping[x] = rune(y) - } - return ascii + string(mapping) -} - -func getUCM(url string) string { - res, err := http.Get(url) - if err != nil { - log.Fatalf("%q: Get: %v", url, err) - } - defer res.Body.Close() - - mapping := make([]rune, 256) - for i := range mapping { - mapping[i] = '\ufffd' - } - - charsFound := 0 - scanner := bufio.NewScanner(res.Body) - for scanner.Scan() { - s := strings.TrimSpace(scanner.Text()) - if s == "" || s[0] == '#' { - continue - } - var c byte - var r rune - if _, err := fmt.Sscanf(s, ` \x%x |0`, &r, &c); err != nil { - continue - } - mapping[c] = r - charsFound++ - } - - if charsFound < 200 { - log.Fatalf("%q: only %d characters found (wrong page format?)", url, charsFound) - } - - return string(mapping) -} - -func main() { - mibs := map[string]bool{} - all := []string{} - - w := gen.NewCodeWriter() - defer w.WriteGoFile("tables.go", "charmap") - - printf := func(s string, a ...interface{}) { fmt.Fprintf(w, s, a...) } - - printf("import (\n") - printf("\t\"golang.org/x/text/encoding\"\n") - printf("\t\"golang.org/x/text/encoding/internal/identifier\"\n") - printf(")\n\n") - for _, e := range encodings { - varNames := strings.Split(e.varName, ",") - all = append(all, varNames...) - varName := varNames[0] - switch { - case strings.HasPrefix(e.mapping, "http://encoding.spec.whatwg.org/"): - e.mapping = getWHATWG(e.mapping) - case strings.HasPrefix(e.mapping, "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/"): - e.mapping = getUCM(e.mapping) - } - - asciiSuperset, low := strings.HasPrefix(e.mapping, ascii), 0x00 - if asciiSuperset { - low = 0x80 - } - lvn := 1 - if strings.HasPrefix(varName, "ISO") || strings.HasPrefix(varName, "KOI") { - lvn = 3 - } - lowerVarName := strings.ToLower(varName[:lvn]) + varName[lvn:] - printf("// %s is the %s encoding.\n", varName, e.name) - if e.comment != "" { - printf("//\n// %s\n", e.comment) - } - printf("var %s encoding.Encoding = &%s\n\nvar %s = charmap{\nname: %q,\n", - varName, lowerVarName, lowerVarName, e.name) - if mibs[e.mib] { - log.Fatalf("MIB type %q declared multiple times.", e.mib) - } - printf("mib: identifier.%s,\n", e.mib) - printf("asciiSuperset: %t,\n", asciiSuperset) - printf("low: 0x%02x,\n", low) - printf("replacement: 0x%02x,\n", e.replacement) - - printf("decode: [256]utf8Enc{\n") - i, backMapping := 0, map[rune]byte{} - for _, c := range e.mapping { - if _, ok := backMapping[c]; !ok && c != utf8.RuneError { - backMapping[c] = byte(i) - } - var buf [8]byte - n := utf8.EncodeRune(buf[:], c) - if n > 3 { - panic(fmt.Sprintf("rune %q (%U) is too long", c, c)) - } - printf("{%d,[3]byte{0x%02x,0x%02x,0x%02x}},", n, buf[0], buf[1], buf[2]) - if i%2 == 1 { - printf("\n") - } - i++ - } - printf("},\n") - - printf("encode: [256]uint32{\n") - encode := make([]uint32, 0, 256) - for c, i := range backMapping { - encode = append(encode, uint32(i)<<24|uint32(c)) - } - sort.Sort(byRune(encode)) - for len(encode) < cap(encode) { - encode = append(encode, encode[len(encode)-1]) - } - for i, enc := range encode { - printf("0x%08x,", enc) - if i%8 == 7 { - printf("\n") - } - } - printf("},\n}\n") - - // Add an estimate of the size of a single charmap{} struct value, which - // includes two 256 elem arrays of 4 bytes and some extra fields, which - // align to 3 uint64s on 64-bit architectures. - w.Size += 2*4*256 + 3*8 - } - // TODO: add proper line breaking. - printf("var listAll = []encoding.Encoding{\n%s,\n}\n\n", strings.Join(all, ",\n")) -} - -type byRune []uint32 - -func (b byRune) Len() int { return len(b) } -func (b byRune) Less(i, j int) bool { return b[i]&0xffffff < b[j]&0xffffff } -func (b byRune) Swap(i, j int) { b[i], b[j] = b[j], b[i] } diff --git a/vendor/golang.org/x/text/encoding/charmap/tables.go b/vendor/golang.org/x/text/encoding/charmap/tables.go index 5ae8dbcfa..cf7281e9e 100644 --- a/vendor/golang.org/x/text/encoding/charmap/tables.go +++ b/vendor/golang.org/x/text/encoding/charmap/tables.go @@ -1,4 +1,4 @@ -// This file was generated by go generate; DO NOT EDIT +// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. package charmap @@ -7,10 +7,185 @@ import ( "golang.org/x/text/encoding/internal/identifier" ) -// CodePage437 is the IBM Code Page 437 encoding. -var CodePage437 encoding.Encoding = &codePage437 +// CodePage037 is the IBM Code Page 037 encoding. +var CodePage037 *Charmap = &codePage037 -var codePage437 = charmap{ +var codePage037 = Charmap{ + name: "IBM Code Page 037", + mib: identifier.IBM037, + asciiSuperset: false, + low: 0x00, + replacement: 0x3f, + decode: [256]utf8Enc{ + {1, [3]byte{0x00, 0x00, 0x00}}, {1, [3]byte{0x01, 0x00, 0x00}}, + {1, [3]byte{0x02, 0x00, 0x00}}, {1, [3]byte{0x03, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x9c, 0x00}}, {1, [3]byte{0x09, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x86, 0x00}}, {1, [3]byte{0x7f, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x97, 0x00}}, {2, [3]byte{0xc2, 0x8d, 0x00}}, + {2, [3]byte{0xc2, 0x8e, 0x00}}, {1, [3]byte{0x0b, 0x00, 0x00}}, + {1, [3]byte{0x0c, 0x00, 0x00}}, {1, [3]byte{0x0d, 0x00, 0x00}}, + {1, [3]byte{0x0e, 0x00, 0x00}}, {1, [3]byte{0x0f, 0x00, 0x00}}, + {1, [3]byte{0x10, 0x00, 0x00}}, {1, [3]byte{0x11, 0x00, 0x00}}, + {1, [3]byte{0x12, 0x00, 0x00}}, {1, [3]byte{0x13, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x9d, 0x00}}, {2, [3]byte{0xc2, 0x85, 0x00}}, + {1, [3]byte{0x08, 0x00, 0x00}}, {2, [3]byte{0xc2, 0x87, 0x00}}, + {1, [3]byte{0x18, 0x00, 0x00}}, {1, [3]byte{0x19, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x92, 0x00}}, {2, [3]byte{0xc2, 0x8f, 0x00}}, + {1, [3]byte{0x1c, 0x00, 0x00}}, {1, [3]byte{0x1d, 0x00, 0x00}}, + {1, [3]byte{0x1e, 0x00, 0x00}}, {1, [3]byte{0x1f, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x80, 0x00}}, {2, [3]byte{0xc2, 0x81, 0x00}}, + {2, [3]byte{0xc2, 0x82, 0x00}}, {2, [3]byte{0xc2, 0x83, 0x00}}, + {2, [3]byte{0xc2, 0x84, 0x00}}, {1, [3]byte{0x0a, 0x00, 0x00}}, + {1, [3]byte{0x17, 0x00, 0x00}}, {1, [3]byte{0x1b, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x88, 0x00}}, {2, [3]byte{0xc2, 0x89, 0x00}}, + {2, [3]byte{0xc2, 0x8a, 0x00}}, {2, [3]byte{0xc2, 0x8b, 0x00}}, + {2, [3]byte{0xc2, 0x8c, 0x00}}, {1, [3]byte{0x05, 0x00, 0x00}}, + {1, [3]byte{0x06, 0x00, 0x00}}, {1, [3]byte{0x07, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x90, 0x00}}, {2, [3]byte{0xc2, 0x91, 0x00}}, + {1, [3]byte{0x16, 0x00, 0x00}}, {2, [3]byte{0xc2, 0x93, 0x00}}, + {2, [3]byte{0xc2, 0x94, 0x00}}, {2, [3]byte{0xc2, 0x95, 0x00}}, + {2, [3]byte{0xc2, 0x96, 0x00}}, {1, [3]byte{0x04, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x98, 0x00}}, {2, [3]byte{0xc2, 0x99, 0x00}}, + {2, [3]byte{0xc2, 0x9a, 0x00}}, {2, [3]byte{0xc2, 0x9b, 0x00}}, + {1, [3]byte{0x14, 0x00, 0x00}}, {1, [3]byte{0x15, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x9e, 0x00}}, {1, [3]byte{0x1a, 0x00, 0x00}}, + {1, [3]byte{0x20, 0x00, 0x00}}, {2, [3]byte{0xc2, 0xa0, 0x00}}, + {2, [3]byte{0xc3, 0xa2, 0x00}}, {2, [3]byte{0xc3, 0xa4, 0x00}}, + {2, [3]byte{0xc3, 0xa0, 0x00}}, {2, [3]byte{0xc3, 0xa1, 0x00}}, + {2, [3]byte{0xc3, 0xa3, 0x00}}, {2, [3]byte{0xc3, 0xa5, 0x00}}, + {2, [3]byte{0xc3, 0xa7, 0x00}}, {2, [3]byte{0xc3, 0xb1, 0x00}}, + {2, [3]byte{0xc2, 0xa2, 0x00}}, {1, [3]byte{0x2e, 0x00, 0x00}}, + {1, [3]byte{0x3c, 0x00, 0x00}}, {1, [3]byte{0x28, 0x00, 0x00}}, + {1, [3]byte{0x2b, 0x00, 0x00}}, {1, [3]byte{0x7c, 0x00, 0x00}}, + {1, [3]byte{0x26, 0x00, 0x00}}, {2, [3]byte{0xc3, 0xa9, 0x00}}, + {2, [3]byte{0xc3, 0xaa, 0x00}}, {2, [3]byte{0xc3, 0xab, 0x00}}, + {2, [3]byte{0xc3, 0xa8, 0x00}}, {2, [3]byte{0xc3, 0xad, 0x00}}, + {2, [3]byte{0xc3, 0xae, 0x00}}, {2, [3]byte{0xc3, 0xaf, 0x00}}, + {2, [3]byte{0xc3, 0xac, 0x00}}, {2, [3]byte{0xc3, 0x9f, 0x00}}, + {1, [3]byte{0x21, 0x00, 0x00}}, {1, [3]byte{0x24, 0x00, 0x00}}, + {1, [3]byte{0x2a, 0x00, 0x00}}, {1, [3]byte{0x29, 0x00, 0x00}}, + {1, [3]byte{0x3b, 0x00, 0x00}}, {2, [3]byte{0xc2, 0xac, 0x00}}, + {1, [3]byte{0x2d, 0x00, 0x00}}, {1, [3]byte{0x2f, 0x00, 0x00}}, + {2, [3]byte{0xc3, 0x82, 0x00}}, {2, [3]byte{0xc3, 0x84, 0x00}}, + {2, [3]byte{0xc3, 0x80, 0x00}}, {2, [3]byte{0xc3, 0x81, 0x00}}, + {2, [3]byte{0xc3, 0x83, 0x00}}, {2, [3]byte{0xc3, 0x85, 0x00}}, + {2, [3]byte{0xc3, 0x87, 0x00}}, {2, [3]byte{0xc3, 0x91, 0x00}}, + {2, [3]byte{0xc2, 0xa6, 0x00}}, {1, [3]byte{0x2c, 0x00, 0x00}}, + {1, [3]byte{0x25, 0x00, 0x00}}, {1, [3]byte{0x5f, 0x00, 0x00}}, + {1, [3]byte{0x3e, 0x00, 0x00}}, {1, [3]byte{0x3f, 0x00, 0x00}}, + {2, [3]byte{0xc3, 0xb8, 0x00}}, {2, [3]byte{0xc3, 0x89, 0x00}}, + {2, [3]byte{0xc3, 0x8a, 0x00}}, {2, [3]byte{0xc3, 0x8b, 0x00}}, + {2, [3]byte{0xc3, 0x88, 0x00}}, {2, [3]byte{0xc3, 0x8d, 0x00}}, + {2, [3]byte{0xc3, 0x8e, 0x00}}, {2, [3]byte{0xc3, 0x8f, 0x00}}, + {2, [3]byte{0xc3, 0x8c, 0x00}}, {1, [3]byte{0x60, 0x00, 0x00}}, + {1, [3]byte{0x3a, 0x00, 0x00}}, {1, [3]byte{0x23, 0x00, 0x00}}, + {1, [3]byte{0x40, 0x00, 0x00}}, {1, [3]byte{0x27, 0x00, 0x00}}, + {1, [3]byte{0x3d, 0x00, 0x00}}, {1, [3]byte{0x22, 0x00, 0x00}}, + {2, [3]byte{0xc3, 0x98, 0x00}}, {1, [3]byte{0x61, 0x00, 0x00}}, + {1, [3]byte{0x62, 0x00, 0x00}}, {1, [3]byte{0x63, 0x00, 0x00}}, + {1, [3]byte{0x64, 0x00, 0x00}}, {1, [3]byte{0x65, 0x00, 0x00}}, + {1, [3]byte{0x66, 0x00, 0x00}}, {1, [3]byte{0x67, 0x00, 0x00}}, + {1, [3]byte{0x68, 0x00, 0x00}}, {1, [3]byte{0x69, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0xab, 0x00}}, {2, [3]byte{0xc2, 0xbb, 0x00}}, + {2, [3]byte{0xc3, 0xb0, 0x00}}, {2, [3]byte{0xc3, 0xbd, 0x00}}, + {2, [3]byte{0xc3, 0xbe, 0x00}}, {2, [3]byte{0xc2, 0xb1, 0x00}}, + {2, [3]byte{0xc2, 0xb0, 0x00}}, {1, [3]byte{0x6a, 0x00, 0x00}}, + {1, [3]byte{0x6b, 0x00, 0x00}}, {1, [3]byte{0x6c, 0x00, 0x00}}, + {1, [3]byte{0x6d, 0x00, 0x00}}, {1, [3]byte{0x6e, 0x00, 0x00}}, + {1, [3]byte{0x6f, 0x00, 0x00}}, {1, [3]byte{0x70, 0x00, 0x00}}, + {1, [3]byte{0x71, 0x00, 0x00}}, {1, [3]byte{0x72, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0xaa, 0x00}}, {2, [3]byte{0xc2, 0xba, 0x00}}, + {2, [3]byte{0xc3, 0xa6, 0x00}}, {2, [3]byte{0xc2, 0xb8, 0x00}}, + {2, [3]byte{0xc3, 0x86, 0x00}}, {2, [3]byte{0xc2, 0xa4, 0x00}}, + {2, [3]byte{0xc2, 0xb5, 0x00}}, {1, [3]byte{0x7e, 0x00, 0x00}}, + {1, [3]byte{0x73, 0x00, 0x00}}, {1, [3]byte{0x74, 0x00, 0x00}}, + {1, [3]byte{0x75, 0x00, 0x00}}, {1, [3]byte{0x76, 0x00, 0x00}}, + {1, [3]byte{0x77, 0x00, 0x00}}, {1, [3]byte{0x78, 0x00, 0x00}}, + {1, [3]byte{0x79, 0x00, 0x00}}, {1, [3]byte{0x7a, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0xa1, 0x00}}, {2, [3]byte{0xc2, 0xbf, 0x00}}, + {2, [3]byte{0xc3, 0x90, 0x00}}, {2, [3]byte{0xc3, 0x9d, 0x00}}, + {2, [3]byte{0xc3, 0x9e, 0x00}}, {2, [3]byte{0xc2, 0xae, 0x00}}, + {1, [3]byte{0x5e, 0x00, 0x00}}, {2, [3]byte{0xc2, 0xa3, 0x00}}, + {2, [3]byte{0xc2, 0xa5, 0x00}}, {2, [3]byte{0xc2, 0xb7, 0x00}}, + {2, [3]byte{0xc2, 0xa9, 0x00}}, {2, [3]byte{0xc2, 0xa7, 0x00}}, + {2, [3]byte{0xc2, 0xb6, 0x00}}, {2, [3]byte{0xc2, 0xbc, 0x00}}, + {2, [3]byte{0xc2, 0xbd, 0x00}}, {2, [3]byte{0xc2, 0xbe, 0x00}}, + {1, [3]byte{0x5b, 0x00, 0x00}}, {1, [3]byte{0x5d, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0xaf, 0x00}}, {2, [3]byte{0xc2, 0xa8, 0x00}}, + {2, [3]byte{0xc2, 0xb4, 0x00}}, {2, [3]byte{0xc3, 0x97, 0x00}}, + {1, [3]byte{0x7b, 0x00, 0x00}}, {1, [3]byte{0x41, 0x00, 0x00}}, + {1, [3]byte{0x42, 0x00, 0x00}}, {1, [3]byte{0x43, 0x00, 0x00}}, + {1, [3]byte{0x44, 0x00, 0x00}}, {1, [3]byte{0x45, 0x00, 0x00}}, + {1, [3]byte{0x46, 0x00, 0x00}}, {1, [3]byte{0x47, 0x00, 0x00}}, + {1, [3]byte{0x48, 0x00, 0x00}}, {1, [3]byte{0x49, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0xad, 0x00}}, {2, [3]byte{0xc3, 0xb4, 0x00}}, + {2, [3]byte{0xc3, 0xb6, 0x00}}, {2, [3]byte{0xc3, 0xb2, 0x00}}, + {2, [3]byte{0xc3, 0xb3, 0x00}}, {2, [3]byte{0xc3, 0xb5, 0x00}}, + {1, [3]byte{0x7d, 0x00, 0x00}}, {1, [3]byte{0x4a, 0x00, 0x00}}, + {1, [3]byte{0x4b, 0x00, 0x00}}, {1, [3]byte{0x4c, 0x00, 0x00}}, + {1, [3]byte{0x4d, 0x00, 0x00}}, {1, [3]byte{0x4e, 0x00, 0x00}}, + {1, [3]byte{0x4f, 0x00, 0x00}}, {1, [3]byte{0x50, 0x00, 0x00}}, + {1, [3]byte{0x51, 0x00, 0x00}}, {1, [3]byte{0x52, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0xb9, 0x00}}, {2, [3]byte{0xc3, 0xbb, 0x00}}, + {2, [3]byte{0xc3, 0xbc, 0x00}}, {2, [3]byte{0xc3, 0xb9, 0x00}}, + {2, [3]byte{0xc3, 0xba, 0x00}}, {2, [3]byte{0xc3, 0xbf, 0x00}}, + {1, [3]byte{0x5c, 0x00, 0x00}}, {2, [3]byte{0xc3, 0xb7, 0x00}}, + {1, [3]byte{0x53, 0x00, 0x00}}, {1, [3]byte{0x54, 0x00, 0x00}}, + {1, [3]byte{0x55, 0x00, 0x00}}, {1, [3]byte{0x56, 0x00, 0x00}}, + {1, [3]byte{0x57, 0x00, 0x00}}, {1, [3]byte{0x58, 0x00, 0x00}}, + {1, [3]byte{0x59, 0x00, 0x00}}, {1, [3]byte{0x5a, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0xb2, 0x00}}, {2, [3]byte{0xc3, 0x94, 0x00}}, + {2, [3]byte{0xc3, 0x96, 0x00}}, {2, [3]byte{0xc3, 0x92, 0x00}}, + {2, [3]byte{0xc3, 0x93, 0x00}}, {2, [3]byte{0xc3, 0x95, 0x00}}, + {1, [3]byte{0x30, 0x00, 0x00}}, {1, [3]byte{0x31, 0x00, 0x00}}, + {1, [3]byte{0x32, 0x00, 0x00}}, {1, [3]byte{0x33, 0x00, 0x00}}, + {1, [3]byte{0x34, 0x00, 0x00}}, {1, [3]byte{0x35, 0x00, 0x00}}, + {1, [3]byte{0x36, 0x00, 0x00}}, {1, [3]byte{0x37, 0x00, 0x00}}, + {1, [3]byte{0x38, 0x00, 0x00}}, {1, [3]byte{0x39, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0xb3, 0x00}}, {2, [3]byte{0xc3, 0x9b, 0x00}}, + {2, [3]byte{0xc3, 0x9c, 0x00}}, {2, [3]byte{0xc3, 0x99, 0x00}}, + {2, [3]byte{0xc3, 0x9a, 0x00}}, {2, [3]byte{0xc2, 0x9f, 0x00}}, + }, + encode: [256]uint32{ + 0x00000000, 0x01000001, 0x02000002, 0x03000003, 0x37000004, 0x2d000005, 0x2e000006, 0x2f000007, + 0x16000008, 0x05000009, 0x2500000a, 0x0b00000b, 0x0c00000c, 0x0d00000d, 0x0e00000e, 0x0f00000f, + 0x10000010, 0x11000011, 0x12000012, 0x13000013, 0x3c000014, 0x3d000015, 0x32000016, 0x26000017, + 0x18000018, 0x19000019, 0x3f00001a, 0x2700001b, 0x1c00001c, 0x1d00001d, 0x1e00001e, 0x1f00001f, + 0x40000020, 0x5a000021, 0x7f000022, 0x7b000023, 0x5b000024, 0x6c000025, 0x50000026, 0x7d000027, + 0x4d000028, 0x5d000029, 0x5c00002a, 0x4e00002b, 0x6b00002c, 0x6000002d, 0x4b00002e, 0x6100002f, + 0xf0000030, 0xf1000031, 0xf2000032, 0xf3000033, 0xf4000034, 0xf5000035, 0xf6000036, 0xf7000037, + 0xf8000038, 0xf9000039, 0x7a00003a, 0x5e00003b, 0x4c00003c, 0x7e00003d, 0x6e00003e, 0x6f00003f, + 0x7c000040, 0xc1000041, 0xc2000042, 0xc3000043, 0xc4000044, 0xc5000045, 0xc6000046, 0xc7000047, + 0xc8000048, 0xc9000049, 0xd100004a, 0xd200004b, 0xd300004c, 0xd400004d, 0xd500004e, 0xd600004f, + 0xd7000050, 0xd8000051, 0xd9000052, 0xe2000053, 0xe3000054, 0xe4000055, 0xe5000056, 0xe6000057, + 0xe7000058, 0xe8000059, 0xe900005a, 0xba00005b, 0xe000005c, 0xbb00005d, 0xb000005e, 0x6d00005f, + 0x79000060, 0x81000061, 0x82000062, 0x83000063, 0x84000064, 0x85000065, 0x86000066, 0x87000067, + 0x88000068, 0x89000069, 0x9100006a, 0x9200006b, 0x9300006c, 0x9400006d, 0x9500006e, 0x9600006f, + 0x97000070, 0x98000071, 0x99000072, 0xa2000073, 0xa3000074, 0xa4000075, 0xa5000076, 0xa6000077, + 0xa7000078, 0xa8000079, 0xa900007a, 0xc000007b, 0x4f00007c, 0xd000007d, 0xa100007e, 0x0700007f, + 0x20000080, 0x21000081, 0x22000082, 0x23000083, 0x24000084, 0x15000085, 0x06000086, 0x17000087, + 0x28000088, 0x29000089, 0x2a00008a, 0x2b00008b, 0x2c00008c, 0x0900008d, 0x0a00008e, 0x1b00008f, + 0x30000090, 0x31000091, 0x1a000092, 0x33000093, 0x34000094, 0x35000095, 0x36000096, 0x08000097, + 0x38000098, 0x39000099, 0x3a00009a, 0x3b00009b, 0x0400009c, 0x1400009d, 0x3e00009e, 0xff00009f, + 0x410000a0, 0xaa0000a1, 0x4a0000a2, 0xb10000a3, 0x9f0000a4, 0xb20000a5, 0x6a0000a6, 0xb50000a7, + 0xbd0000a8, 0xb40000a9, 0x9a0000aa, 0x8a0000ab, 0x5f0000ac, 0xca0000ad, 0xaf0000ae, 0xbc0000af, + 0x900000b0, 0x8f0000b1, 0xea0000b2, 0xfa0000b3, 0xbe0000b4, 0xa00000b5, 0xb60000b6, 0xb30000b7, + 0x9d0000b8, 0xda0000b9, 0x9b0000ba, 0x8b0000bb, 0xb70000bc, 0xb80000bd, 0xb90000be, 0xab0000bf, + 0x640000c0, 0x650000c1, 0x620000c2, 0x660000c3, 0x630000c4, 0x670000c5, 0x9e0000c6, 0x680000c7, + 0x740000c8, 0x710000c9, 0x720000ca, 0x730000cb, 0x780000cc, 0x750000cd, 0x760000ce, 0x770000cf, + 0xac0000d0, 0x690000d1, 0xed0000d2, 0xee0000d3, 0xeb0000d4, 0xef0000d5, 0xec0000d6, 0xbf0000d7, + 0x800000d8, 0xfd0000d9, 0xfe0000da, 0xfb0000db, 0xfc0000dc, 0xad0000dd, 0xae0000de, 0x590000df, + 0x440000e0, 0x450000e1, 0x420000e2, 0x460000e3, 0x430000e4, 0x470000e5, 0x9c0000e6, 0x480000e7, + 0x540000e8, 0x510000e9, 0x520000ea, 0x530000eb, 0x580000ec, 0x550000ed, 0x560000ee, 0x570000ef, + 0x8c0000f0, 0x490000f1, 0xcd0000f2, 0xce0000f3, 0xcb0000f4, 0xcf0000f5, 0xcc0000f6, 0xe10000f7, + 0x700000f8, 0xdd0000f9, 0xde0000fa, 0xdb0000fb, 0xdc0000fc, 0x8d0000fd, 0x8e0000fe, 0xdf0000ff, + }, +} + +// CodePage437 is the IBM Code Page 437 encoding. +var CodePage437 *Charmap = &codePage437 + +var codePage437 = Charmap{ name: "IBM Code Page 437", mib: identifier.PC8CodePage437, asciiSuperset: true, @@ -183,9 +358,9 @@ var codePage437 = charmap{ } // CodePage850 is the IBM Code Page 850 encoding. -var CodePage850 encoding.Encoding = &codePage850 +var CodePage850 *Charmap = &codePage850 -var codePage850 = charmap{ +var codePage850 = Charmap{ name: "IBM Code Page 850", mib: identifier.PC850Multilingual, asciiSuperset: true, @@ -358,9 +533,9 @@ var codePage850 = charmap{ } // CodePage852 is the IBM Code Page 852 encoding. -var CodePage852 encoding.Encoding = &codePage852 +var CodePage852 *Charmap = &codePage852 -var codePage852 = charmap{ +var codePage852 = Charmap{ name: "IBM Code Page 852", mib: identifier.PCp852, asciiSuperset: true, @@ -533,9 +708,9 @@ var codePage852 = charmap{ } // CodePage855 is the IBM Code Page 855 encoding. -var CodePage855 encoding.Encoding = &codePage855 +var CodePage855 *Charmap = &codePage855 -var codePage855 = charmap{ +var codePage855 = Charmap{ name: "IBM Code Page 855", mib: identifier.IBM855, asciiSuperset: true, @@ -708,9 +883,9 @@ var codePage855 = charmap{ } // CodePage858 is the Windows Code Page 858 encoding. -var CodePage858 encoding.Encoding = &codePage858 +var CodePage858 *Charmap = &codePage858 -var codePage858 = charmap{ +var codePage858 = Charmap{ name: "Windows Code Page 858", mib: identifier.IBM00858, asciiSuperset: true, @@ -883,9 +1058,9 @@ var codePage858 = charmap{ } // CodePage860 is the IBM Code Page 860 encoding. -var CodePage860 encoding.Encoding = &codePage860 +var CodePage860 *Charmap = &codePage860 -var codePage860 = charmap{ +var codePage860 = Charmap{ name: "IBM Code Page 860", mib: identifier.IBM860, asciiSuperset: true, @@ -1058,9 +1233,9 @@ var codePage860 = charmap{ } // CodePage862 is the IBM Code Page 862 encoding. -var CodePage862 encoding.Encoding = &codePage862 +var CodePage862 *Charmap = &codePage862 -var codePage862 = charmap{ +var codePage862 = Charmap{ name: "IBM Code Page 862", mib: identifier.PC862LatinHebrew, asciiSuperset: true, @@ -1233,9 +1408,9 @@ var codePage862 = charmap{ } // CodePage863 is the IBM Code Page 863 encoding. -var CodePage863 encoding.Encoding = &codePage863 +var CodePage863 *Charmap = &codePage863 -var codePage863 = charmap{ +var codePage863 = Charmap{ name: "IBM Code Page 863", mib: identifier.IBM863, asciiSuperset: true, @@ -1408,9 +1583,9 @@ var codePage863 = charmap{ } // CodePage865 is the IBM Code Page 865 encoding. -var CodePage865 encoding.Encoding = &codePage865 +var CodePage865 *Charmap = &codePage865 -var codePage865 = charmap{ +var codePage865 = Charmap{ name: "IBM Code Page 865", mib: identifier.IBM865, asciiSuperset: true, @@ -1583,9 +1758,9 @@ var codePage865 = charmap{ } // CodePage866 is the IBM Code Page 866 encoding. -var CodePage866 encoding.Encoding = &codePage866 +var CodePage866 *Charmap = &codePage866 -var codePage866 = charmap{ +var codePage866 = Charmap{ name: "IBM Code Page 866", mib: identifier.IBM866, asciiSuperset: true, @@ -1757,10 +1932,360 @@ var codePage866 = charmap{ }, } -// ISO8859_1 is the ISO 8859-1 encoding. -var ISO8859_1 encoding.Encoding = &iso8859_1 +// CodePage1047 is the IBM Code Page 1047 encoding. +var CodePage1047 *Charmap = &codePage1047 -var iso8859_1 = charmap{ +var codePage1047 = Charmap{ + name: "IBM Code Page 1047", + mib: identifier.IBM1047, + asciiSuperset: false, + low: 0x00, + replacement: 0x3f, + decode: [256]utf8Enc{ + {1, [3]byte{0x00, 0x00, 0x00}}, {1, [3]byte{0x01, 0x00, 0x00}}, + {1, [3]byte{0x02, 0x00, 0x00}}, {1, [3]byte{0x03, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x9c, 0x00}}, {1, [3]byte{0x09, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x86, 0x00}}, {1, [3]byte{0x7f, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x97, 0x00}}, {2, [3]byte{0xc2, 0x8d, 0x00}}, + {2, [3]byte{0xc2, 0x8e, 0x00}}, {1, [3]byte{0x0b, 0x00, 0x00}}, + {1, [3]byte{0x0c, 0x00, 0x00}}, {1, [3]byte{0x0d, 0x00, 0x00}}, + {1, [3]byte{0x0e, 0x00, 0x00}}, {1, [3]byte{0x0f, 0x00, 0x00}}, + {1, [3]byte{0x10, 0x00, 0x00}}, {1, [3]byte{0x11, 0x00, 0x00}}, + {1, [3]byte{0x12, 0x00, 0x00}}, {1, [3]byte{0x13, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x9d, 0x00}}, {2, [3]byte{0xc2, 0x85, 0x00}}, + {1, [3]byte{0x08, 0x00, 0x00}}, {2, [3]byte{0xc2, 0x87, 0x00}}, + {1, [3]byte{0x18, 0x00, 0x00}}, {1, [3]byte{0x19, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x92, 0x00}}, {2, [3]byte{0xc2, 0x8f, 0x00}}, + {1, [3]byte{0x1c, 0x00, 0x00}}, {1, [3]byte{0x1d, 0x00, 0x00}}, + {1, [3]byte{0x1e, 0x00, 0x00}}, {1, [3]byte{0x1f, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x80, 0x00}}, {2, [3]byte{0xc2, 0x81, 0x00}}, + {2, [3]byte{0xc2, 0x82, 0x00}}, {2, [3]byte{0xc2, 0x83, 0x00}}, + {2, [3]byte{0xc2, 0x84, 0x00}}, {1, [3]byte{0x0a, 0x00, 0x00}}, + {1, [3]byte{0x17, 0x00, 0x00}}, {1, [3]byte{0x1b, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x88, 0x00}}, {2, [3]byte{0xc2, 0x89, 0x00}}, + {2, [3]byte{0xc2, 0x8a, 0x00}}, {2, [3]byte{0xc2, 0x8b, 0x00}}, + {2, [3]byte{0xc2, 0x8c, 0x00}}, {1, [3]byte{0x05, 0x00, 0x00}}, + {1, [3]byte{0x06, 0x00, 0x00}}, {1, [3]byte{0x07, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x90, 0x00}}, {2, [3]byte{0xc2, 0x91, 0x00}}, + {1, [3]byte{0x16, 0x00, 0x00}}, {2, [3]byte{0xc2, 0x93, 0x00}}, + {2, [3]byte{0xc2, 0x94, 0x00}}, {2, [3]byte{0xc2, 0x95, 0x00}}, + {2, [3]byte{0xc2, 0x96, 0x00}}, {1, [3]byte{0x04, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x98, 0x00}}, {2, [3]byte{0xc2, 0x99, 0x00}}, + {2, [3]byte{0xc2, 0x9a, 0x00}}, {2, [3]byte{0xc2, 0x9b, 0x00}}, + {1, [3]byte{0x14, 0x00, 0x00}}, {1, [3]byte{0x15, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x9e, 0x00}}, {1, [3]byte{0x1a, 0x00, 0x00}}, + {1, [3]byte{0x20, 0x00, 0x00}}, {2, [3]byte{0xc2, 0xa0, 0x00}}, + {2, [3]byte{0xc3, 0xa2, 0x00}}, {2, [3]byte{0xc3, 0xa4, 0x00}}, + {2, [3]byte{0xc3, 0xa0, 0x00}}, {2, [3]byte{0xc3, 0xa1, 0x00}}, + {2, [3]byte{0xc3, 0xa3, 0x00}}, {2, [3]byte{0xc3, 0xa5, 0x00}}, + {2, [3]byte{0xc3, 0xa7, 0x00}}, {2, [3]byte{0xc3, 0xb1, 0x00}}, + {2, [3]byte{0xc2, 0xa2, 0x00}}, {1, [3]byte{0x2e, 0x00, 0x00}}, + {1, [3]byte{0x3c, 0x00, 0x00}}, {1, [3]byte{0x28, 0x00, 0x00}}, + {1, [3]byte{0x2b, 0x00, 0x00}}, {1, [3]byte{0x7c, 0x00, 0x00}}, + {1, [3]byte{0x26, 0x00, 0x00}}, {2, [3]byte{0xc3, 0xa9, 0x00}}, + {2, [3]byte{0xc3, 0xaa, 0x00}}, {2, [3]byte{0xc3, 0xab, 0x00}}, + {2, [3]byte{0xc3, 0xa8, 0x00}}, {2, [3]byte{0xc3, 0xad, 0x00}}, + {2, [3]byte{0xc3, 0xae, 0x00}}, {2, [3]byte{0xc3, 0xaf, 0x00}}, + {2, [3]byte{0xc3, 0xac, 0x00}}, {2, [3]byte{0xc3, 0x9f, 0x00}}, + {1, [3]byte{0x21, 0x00, 0x00}}, {1, [3]byte{0x24, 0x00, 0x00}}, + {1, [3]byte{0x2a, 0x00, 0x00}}, {1, [3]byte{0x29, 0x00, 0x00}}, + {1, [3]byte{0x3b, 0x00, 0x00}}, {1, [3]byte{0x5e, 0x00, 0x00}}, + {1, [3]byte{0x2d, 0x00, 0x00}}, {1, [3]byte{0x2f, 0x00, 0x00}}, + {2, [3]byte{0xc3, 0x82, 0x00}}, {2, [3]byte{0xc3, 0x84, 0x00}}, + {2, [3]byte{0xc3, 0x80, 0x00}}, {2, [3]byte{0xc3, 0x81, 0x00}}, + {2, [3]byte{0xc3, 0x83, 0x00}}, {2, [3]byte{0xc3, 0x85, 0x00}}, + {2, [3]byte{0xc3, 0x87, 0x00}}, {2, [3]byte{0xc3, 0x91, 0x00}}, + {2, [3]byte{0xc2, 0xa6, 0x00}}, {1, [3]byte{0x2c, 0x00, 0x00}}, + {1, [3]byte{0x25, 0x00, 0x00}}, {1, [3]byte{0x5f, 0x00, 0x00}}, + {1, [3]byte{0x3e, 0x00, 0x00}}, {1, [3]byte{0x3f, 0x00, 0x00}}, + {2, [3]byte{0xc3, 0xb8, 0x00}}, {2, [3]byte{0xc3, 0x89, 0x00}}, + {2, [3]byte{0xc3, 0x8a, 0x00}}, {2, [3]byte{0xc3, 0x8b, 0x00}}, + {2, [3]byte{0xc3, 0x88, 0x00}}, {2, [3]byte{0xc3, 0x8d, 0x00}}, + {2, [3]byte{0xc3, 0x8e, 0x00}}, {2, [3]byte{0xc3, 0x8f, 0x00}}, + {2, [3]byte{0xc3, 0x8c, 0x00}}, {1, [3]byte{0x60, 0x00, 0x00}}, + {1, [3]byte{0x3a, 0x00, 0x00}}, {1, [3]byte{0x23, 0x00, 0x00}}, + {1, [3]byte{0x40, 0x00, 0x00}}, {1, [3]byte{0x27, 0x00, 0x00}}, + {1, [3]byte{0x3d, 0x00, 0x00}}, {1, [3]byte{0x22, 0x00, 0x00}}, + {2, [3]byte{0xc3, 0x98, 0x00}}, {1, [3]byte{0x61, 0x00, 0x00}}, + {1, [3]byte{0x62, 0x00, 0x00}}, {1, [3]byte{0x63, 0x00, 0x00}}, + {1, [3]byte{0x64, 0x00, 0x00}}, {1, [3]byte{0x65, 0x00, 0x00}}, + {1, [3]byte{0x66, 0x00, 0x00}}, {1, [3]byte{0x67, 0x00, 0x00}}, + {1, [3]byte{0x68, 0x00, 0x00}}, {1, [3]byte{0x69, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0xab, 0x00}}, {2, [3]byte{0xc2, 0xbb, 0x00}}, + {2, [3]byte{0xc3, 0xb0, 0x00}}, {2, [3]byte{0xc3, 0xbd, 0x00}}, + {2, [3]byte{0xc3, 0xbe, 0x00}}, {2, [3]byte{0xc2, 0xb1, 0x00}}, + {2, [3]byte{0xc2, 0xb0, 0x00}}, {1, [3]byte{0x6a, 0x00, 0x00}}, + {1, [3]byte{0x6b, 0x00, 0x00}}, {1, [3]byte{0x6c, 0x00, 0x00}}, + {1, [3]byte{0x6d, 0x00, 0x00}}, {1, [3]byte{0x6e, 0x00, 0x00}}, + {1, [3]byte{0x6f, 0x00, 0x00}}, {1, [3]byte{0x70, 0x00, 0x00}}, + {1, [3]byte{0x71, 0x00, 0x00}}, {1, [3]byte{0x72, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0xaa, 0x00}}, {2, [3]byte{0xc2, 0xba, 0x00}}, + {2, [3]byte{0xc3, 0xa6, 0x00}}, {2, [3]byte{0xc2, 0xb8, 0x00}}, + {2, [3]byte{0xc3, 0x86, 0x00}}, {2, [3]byte{0xc2, 0xa4, 0x00}}, + {2, [3]byte{0xc2, 0xb5, 0x00}}, {1, [3]byte{0x7e, 0x00, 0x00}}, + {1, [3]byte{0x73, 0x00, 0x00}}, {1, [3]byte{0x74, 0x00, 0x00}}, + {1, [3]byte{0x75, 0x00, 0x00}}, {1, [3]byte{0x76, 0x00, 0x00}}, + {1, [3]byte{0x77, 0x00, 0x00}}, {1, [3]byte{0x78, 0x00, 0x00}}, + {1, [3]byte{0x79, 0x00, 0x00}}, {1, [3]byte{0x7a, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0xa1, 0x00}}, {2, [3]byte{0xc2, 0xbf, 0x00}}, + {2, [3]byte{0xc3, 0x90, 0x00}}, {1, [3]byte{0x5b, 0x00, 0x00}}, + {2, [3]byte{0xc3, 0x9e, 0x00}}, {2, [3]byte{0xc2, 0xae, 0x00}}, + {2, [3]byte{0xc2, 0xac, 0x00}}, {2, [3]byte{0xc2, 0xa3, 0x00}}, + {2, [3]byte{0xc2, 0xa5, 0x00}}, {2, [3]byte{0xc2, 0xb7, 0x00}}, + {2, [3]byte{0xc2, 0xa9, 0x00}}, {2, [3]byte{0xc2, 0xa7, 0x00}}, + {2, [3]byte{0xc2, 0xb6, 0x00}}, {2, [3]byte{0xc2, 0xbc, 0x00}}, + {2, [3]byte{0xc2, 0xbd, 0x00}}, {2, [3]byte{0xc2, 0xbe, 0x00}}, + {2, [3]byte{0xc3, 0x9d, 0x00}}, {2, [3]byte{0xc2, 0xa8, 0x00}}, + {2, [3]byte{0xc2, 0xaf, 0x00}}, {1, [3]byte{0x5d, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0xb4, 0x00}}, {2, [3]byte{0xc3, 0x97, 0x00}}, + {1, [3]byte{0x7b, 0x00, 0x00}}, {1, [3]byte{0x41, 0x00, 0x00}}, + {1, [3]byte{0x42, 0x00, 0x00}}, {1, [3]byte{0x43, 0x00, 0x00}}, + {1, [3]byte{0x44, 0x00, 0x00}}, {1, [3]byte{0x45, 0x00, 0x00}}, + {1, [3]byte{0x46, 0x00, 0x00}}, {1, [3]byte{0x47, 0x00, 0x00}}, + {1, [3]byte{0x48, 0x00, 0x00}}, {1, [3]byte{0x49, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0xad, 0x00}}, {2, [3]byte{0xc3, 0xb4, 0x00}}, + {2, [3]byte{0xc3, 0xb6, 0x00}}, {2, [3]byte{0xc3, 0xb2, 0x00}}, + {2, [3]byte{0xc3, 0xb3, 0x00}}, {2, [3]byte{0xc3, 0xb5, 0x00}}, + {1, [3]byte{0x7d, 0x00, 0x00}}, {1, [3]byte{0x4a, 0x00, 0x00}}, + {1, [3]byte{0x4b, 0x00, 0x00}}, {1, [3]byte{0x4c, 0x00, 0x00}}, + {1, [3]byte{0x4d, 0x00, 0x00}}, {1, [3]byte{0x4e, 0x00, 0x00}}, + {1, [3]byte{0x4f, 0x00, 0x00}}, {1, [3]byte{0x50, 0x00, 0x00}}, + {1, [3]byte{0x51, 0x00, 0x00}}, {1, [3]byte{0x52, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0xb9, 0x00}}, {2, [3]byte{0xc3, 0xbb, 0x00}}, + {2, [3]byte{0xc3, 0xbc, 0x00}}, {2, [3]byte{0xc3, 0xb9, 0x00}}, + {2, [3]byte{0xc3, 0xba, 0x00}}, {2, [3]byte{0xc3, 0xbf, 0x00}}, + {1, [3]byte{0x5c, 0x00, 0x00}}, {2, [3]byte{0xc3, 0xb7, 0x00}}, + {1, [3]byte{0x53, 0x00, 0x00}}, {1, [3]byte{0x54, 0x00, 0x00}}, + {1, [3]byte{0x55, 0x00, 0x00}}, {1, [3]byte{0x56, 0x00, 0x00}}, + {1, [3]byte{0x57, 0x00, 0x00}}, {1, [3]byte{0x58, 0x00, 0x00}}, + {1, [3]byte{0x59, 0x00, 0x00}}, {1, [3]byte{0x5a, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0xb2, 0x00}}, {2, [3]byte{0xc3, 0x94, 0x00}}, + {2, [3]byte{0xc3, 0x96, 0x00}}, {2, [3]byte{0xc3, 0x92, 0x00}}, + {2, [3]byte{0xc3, 0x93, 0x00}}, {2, [3]byte{0xc3, 0x95, 0x00}}, + {1, [3]byte{0x30, 0x00, 0x00}}, {1, [3]byte{0x31, 0x00, 0x00}}, + {1, [3]byte{0x32, 0x00, 0x00}}, {1, [3]byte{0x33, 0x00, 0x00}}, + {1, [3]byte{0x34, 0x00, 0x00}}, {1, [3]byte{0x35, 0x00, 0x00}}, + {1, [3]byte{0x36, 0x00, 0x00}}, {1, [3]byte{0x37, 0x00, 0x00}}, + {1, [3]byte{0x38, 0x00, 0x00}}, {1, [3]byte{0x39, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0xb3, 0x00}}, {2, [3]byte{0xc3, 0x9b, 0x00}}, + {2, [3]byte{0xc3, 0x9c, 0x00}}, {2, [3]byte{0xc3, 0x99, 0x00}}, + {2, [3]byte{0xc3, 0x9a, 0x00}}, {2, [3]byte{0xc2, 0x9f, 0x00}}, + }, + encode: [256]uint32{ + 0x00000000, 0x01000001, 0x02000002, 0x03000003, 0x37000004, 0x2d000005, 0x2e000006, 0x2f000007, + 0x16000008, 0x05000009, 0x2500000a, 0x0b00000b, 0x0c00000c, 0x0d00000d, 0x0e00000e, 0x0f00000f, + 0x10000010, 0x11000011, 0x12000012, 0x13000013, 0x3c000014, 0x3d000015, 0x32000016, 0x26000017, + 0x18000018, 0x19000019, 0x3f00001a, 0x2700001b, 0x1c00001c, 0x1d00001d, 0x1e00001e, 0x1f00001f, + 0x40000020, 0x5a000021, 0x7f000022, 0x7b000023, 0x5b000024, 0x6c000025, 0x50000026, 0x7d000027, + 0x4d000028, 0x5d000029, 0x5c00002a, 0x4e00002b, 0x6b00002c, 0x6000002d, 0x4b00002e, 0x6100002f, + 0xf0000030, 0xf1000031, 0xf2000032, 0xf3000033, 0xf4000034, 0xf5000035, 0xf6000036, 0xf7000037, + 0xf8000038, 0xf9000039, 0x7a00003a, 0x5e00003b, 0x4c00003c, 0x7e00003d, 0x6e00003e, 0x6f00003f, + 0x7c000040, 0xc1000041, 0xc2000042, 0xc3000043, 0xc4000044, 0xc5000045, 0xc6000046, 0xc7000047, + 0xc8000048, 0xc9000049, 0xd100004a, 0xd200004b, 0xd300004c, 0xd400004d, 0xd500004e, 0xd600004f, + 0xd7000050, 0xd8000051, 0xd9000052, 0xe2000053, 0xe3000054, 0xe4000055, 0xe5000056, 0xe6000057, + 0xe7000058, 0xe8000059, 0xe900005a, 0xad00005b, 0xe000005c, 0xbd00005d, 0x5f00005e, 0x6d00005f, + 0x79000060, 0x81000061, 0x82000062, 0x83000063, 0x84000064, 0x85000065, 0x86000066, 0x87000067, + 0x88000068, 0x89000069, 0x9100006a, 0x9200006b, 0x9300006c, 0x9400006d, 0x9500006e, 0x9600006f, + 0x97000070, 0x98000071, 0x99000072, 0xa2000073, 0xa3000074, 0xa4000075, 0xa5000076, 0xa6000077, + 0xa7000078, 0xa8000079, 0xa900007a, 0xc000007b, 0x4f00007c, 0xd000007d, 0xa100007e, 0x0700007f, + 0x20000080, 0x21000081, 0x22000082, 0x23000083, 0x24000084, 0x15000085, 0x06000086, 0x17000087, + 0x28000088, 0x29000089, 0x2a00008a, 0x2b00008b, 0x2c00008c, 0x0900008d, 0x0a00008e, 0x1b00008f, + 0x30000090, 0x31000091, 0x1a000092, 0x33000093, 0x34000094, 0x35000095, 0x36000096, 0x08000097, + 0x38000098, 0x39000099, 0x3a00009a, 0x3b00009b, 0x0400009c, 0x1400009d, 0x3e00009e, 0xff00009f, + 0x410000a0, 0xaa0000a1, 0x4a0000a2, 0xb10000a3, 0x9f0000a4, 0xb20000a5, 0x6a0000a6, 0xb50000a7, + 0xbb0000a8, 0xb40000a9, 0x9a0000aa, 0x8a0000ab, 0xb00000ac, 0xca0000ad, 0xaf0000ae, 0xbc0000af, + 0x900000b0, 0x8f0000b1, 0xea0000b2, 0xfa0000b3, 0xbe0000b4, 0xa00000b5, 0xb60000b6, 0xb30000b7, + 0x9d0000b8, 0xda0000b9, 0x9b0000ba, 0x8b0000bb, 0xb70000bc, 0xb80000bd, 0xb90000be, 0xab0000bf, + 0x640000c0, 0x650000c1, 0x620000c2, 0x660000c3, 0x630000c4, 0x670000c5, 0x9e0000c6, 0x680000c7, + 0x740000c8, 0x710000c9, 0x720000ca, 0x730000cb, 0x780000cc, 0x750000cd, 0x760000ce, 0x770000cf, + 0xac0000d0, 0x690000d1, 0xed0000d2, 0xee0000d3, 0xeb0000d4, 0xef0000d5, 0xec0000d6, 0xbf0000d7, + 0x800000d8, 0xfd0000d9, 0xfe0000da, 0xfb0000db, 0xfc0000dc, 0xba0000dd, 0xae0000de, 0x590000df, + 0x440000e0, 0x450000e1, 0x420000e2, 0x460000e3, 0x430000e4, 0x470000e5, 0x9c0000e6, 0x480000e7, + 0x540000e8, 0x510000e9, 0x520000ea, 0x530000eb, 0x580000ec, 0x550000ed, 0x560000ee, 0x570000ef, + 0x8c0000f0, 0x490000f1, 0xcd0000f2, 0xce0000f3, 0xcb0000f4, 0xcf0000f5, 0xcc0000f6, 0xe10000f7, + 0x700000f8, 0xdd0000f9, 0xde0000fa, 0xdb0000fb, 0xdc0000fc, 0x8d0000fd, 0x8e0000fe, 0xdf0000ff, + }, +} + +// CodePage1140 is the IBM Code Page 1140 encoding. +var CodePage1140 *Charmap = &codePage1140 + +var codePage1140 = Charmap{ + name: "IBM Code Page 1140", + mib: identifier.IBM01140, + asciiSuperset: false, + low: 0x00, + replacement: 0x3f, + decode: [256]utf8Enc{ + {1, [3]byte{0x00, 0x00, 0x00}}, {1, [3]byte{0x01, 0x00, 0x00}}, + {1, [3]byte{0x02, 0x00, 0x00}}, {1, [3]byte{0x03, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x9c, 0x00}}, {1, [3]byte{0x09, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x86, 0x00}}, {1, [3]byte{0x7f, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x97, 0x00}}, {2, [3]byte{0xc2, 0x8d, 0x00}}, + {2, [3]byte{0xc2, 0x8e, 0x00}}, {1, [3]byte{0x0b, 0x00, 0x00}}, + {1, [3]byte{0x0c, 0x00, 0x00}}, {1, [3]byte{0x0d, 0x00, 0x00}}, + {1, [3]byte{0x0e, 0x00, 0x00}}, {1, [3]byte{0x0f, 0x00, 0x00}}, + {1, [3]byte{0x10, 0x00, 0x00}}, {1, [3]byte{0x11, 0x00, 0x00}}, + {1, [3]byte{0x12, 0x00, 0x00}}, {1, [3]byte{0x13, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x9d, 0x00}}, {2, [3]byte{0xc2, 0x85, 0x00}}, + {1, [3]byte{0x08, 0x00, 0x00}}, {2, [3]byte{0xc2, 0x87, 0x00}}, + {1, [3]byte{0x18, 0x00, 0x00}}, {1, [3]byte{0x19, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x92, 0x00}}, {2, [3]byte{0xc2, 0x8f, 0x00}}, + {1, [3]byte{0x1c, 0x00, 0x00}}, {1, [3]byte{0x1d, 0x00, 0x00}}, + {1, [3]byte{0x1e, 0x00, 0x00}}, {1, [3]byte{0x1f, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x80, 0x00}}, {2, [3]byte{0xc2, 0x81, 0x00}}, + {2, [3]byte{0xc2, 0x82, 0x00}}, {2, [3]byte{0xc2, 0x83, 0x00}}, + {2, [3]byte{0xc2, 0x84, 0x00}}, {1, [3]byte{0x0a, 0x00, 0x00}}, + {1, [3]byte{0x17, 0x00, 0x00}}, {1, [3]byte{0x1b, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x88, 0x00}}, {2, [3]byte{0xc2, 0x89, 0x00}}, + {2, [3]byte{0xc2, 0x8a, 0x00}}, {2, [3]byte{0xc2, 0x8b, 0x00}}, + {2, [3]byte{0xc2, 0x8c, 0x00}}, {1, [3]byte{0x05, 0x00, 0x00}}, + {1, [3]byte{0x06, 0x00, 0x00}}, {1, [3]byte{0x07, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x90, 0x00}}, {2, [3]byte{0xc2, 0x91, 0x00}}, + {1, [3]byte{0x16, 0x00, 0x00}}, {2, [3]byte{0xc2, 0x93, 0x00}}, + {2, [3]byte{0xc2, 0x94, 0x00}}, {2, [3]byte{0xc2, 0x95, 0x00}}, + {2, [3]byte{0xc2, 0x96, 0x00}}, {1, [3]byte{0x04, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x98, 0x00}}, {2, [3]byte{0xc2, 0x99, 0x00}}, + {2, [3]byte{0xc2, 0x9a, 0x00}}, {2, [3]byte{0xc2, 0x9b, 0x00}}, + {1, [3]byte{0x14, 0x00, 0x00}}, {1, [3]byte{0x15, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x9e, 0x00}}, {1, [3]byte{0x1a, 0x00, 0x00}}, + {1, [3]byte{0x20, 0x00, 0x00}}, {2, [3]byte{0xc2, 0xa0, 0x00}}, + {2, [3]byte{0xc3, 0xa2, 0x00}}, {2, [3]byte{0xc3, 0xa4, 0x00}}, + {2, [3]byte{0xc3, 0xa0, 0x00}}, {2, [3]byte{0xc3, 0xa1, 0x00}}, + {2, [3]byte{0xc3, 0xa3, 0x00}}, {2, [3]byte{0xc3, 0xa5, 0x00}}, + {2, [3]byte{0xc3, 0xa7, 0x00}}, {2, [3]byte{0xc3, 0xb1, 0x00}}, + {2, [3]byte{0xc2, 0xa2, 0x00}}, {1, [3]byte{0x2e, 0x00, 0x00}}, + {1, [3]byte{0x3c, 0x00, 0x00}}, {1, [3]byte{0x28, 0x00, 0x00}}, + {1, [3]byte{0x2b, 0x00, 0x00}}, {1, [3]byte{0x7c, 0x00, 0x00}}, + {1, [3]byte{0x26, 0x00, 0x00}}, {2, [3]byte{0xc3, 0xa9, 0x00}}, + {2, [3]byte{0xc3, 0xaa, 0x00}}, {2, [3]byte{0xc3, 0xab, 0x00}}, + {2, [3]byte{0xc3, 0xa8, 0x00}}, {2, [3]byte{0xc3, 0xad, 0x00}}, + {2, [3]byte{0xc3, 0xae, 0x00}}, {2, [3]byte{0xc3, 0xaf, 0x00}}, + {2, [3]byte{0xc3, 0xac, 0x00}}, {2, [3]byte{0xc3, 0x9f, 0x00}}, + {1, [3]byte{0x21, 0x00, 0x00}}, {1, [3]byte{0x24, 0x00, 0x00}}, + {1, [3]byte{0x2a, 0x00, 0x00}}, {1, [3]byte{0x29, 0x00, 0x00}}, + {1, [3]byte{0x3b, 0x00, 0x00}}, {2, [3]byte{0xc2, 0xac, 0x00}}, + {1, [3]byte{0x2d, 0x00, 0x00}}, {1, [3]byte{0x2f, 0x00, 0x00}}, + {2, [3]byte{0xc3, 0x82, 0x00}}, {2, [3]byte{0xc3, 0x84, 0x00}}, + {2, [3]byte{0xc3, 0x80, 0x00}}, {2, [3]byte{0xc3, 0x81, 0x00}}, + {2, [3]byte{0xc3, 0x83, 0x00}}, {2, [3]byte{0xc3, 0x85, 0x00}}, + {2, [3]byte{0xc3, 0x87, 0x00}}, {2, [3]byte{0xc3, 0x91, 0x00}}, + {2, [3]byte{0xc2, 0xa6, 0x00}}, {1, [3]byte{0x2c, 0x00, 0x00}}, + {1, [3]byte{0x25, 0x00, 0x00}}, {1, [3]byte{0x5f, 0x00, 0x00}}, + {1, [3]byte{0x3e, 0x00, 0x00}}, {1, [3]byte{0x3f, 0x00, 0x00}}, + {2, [3]byte{0xc3, 0xb8, 0x00}}, {2, [3]byte{0xc3, 0x89, 0x00}}, + {2, [3]byte{0xc3, 0x8a, 0x00}}, {2, [3]byte{0xc3, 0x8b, 0x00}}, + {2, [3]byte{0xc3, 0x88, 0x00}}, {2, [3]byte{0xc3, 0x8d, 0x00}}, + {2, [3]byte{0xc3, 0x8e, 0x00}}, {2, [3]byte{0xc3, 0x8f, 0x00}}, + {2, [3]byte{0xc3, 0x8c, 0x00}}, {1, [3]byte{0x60, 0x00, 0x00}}, + {1, [3]byte{0x3a, 0x00, 0x00}}, {1, [3]byte{0x23, 0x00, 0x00}}, + {1, [3]byte{0x40, 0x00, 0x00}}, {1, [3]byte{0x27, 0x00, 0x00}}, + {1, [3]byte{0x3d, 0x00, 0x00}}, {1, [3]byte{0x22, 0x00, 0x00}}, + {2, [3]byte{0xc3, 0x98, 0x00}}, {1, [3]byte{0x61, 0x00, 0x00}}, + {1, [3]byte{0x62, 0x00, 0x00}}, {1, [3]byte{0x63, 0x00, 0x00}}, + {1, [3]byte{0x64, 0x00, 0x00}}, {1, [3]byte{0x65, 0x00, 0x00}}, + {1, [3]byte{0x66, 0x00, 0x00}}, {1, [3]byte{0x67, 0x00, 0x00}}, + {1, [3]byte{0x68, 0x00, 0x00}}, {1, [3]byte{0x69, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0xab, 0x00}}, {2, [3]byte{0xc2, 0xbb, 0x00}}, + {2, [3]byte{0xc3, 0xb0, 0x00}}, {2, [3]byte{0xc3, 0xbd, 0x00}}, + {2, [3]byte{0xc3, 0xbe, 0x00}}, {2, [3]byte{0xc2, 0xb1, 0x00}}, + {2, [3]byte{0xc2, 0xb0, 0x00}}, {1, [3]byte{0x6a, 0x00, 0x00}}, + {1, [3]byte{0x6b, 0x00, 0x00}}, {1, [3]byte{0x6c, 0x00, 0x00}}, + {1, [3]byte{0x6d, 0x00, 0x00}}, {1, [3]byte{0x6e, 0x00, 0x00}}, + {1, [3]byte{0x6f, 0x00, 0x00}}, {1, [3]byte{0x70, 0x00, 0x00}}, + {1, [3]byte{0x71, 0x00, 0x00}}, {1, [3]byte{0x72, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0xaa, 0x00}}, {2, [3]byte{0xc2, 0xba, 0x00}}, + {2, [3]byte{0xc3, 0xa6, 0x00}}, {2, [3]byte{0xc2, 0xb8, 0x00}}, + {2, [3]byte{0xc3, 0x86, 0x00}}, {3, [3]byte{0xe2, 0x82, 0xac}}, + {2, [3]byte{0xc2, 0xb5, 0x00}}, {1, [3]byte{0x7e, 0x00, 0x00}}, + {1, [3]byte{0x73, 0x00, 0x00}}, {1, [3]byte{0x74, 0x00, 0x00}}, + {1, [3]byte{0x75, 0x00, 0x00}}, {1, [3]byte{0x76, 0x00, 0x00}}, + {1, [3]byte{0x77, 0x00, 0x00}}, {1, [3]byte{0x78, 0x00, 0x00}}, + {1, [3]byte{0x79, 0x00, 0x00}}, {1, [3]byte{0x7a, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0xa1, 0x00}}, {2, [3]byte{0xc2, 0xbf, 0x00}}, + {2, [3]byte{0xc3, 0x90, 0x00}}, {2, [3]byte{0xc3, 0x9d, 0x00}}, + {2, [3]byte{0xc3, 0x9e, 0x00}}, {2, [3]byte{0xc2, 0xae, 0x00}}, + {1, [3]byte{0x5e, 0x00, 0x00}}, {2, [3]byte{0xc2, 0xa3, 0x00}}, + {2, [3]byte{0xc2, 0xa5, 0x00}}, {2, [3]byte{0xc2, 0xb7, 0x00}}, + {2, [3]byte{0xc2, 0xa9, 0x00}}, {2, [3]byte{0xc2, 0xa7, 0x00}}, + {2, [3]byte{0xc2, 0xb6, 0x00}}, {2, [3]byte{0xc2, 0xbc, 0x00}}, + {2, [3]byte{0xc2, 0xbd, 0x00}}, {2, [3]byte{0xc2, 0xbe, 0x00}}, + {1, [3]byte{0x5b, 0x00, 0x00}}, {1, [3]byte{0x5d, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0xaf, 0x00}}, {2, [3]byte{0xc2, 0xa8, 0x00}}, + {2, [3]byte{0xc2, 0xb4, 0x00}}, {2, [3]byte{0xc3, 0x97, 0x00}}, + {1, [3]byte{0x7b, 0x00, 0x00}}, {1, [3]byte{0x41, 0x00, 0x00}}, + {1, [3]byte{0x42, 0x00, 0x00}}, {1, [3]byte{0x43, 0x00, 0x00}}, + {1, [3]byte{0x44, 0x00, 0x00}}, {1, [3]byte{0x45, 0x00, 0x00}}, + {1, [3]byte{0x46, 0x00, 0x00}}, {1, [3]byte{0x47, 0x00, 0x00}}, + {1, [3]byte{0x48, 0x00, 0x00}}, {1, [3]byte{0x49, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0xad, 0x00}}, {2, [3]byte{0xc3, 0xb4, 0x00}}, + {2, [3]byte{0xc3, 0xb6, 0x00}}, {2, [3]byte{0xc3, 0xb2, 0x00}}, + {2, [3]byte{0xc3, 0xb3, 0x00}}, {2, [3]byte{0xc3, 0xb5, 0x00}}, + {1, [3]byte{0x7d, 0x00, 0x00}}, {1, [3]byte{0x4a, 0x00, 0x00}}, + {1, [3]byte{0x4b, 0x00, 0x00}}, {1, [3]byte{0x4c, 0x00, 0x00}}, + {1, [3]byte{0x4d, 0x00, 0x00}}, {1, [3]byte{0x4e, 0x00, 0x00}}, + {1, [3]byte{0x4f, 0x00, 0x00}}, {1, [3]byte{0x50, 0x00, 0x00}}, + {1, [3]byte{0x51, 0x00, 0x00}}, {1, [3]byte{0x52, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0xb9, 0x00}}, {2, [3]byte{0xc3, 0xbb, 0x00}}, + {2, [3]byte{0xc3, 0xbc, 0x00}}, {2, [3]byte{0xc3, 0xb9, 0x00}}, + {2, [3]byte{0xc3, 0xba, 0x00}}, {2, [3]byte{0xc3, 0xbf, 0x00}}, + {1, [3]byte{0x5c, 0x00, 0x00}}, {2, [3]byte{0xc3, 0xb7, 0x00}}, + {1, [3]byte{0x53, 0x00, 0x00}}, {1, [3]byte{0x54, 0x00, 0x00}}, + {1, [3]byte{0x55, 0x00, 0x00}}, {1, [3]byte{0x56, 0x00, 0x00}}, + {1, [3]byte{0x57, 0x00, 0x00}}, {1, [3]byte{0x58, 0x00, 0x00}}, + {1, [3]byte{0x59, 0x00, 0x00}}, {1, [3]byte{0x5a, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0xb2, 0x00}}, {2, [3]byte{0xc3, 0x94, 0x00}}, + {2, [3]byte{0xc3, 0x96, 0x00}}, {2, [3]byte{0xc3, 0x92, 0x00}}, + {2, [3]byte{0xc3, 0x93, 0x00}}, {2, [3]byte{0xc3, 0x95, 0x00}}, + {1, [3]byte{0x30, 0x00, 0x00}}, {1, [3]byte{0x31, 0x00, 0x00}}, + {1, [3]byte{0x32, 0x00, 0x00}}, {1, [3]byte{0x33, 0x00, 0x00}}, + {1, [3]byte{0x34, 0x00, 0x00}}, {1, [3]byte{0x35, 0x00, 0x00}}, + {1, [3]byte{0x36, 0x00, 0x00}}, {1, [3]byte{0x37, 0x00, 0x00}}, + {1, [3]byte{0x38, 0x00, 0x00}}, {1, [3]byte{0x39, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0xb3, 0x00}}, {2, [3]byte{0xc3, 0x9b, 0x00}}, + {2, [3]byte{0xc3, 0x9c, 0x00}}, {2, [3]byte{0xc3, 0x99, 0x00}}, + {2, [3]byte{0xc3, 0x9a, 0x00}}, {2, [3]byte{0xc2, 0x9f, 0x00}}, + }, + encode: [256]uint32{ + 0x00000000, 0x01000001, 0x02000002, 0x03000003, 0x37000004, 0x2d000005, 0x2e000006, 0x2f000007, + 0x16000008, 0x05000009, 0x2500000a, 0x0b00000b, 0x0c00000c, 0x0d00000d, 0x0e00000e, 0x0f00000f, + 0x10000010, 0x11000011, 0x12000012, 0x13000013, 0x3c000014, 0x3d000015, 0x32000016, 0x26000017, + 0x18000018, 0x19000019, 0x3f00001a, 0x2700001b, 0x1c00001c, 0x1d00001d, 0x1e00001e, 0x1f00001f, + 0x40000020, 0x5a000021, 0x7f000022, 0x7b000023, 0x5b000024, 0x6c000025, 0x50000026, 0x7d000027, + 0x4d000028, 0x5d000029, 0x5c00002a, 0x4e00002b, 0x6b00002c, 0x6000002d, 0x4b00002e, 0x6100002f, + 0xf0000030, 0xf1000031, 0xf2000032, 0xf3000033, 0xf4000034, 0xf5000035, 0xf6000036, 0xf7000037, + 0xf8000038, 0xf9000039, 0x7a00003a, 0x5e00003b, 0x4c00003c, 0x7e00003d, 0x6e00003e, 0x6f00003f, + 0x7c000040, 0xc1000041, 0xc2000042, 0xc3000043, 0xc4000044, 0xc5000045, 0xc6000046, 0xc7000047, + 0xc8000048, 0xc9000049, 0xd100004a, 0xd200004b, 0xd300004c, 0xd400004d, 0xd500004e, 0xd600004f, + 0xd7000050, 0xd8000051, 0xd9000052, 0xe2000053, 0xe3000054, 0xe4000055, 0xe5000056, 0xe6000057, + 0xe7000058, 0xe8000059, 0xe900005a, 0xba00005b, 0xe000005c, 0xbb00005d, 0xb000005e, 0x6d00005f, + 0x79000060, 0x81000061, 0x82000062, 0x83000063, 0x84000064, 0x85000065, 0x86000066, 0x87000067, + 0x88000068, 0x89000069, 0x9100006a, 0x9200006b, 0x9300006c, 0x9400006d, 0x9500006e, 0x9600006f, + 0x97000070, 0x98000071, 0x99000072, 0xa2000073, 0xa3000074, 0xa4000075, 0xa5000076, 0xa6000077, + 0xa7000078, 0xa8000079, 0xa900007a, 0xc000007b, 0x4f00007c, 0xd000007d, 0xa100007e, 0x0700007f, + 0x20000080, 0x21000081, 0x22000082, 0x23000083, 0x24000084, 0x15000085, 0x06000086, 0x17000087, + 0x28000088, 0x29000089, 0x2a00008a, 0x2b00008b, 0x2c00008c, 0x0900008d, 0x0a00008e, 0x1b00008f, + 0x30000090, 0x31000091, 0x1a000092, 0x33000093, 0x34000094, 0x35000095, 0x36000096, 0x08000097, + 0x38000098, 0x39000099, 0x3a00009a, 0x3b00009b, 0x0400009c, 0x1400009d, 0x3e00009e, 0xff00009f, + 0x410000a0, 0xaa0000a1, 0x4a0000a2, 0xb10000a3, 0xb20000a5, 0x6a0000a6, 0xb50000a7, 0xbd0000a8, + 0xb40000a9, 0x9a0000aa, 0x8a0000ab, 0x5f0000ac, 0xca0000ad, 0xaf0000ae, 0xbc0000af, 0x900000b0, + 0x8f0000b1, 0xea0000b2, 0xfa0000b3, 0xbe0000b4, 0xa00000b5, 0xb60000b6, 0xb30000b7, 0x9d0000b8, + 0xda0000b9, 0x9b0000ba, 0x8b0000bb, 0xb70000bc, 0xb80000bd, 0xb90000be, 0xab0000bf, 0x640000c0, + 0x650000c1, 0x620000c2, 0x660000c3, 0x630000c4, 0x670000c5, 0x9e0000c6, 0x680000c7, 0x740000c8, + 0x710000c9, 0x720000ca, 0x730000cb, 0x780000cc, 0x750000cd, 0x760000ce, 0x770000cf, 0xac0000d0, + 0x690000d1, 0xed0000d2, 0xee0000d3, 0xeb0000d4, 0xef0000d5, 0xec0000d6, 0xbf0000d7, 0x800000d8, + 0xfd0000d9, 0xfe0000da, 0xfb0000db, 0xfc0000dc, 0xad0000dd, 0xae0000de, 0x590000df, 0x440000e0, + 0x450000e1, 0x420000e2, 0x460000e3, 0x430000e4, 0x470000e5, 0x9c0000e6, 0x480000e7, 0x540000e8, + 0x510000e9, 0x520000ea, 0x530000eb, 0x580000ec, 0x550000ed, 0x560000ee, 0x570000ef, 0x8c0000f0, + 0x490000f1, 0xcd0000f2, 0xce0000f3, 0xcb0000f4, 0xcf0000f5, 0xcc0000f6, 0xe10000f7, 0x700000f8, + 0xdd0000f9, 0xde0000fa, 0xdb0000fb, 0xdc0000fc, 0x8d0000fd, 0x8e0000fe, 0xdf0000ff, 0x9f0020ac, + }, +} + +// ISO8859_1 is the ISO 8859-1 encoding. +var ISO8859_1 *Charmap = &iso8859_1 + +var iso8859_1 = Charmap{ name: "ISO 8859-1", mib: identifier.ISOLatin1, asciiSuperset: true, @@ -1933,9 +2458,9 @@ var iso8859_1 = charmap{ } // ISO8859_2 is the ISO 8859-2 encoding. -var ISO8859_2 encoding.Encoding = &iso8859_2 +var ISO8859_2 *Charmap = &iso8859_2 -var iso8859_2 = charmap{ +var iso8859_2 = Charmap{ name: "ISO 8859-2", mib: identifier.ISOLatin2, asciiSuperset: true, @@ -2108,9 +2633,9 @@ var iso8859_2 = charmap{ } // ISO8859_3 is the ISO 8859-3 encoding. -var ISO8859_3 encoding.Encoding = &iso8859_3 +var ISO8859_3 *Charmap = &iso8859_3 -var iso8859_3 = charmap{ +var iso8859_3 = Charmap{ name: "ISO 8859-3", mib: identifier.ISOLatin3, asciiSuperset: true, @@ -2283,9 +2808,9 @@ var iso8859_3 = charmap{ } // ISO8859_4 is the ISO 8859-4 encoding. -var ISO8859_4 encoding.Encoding = &iso8859_4 +var ISO8859_4 *Charmap = &iso8859_4 -var iso8859_4 = charmap{ +var iso8859_4 = Charmap{ name: "ISO 8859-4", mib: identifier.ISOLatin4, asciiSuperset: true, @@ -2458,9 +2983,9 @@ var iso8859_4 = charmap{ } // ISO8859_5 is the ISO 8859-5 encoding. -var ISO8859_5 encoding.Encoding = &iso8859_5 +var ISO8859_5 *Charmap = &iso8859_5 -var iso8859_5 = charmap{ +var iso8859_5 = Charmap{ name: "ISO 8859-5", mib: identifier.ISOLatinCyrillic, asciiSuperset: true, @@ -2633,9 +3158,9 @@ var iso8859_5 = charmap{ } // ISO8859_6 is the ISO 8859-6 encoding. -var ISO8859_6 encoding.Encoding = &iso8859_6 +var ISO8859_6 *Charmap = &iso8859_6 -var iso8859_6 = charmap{ +var iso8859_6 = Charmap{ name: "ISO 8859-6", mib: identifier.ISOLatinArabic, asciiSuperset: true, @@ -2808,9 +3333,9 @@ var iso8859_6 = charmap{ } // ISO8859_7 is the ISO 8859-7 encoding. -var ISO8859_7 encoding.Encoding = &iso8859_7 +var ISO8859_7 *Charmap = &iso8859_7 -var iso8859_7 = charmap{ +var iso8859_7 = Charmap{ name: "ISO 8859-7", mib: identifier.ISOLatinGreek, asciiSuperset: true, @@ -2983,9 +3508,9 @@ var iso8859_7 = charmap{ } // ISO8859_8 is the ISO 8859-8 encoding. -var ISO8859_8 encoding.Encoding = &iso8859_8 +var ISO8859_8 *Charmap = &iso8859_8 -var iso8859_8 = charmap{ +var iso8859_8 = Charmap{ name: "ISO 8859-8", mib: identifier.ISOLatinHebrew, asciiSuperset: true, @@ -3157,10 +3682,185 @@ var iso8859_8 = charmap{ }, } -// ISO8859_10 is the ISO 8859-10 encoding. -var ISO8859_10 encoding.Encoding = &iso8859_10 +// ISO8859_9 is the ISO 8859-9 encoding. +var ISO8859_9 *Charmap = &iso8859_9 -var iso8859_10 = charmap{ +var iso8859_9 = Charmap{ + name: "ISO 8859-9", + mib: identifier.ISOLatin5, + asciiSuperset: true, + low: 0x80, + replacement: 0x1a, + decode: [256]utf8Enc{ + {1, [3]byte{0x00, 0x00, 0x00}}, {1, [3]byte{0x01, 0x00, 0x00}}, + {1, [3]byte{0x02, 0x00, 0x00}}, {1, [3]byte{0x03, 0x00, 0x00}}, + {1, [3]byte{0x04, 0x00, 0x00}}, {1, [3]byte{0x05, 0x00, 0x00}}, + {1, [3]byte{0x06, 0x00, 0x00}}, {1, [3]byte{0x07, 0x00, 0x00}}, + {1, [3]byte{0x08, 0x00, 0x00}}, {1, [3]byte{0x09, 0x00, 0x00}}, + {1, [3]byte{0x0a, 0x00, 0x00}}, {1, [3]byte{0x0b, 0x00, 0x00}}, + {1, [3]byte{0x0c, 0x00, 0x00}}, {1, [3]byte{0x0d, 0x00, 0x00}}, + {1, [3]byte{0x0e, 0x00, 0x00}}, {1, [3]byte{0x0f, 0x00, 0x00}}, + {1, [3]byte{0x10, 0x00, 0x00}}, {1, [3]byte{0x11, 0x00, 0x00}}, + {1, [3]byte{0x12, 0x00, 0x00}}, {1, [3]byte{0x13, 0x00, 0x00}}, + {1, [3]byte{0x14, 0x00, 0x00}}, {1, [3]byte{0x15, 0x00, 0x00}}, + {1, [3]byte{0x16, 0x00, 0x00}}, {1, [3]byte{0x17, 0x00, 0x00}}, + {1, [3]byte{0x18, 0x00, 0x00}}, {1, [3]byte{0x19, 0x00, 0x00}}, + {1, [3]byte{0x1a, 0x00, 0x00}}, {1, [3]byte{0x1b, 0x00, 0x00}}, + {1, [3]byte{0x1c, 0x00, 0x00}}, {1, [3]byte{0x1d, 0x00, 0x00}}, + {1, [3]byte{0x1e, 0x00, 0x00}}, {1, [3]byte{0x1f, 0x00, 0x00}}, + {1, [3]byte{0x20, 0x00, 0x00}}, {1, [3]byte{0x21, 0x00, 0x00}}, + {1, [3]byte{0x22, 0x00, 0x00}}, {1, [3]byte{0x23, 0x00, 0x00}}, + {1, [3]byte{0x24, 0x00, 0x00}}, {1, [3]byte{0x25, 0x00, 0x00}}, + {1, [3]byte{0x26, 0x00, 0x00}}, {1, [3]byte{0x27, 0x00, 0x00}}, + {1, [3]byte{0x28, 0x00, 0x00}}, {1, [3]byte{0x29, 0x00, 0x00}}, + {1, [3]byte{0x2a, 0x00, 0x00}}, {1, [3]byte{0x2b, 0x00, 0x00}}, + {1, [3]byte{0x2c, 0x00, 0x00}}, {1, [3]byte{0x2d, 0x00, 0x00}}, + {1, [3]byte{0x2e, 0x00, 0x00}}, {1, [3]byte{0x2f, 0x00, 0x00}}, + {1, [3]byte{0x30, 0x00, 0x00}}, {1, [3]byte{0x31, 0x00, 0x00}}, + {1, [3]byte{0x32, 0x00, 0x00}}, {1, [3]byte{0x33, 0x00, 0x00}}, + {1, [3]byte{0x34, 0x00, 0x00}}, {1, [3]byte{0x35, 0x00, 0x00}}, + {1, [3]byte{0x36, 0x00, 0x00}}, {1, [3]byte{0x37, 0x00, 0x00}}, + {1, [3]byte{0x38, 0x00, 0x00}}, {1, [3]byte{0x39, 0x00, 0x00}}, + {1, [3]byte{0x3a, 0x00, 0x00}}, {1, [3]byte{0x3b, 0x00, 0x00}}, + {1, [3]byte{0x3c, 0x00, 0x00}}, {1, [3]byte{0x3d, 0x00, 0x00}}, + {1, [3]byte{0x3e, 0x00, 0x00}}, {1, [3]byte{0x3f, 0x00, 0x00}}, + {1, [3]byte{0x40, 0x00, 0x00}}, {1, [3]byte{0x41, 0x00, 0x00}}, + {1, [3]byte{0x42, 0x00, 0x00}}, {1, [3]byte{0x43, 0x00, 0x00}}, + {1, [3]byte{0x44, 0x00, 0x00}}, {1, [3]byte{0x45, 0x00, 0x00}}, + {1, [3]byte{0x46, 0x00, 0x00}}, {1, [3]byte{0x47, 0x00, 0x00}}, + {1, [3]byte{0x48, 0x00, 0x00}}, {1, [3]byte{0x49, 0x00, 0x00}}, + {1, [3]byte{0x4a, 0x00, 0x00}}, {1, [3]byte{0x4b, 0x00, 0x00}}, + {1, [3]byte{0x4c, 0x00, 0x00}}, {1, [3]byte{0x4d, 0x00, 0x00}}, + {1, [3]byte{0x4e, 0x00, 0x00}}, {1, [3]byte{0x4f, 0x00, 0x00}}, + {1, [3]byte{0x50, 0x00, 0x00}}, {1, [3]byte{0x51, 0x00, 0x00}}, + {1, [3]byte{0x52, 0x00, 0x00}}, {1, [3]byte{0x53, 0x00, 0x00}}, + {1, [3]byte{0x54, 0x00, 0x00}}, {1, [3]byte{0x55, 0x00, 0x00}}, + {1, [3]byte{0x56, 0x00, 0x00}}, {1, [3]byte{0x57, 0x00, 0x00}}, + {1, [3]byte{0x58, 0x00, 0x00}}, {1, [3]byte{0x59, 0x00, 0x00}}, + {1, [3]byte{0x5a, 0x00, 0x00}}, {1, [3]byte{0x5b, 0x00, 0x00}}, + {1, [3]byte{0x5c, 0x00, 0x00}}, {1, [3]byte{0x5d, 0x00, 0x00}}, + {1, [3]byte{0x5e, 0x00, 0x00}}, {1, [3]byte{0x5f, 0x00, 0x00}}, + {1, [3]byte{0x60, 0x00, 0x00}}, {1, [3]byte{0x61, 0x00, 0x00}}, + {1, [3]byte{0x62, 0x00, 0x00}}, {1, [3]byte{0x63, 0x00, 0x00}}, + {1, [3]byte{0x64, 0x00, 0x00}}, {1, [3]byte{0x65, 0x00, 0x00}}, + {1, [3]byte{0x66, 0x00, 0x00}}, {1, [3]byte{0x67, 0x00, 0x00}}, + {1, [3]byte{0x68, 0x00, 0x00}}, {1, [3]byte{0x69, 0x00, 0x00}}, + {1, [3]byte{0x6a, 0x00, 0x00}}, {1, [3]byte{0x6b, 0x00, 0x00}}, + {1, [3]byte{0x6c, 0x00, 0x00}}, {1, [3]byte{0x6d, 0x00, 0x00}}, + {1, [3]byte{0x6e, 0x00, 0x00}}, {1, [3]byte{0x6f, 0x00, 0x00}}, + {1, [3]byte{0x70, 0x00, 0x00}}, {1, [3]byte{0x71, 0x00, 0x00}}, + {1, [3]byte{0x72, 0x00, 0x00}}, {1, [3]byte{0x73, 0x00, 0x00}}, + {1, [3]byte{0x74, 0x00, 0x00}}, {1, [3]byte{0x75, 0x00, 0x00}}, + {1, [3]byte{0x76, 0x00, 0x00}}, {1, [3]byte{0x77, 0x00, 0x00}}, + {1, [3]byte{0x78, 0x00, 0x00}}, {1, [3]byte{0x79, 0x00, 0x00}}, + {1, [3]byte{0x7a, 0x00, 0x00}}, {1, [3]byte{0x7b, 0x00, 0x00}}, + {1, [3]byte{0x7c, 0x00, 0x00}}, {1, [3]byte{0x7d, 0x00, 0x00}}, + {1, [3]byte{0x7e, 0x00, 0x00}}, {1, [3]byte{0x7f, 0x00, 0x00}}, + {2, [3]byte{0xc2, 0x80, 0x00}}, {2, [3]byte{0xc2, 0x81, 0x00}}, + {2, [3]byte{0xc2, 0x82, 0x00}}, {2, [3]byte{0xc2, 0x83, 0x00}}, + {2, [3]byte{0xc2, 0x84, 0x00}}, {2, [3]byte{0xc2, 0x85, 0x00}}, + {2, [3]byte{0xc2, 0x86, 0x00}}, {2, [3]byte{0xc2, 0x87, 0x00}}, + {2, [3]byte{0xc2, 0x88, 0x00}}, {2, [3]byte{0xc2, 0x89, 0x00}}, + {2, [3]byte{0xc2, 0x8a, 0x00}}, {2, [3]byte{0xc2, 0x8b, 0x00}}, + {2, [3]byte{0xc2, 0x8c, 0x00}}, {2, [3]byte{0xc2, 0x8d, 0x00}}, + {2, [3]byte{0xc2, 0x8e, 0x00}}, {2, [3]byte{0xc2, 0x8f, 0x00}}, + {2, [3]byte{0xc2, 0x90, 0x00}}, {2, [3]byte{0xc2, 0x91, 0x00}}, + {2, [3]byte{0xc2, 0x92, 0x00}}, {2, [3]byte{0xc2, 0x93, 0x00}}, + {2, [3]byte{0xc2, 0x94, 0x00}}, {2, [3]byte{0xc2, 0x95, 0x00}}, + {2, [3]byte{0xc2, 0x96, 0x00}}, {2, [3]byte{0xc2, 0x97, 0x00}}, + {2, [3]byte{0xc2, 0x98, 0x00}}, {2, [3]byte{0xc2, 0x99, 0x00}}, + {2, [3]byte{0xc2, 0x9a, 0x00}}, {2, [3]byte{0xc2, 0x9b, 0x00}}, + {2, [3]byte{0xc2, 0x9c, 0x00}}, {2, [3]byte{0xc2, 0x9d, 0x00}}, + {2, [3]byte{0xc2, 0x9e, 0x00}}, {2, [3]byte{0xc2, 0x9f, 0x00}}, + {2, [3]byte{0xc2, 0xa0, 0x00}}, {2, [3]byte{0xc2, 0xa1, 0x00}}, + {2, [3]byte{0xc2, 0xa2, 0x00}}, {2, [3]byte{0xc2, 0xa3, 0x00}}, + {2, [3]byte{0xc2, 0xa4, 0x00}}, {2, [3]byte{0xc2, 0xa5, 0x00}}, + {2, [3]byte{0xc2, 0xa6, 0x00}}, {2, [3]byte{0xc2, 0xa7, 0x00}}, + {2, [3]byte{0xc2, 0xa8, 0x00}}, {2, [3]byte{0xc2, 0xa9, 0x00}}, + {2, [3]byte{0xc2, 0xaa, 0x00}}, {2, [3]byte{0xc2, 0xab, 0x00}}, + {2, [3]byte{0xc2, 0xac, 0x00}}, {2, [3]byte{0xc2, 0xad, 0x00}}, + {2, [3]byte{0xc2, 0xae, 0x00}}, {2, [3]byte{0xc2, 0xaf, 0x00}}, + {2, [3]byte{0xc2, 0xb0, 0x00}}, {2, [3]byte{0xc2, 0xb1, 0x00}}, + {2, [3]byte{0xc2, 0xb2, 0x00}}, {2, [3]byte{0xc2, 0xb3, 0x00}}, + {2, [3]byte{0xc2, 0xb4, 0x00}}, {2, [3]byte{0xc2, 0xb5, 0x00}}, + {2, [3]byte{0xc2, 0xb6, 0x00}}, {2, [3]byte{0xc2, 0xb7, 0x00}}, + {2, [3]byte{0xc2, 0xb8, 0x00}}, {2, [3]byte{0xc2, 0xb9, 0x00}}, + {2, [3]byte{0xc2, 0xba, 0x00}}, {2, [3]byte{0xc2, 0xbb, 0x00}}, + {2, [3]byte{0xc2, 0xbc, 0x00}}, {2, [3]byte{0xc2, 0xbd, 0x00}}, + {2, [3]byte{0xc2, 0xbe, 0x00}}, {2, [3]byte{0xc2, 0xbf, 0x00}}, + {2, [3]byte{0xc3, 0x80, 0x00}}, {2, [3]byte{0xc3, 0x81, 0x00}}, + {2, [3]byte{0xc3, 0x82, 0x00}}, {2, [3]byte{0xc3, 0x83, 0x00}}, + {2, [3]byte{0xc3, 0x84, 0x00}}, {2, [3]byte{0xc3, 0x85, 0x00}}, + {2, [3]byte{0xc3, 0x86, 0x00}}, {2, [3]byte{0xc3, 0x87, 0x00}}, + {2, [3]byte{0xc3, 0x88, 0x00}}, {2, [3]byte{0xc3, 0x89, 0x00}}, + {2, [3]byte{0xc3, 0x8a, 0x00}}, {2, [3]byte{0xc3, 0x8b, 0x00}}, + {2, [3]byte{0xc3, 0x8c, 0x00}}, {2, [3]byte{0xc3, 0x8d, 0x00}}, + {2, [3]byte{0xc3, 0x8e, 0x00}}, {2, [3]byte{0xc3, 0x8f, 0x00}}, + {2, [3]byte{0xc4, 0x9e, 0x00}}, {2, [3]byte{0xc3, 0x91, 0x00}}, + {2, [3]byte{0xc3, 0x92, 0x00}}, {2, [3]byte{0xc3, 0x93, 0x00}}, + {2, [3]byte{0xc3, 0x94, 0x00}}, {2, [3]byte{0xc3, 0x95, 0x00}}, + {2, [3]byte{0xc3, 0x96, 0x00}}, {2, [3]byte{0xc3, 0x97, 0x00}}, + {2, [3]byte{0xc3, 0x98, 0x00}}, {2, [3]byte{0xc3, 0x99, 0x00}}, + {2, [3]byte{0xc3, 0x9a, 0x00}}, {2, [3]byte{0xc3, 0x9b, 0x00}}, + {2, [3]byte{0xc3, 0x9c, 0x00}}, {2, [3]byte{0xc4, 0xb0, 0x00}}, + {2, [3]byte{0xc5, 0x9e, 0x00}}, {2, [3]byte{0xc3, 0x9f, 0x00}}, + {2, [3]byte{0xc3, 0xa0, 0x00}}, {2, [3]byte{0xc3, 0xa1, 0x00}}, + {2, [3]byte{0xc3, 0xa2, 0x00}}, {2, [3]byte{0xc3, 0xa3, 0x00}}, + {2, [3]byte{0xc3, 0xa4, 0x00}}, {2, [3]byte{0xc3, 0xa5, 0x00}}, + {2, [3]byte{0xc3, 0xa6, 0x00}}, {2, [3]byte{0xc3, 0xa7, 0x00}}, + {2, [3]byte{0xc3, 0xa8, 0x00}}, {2, [3]byte{0xc3, 0xa9, 0x00}}, + {2, [3]byte{0xc3, 0xaa, 0x00}}, {2, [3]byte{0xc3, 0xab, 0x00}}, + {2, [3]byte{0xc3, 0xac, 0x00}}, {2, [3]byte{0xc3, 0xad, 0x00}}, + {2, [3]byte{0xc3, 0xae, 0x00}}, {2, [3]byte{0xc3, 0xaf, 0x00}}, + {2, [3]byte{0xc4, 0x9f, 0x00}}, {2, [3]byte{0xc3, 0xb1, 0x00}}, + {2, [3]byte{0xc3, 0xb2, 0x00}}, {2, [3]byte{0xc3, 0xb3, 0x00}}, + {2, [3]byte{0xc3, 0xb4, 0x00}}, {2, [3]byte{0xc3, 0xb5, 0x00}}, + {2, [3]byte{0xc3, 0xb6, 0x00}}, {2, [3]byte{0xc3, 0xb7, 0x00}}, + {2, [3]byte{0xc3, 0xb8, 0x00}}, {2, [3]byte{0xc3, 0xb9, 0x00}}, + {2, [3]byte{0xc3, 0xba, 0x00}}, {2, [3]byte{0xc3, 0xbb, 0x00}}, + {2, [3]byte{0xc3, 0xbc, 0x00}}, {2, [3]byte{0xc4, 0xb1, 0x00}}, + {2, [3]byte{0xc5, 0x9f, 0x00}}, {2, [3]byte{0xc3, 0xbf, 0x00}}, + }, + encode: [256]uint32{ + 0x00000000, 0x01000001, 0x02000002, 0x03000003, 0x04000004, 0x05000005, 0x06000006, 0x07000007, + 0x08000008, 0x09000009, 0x0a00000a, 0x0b00000b, 0x0c00000c, 0x0d00000d, 0x0e00000e, 0x0f00000f, + 0x10000010, 0x11000011, 0x12000012, 0x13000013, 0x14000014, 0x15000015, 0x16000016, 0x17000017, + 0x18000018, 0x19000019, 0x1a00001a, 0x1b00001b, 0x1c00001c, 0x1d00001d, 0x1e00001e, 0x1f00001f, + 0x20000020, 0x21000021, 0x22000022, 0x23000023, 0x24000024, 0x25000025, 0x26000026, 0x27000027, + 0x28000028, 0x29000029, 0x2a00002a, 0x2b00002b, 0x2c00002c, 0x2d00002d, 0x2e00002e, 0x2f00002f, + 0x30000030, 0x31000031, 0x32000032, 0x33000033, 0x34000034, 0x35000035, 0x36000036, 0x37000037, + 0x38000038, 0x39000039, 0x3a00003a, 0x3b00003b, 0x3c00003c, 0x3d00003d, 0x3e00003e, 0x3f00003f, + 0x40000040, 0x41000041, 0x42000042, 0x43000043, 0x44000044, 0x45000045, 0x46000046, 0x47000047, + 0x48000048, 0x49000049, 0x4a00004a, 0x4b00004b, 0x4c00004c, 0x4d00004d, 0x4e00004e, 0x4f00004f, + 0x50000050, 0x51000051, 0x52000052, 0x53000053, 0x54000054, 0x55000055, 0x56000056, 0x57000057, + 0x58000058, 0x59000059, 0x5a00005a, 0x5b00005b, 0x5c00005c, 0x5d00005d, 0x5e00005e, 0x5f00005f, + 0x60000060, 0x61000061, 0x62000062, 0x63000063, 0x64000064, 0x65000065, 0x66000066, 0x67000067, + 0x68000068, 0x69000069, 0x6a00006a, 0x6b00006b, 0x6c00006c, 0x6d00006d, 0x6e00006e, 0x6f00006f, + 0x70000070, 0x71000071, 0x72000072, 0x73000073, 0x74000074, 0x75000075, 0x76000076, 0x77000077, + 0x78000078, 0x79000079, 0x7a00007a, 0x7b00007b, 0x7c00007c, 0x7d00007d, 0x7e00007e, 0x7f00007f, + 0x80000080, 0x81000081, 0x82000082, 0x83000083, 0x84000084, 0x85000085, 0x86000086, 0x87000087, + 0x88000088, 0x89000089, 0x8a00008a, 0x8b00008b, 0x8c00008c, 0x8d00008d, 0x8e00008e, 0x8f00008f, + 0x90000090, 0x91000091, 0x92000092, 0x93000093, 0x94000094, 0x95000095, 0x96000096, 0x97000097, + 0x98000098, 0x99000099, 0x9a00009a, 0x9b00009b, 0x9c00009c, 0x9d00009d, 0x9e00009e, 0x9f00009f, + 0xa00000a0, 0xa10000a1, 0xa20000a2, 0xa30000a3, 0xa40000a4, 0xa50000a5, 0xa60000a6, 0xa70000a7, + 0xa80000a8, 0xa90000a9, 0xaa0000aa, 0xab0000ab, 0xac0000ac, 0xad0000ad, 0xae0000ae, 0xaf0000af, + 0xb00000b0, 0xb10000b1, 0xb20000b2, 0xb30000b3, 0xb40000b4, 0xb50000b5, 0xb60000b6, 0xb70000b7, + 0xb80000b8, 0xb90000b9, 0xba0000ba, 0xbb0000bb, 0xbc0000bc, 0xbd0000bd, 0xbe0000be, 0xbf0000bf, + 0xc00000c0, 0xc10000c1, 0xc20000c2, 0xc30000c3, 0xc40000c4, 0xc50000c5, 0xc60000c6, 0xc70000c7, + 0xc80000c8, 0xc90000c9, 0xca0000ca, 0xcb0000cb, 0xcc0000cc, 0xcd0000cd, 0xce0000ce, 0xcf0000cf, + 0xd10000d1, 0xd20000d2, 0xd30000d3, 0xd40000d4, 0xd50000d5, 0xd60000d6, 0xd70000d7, 0xd80000d8, + 0xd90000d9, 0xda0000da, 0xdb0000db, 0xdc0000dc, 0xdf0000df, 0xe00000e0, 0xe10000e1, 0xe20000e2, + 0xe30000e3, 0xe40000e4, 0xe50000e5, 0xe60000e6, 0xe70000e7, 0xe80000e8, 0xe90000e9, 0xea0000ea, + 0xeb0000eb, 0xec0000ec, 0xed0000ed, 0xee0000ee, 0xef0000ef, 0xf10000f1, 0xf20000f2, 0xf30000f3, + 0xf40000f4, 0xf50000f5, 0xf60000f6, 0xf70000f7, 0xf80000f8, 0xf90000f9, 0xfa0000fa, 0xfb0000fb, + 0xfc0000fc, 0xff0000ff, 0xd000011e, 0xf000011f, 0xdd000130, 0xfd000131, 0xde00015e, 0xfe00015f, + }, +} + +// ISO8859_10 is the ISO 8859-10 encoding. +var ISO8859_10 *Charmap = &iso8859_10 + +var iso8859_10 = Charmap{ name: "ISO 8859-10", mib: identifier.ISOLatin6, asciiSuperset: true, @@ -3333,9 +4033,9 @@ var iso8859_10 = charmap{ } // ISO8859_13 is the ISO 8859-13 encoding. -var ISO8859_13 encoding.Encoding = &iso8859_13 +var ISO8859_13 *Charmap = &iso8859_13 -var iso8859_13 = charmap{ +var iso8859_13 = Charmap{ name: "ISO 8859-13", mib: identifier.ISO885913, asciiSuperset: true, @@ -3508,9 +4208,9 @@ var iso8859_13 = charmap{ } // ISO8859_14 is the ISO 8859-14 encoding. -var ISO8859_14 encoding.Encoding = &iso8859_14 +var ISO8859_14 *Charmap = &iso8859_14 -var iso8859_14 = charmap{ +var iso8859_14 = Charmap{ name: "ISO 8859-14", mib: identifier.ISO885914, asciiSuperset: true, @@ -3683,9 +4383,9 @@ var iso8859_14 = charmap{ } // ISO8859_15 is the ISO 8859-15 encoding. -var ISO8859_15 encoding.Encoding = &iso8859_15 +var ISO8859_15 *Charmap = &iso8859_15 -var iso8859_15 = charmap{ +var iso8859_15 = Charmap{ name: "ISO 8859-15", mib: identifier.ISO885915, asciiSuperset: true, @@ -3858,9 +4558,9 @@ var iso8859_15 = charmap{ } // ISO8859_16 is the ISO 8859-16 encoding. -var ISO8859_16 encoding.Encoding = &iso8859_16 +var ISO8859_16 *Charmap = &iso8859_16 -var iso8859_16 = charmap{ +var iso8859_16 = Charmap{ name: "ISO 8859-16", mib: identifier.ISO885916, asciiSuperset: true, @@ -4033,9 +4733,9 @@ var iso8859_16 = charmap{ } // KOI8R is the KOI8-R encoding. -var KOI8R encoding.Encoding = &koi8R +var KOI8R *Charmap = &koi8R -var koi8R = charmap{ +var koi8R = Charmap{ name: "KOI8-R", mib: identifier.KOI8R, asciiSuperset: true, @@ -4208,9 +4908,9 @@ var koi8R = charmap{ } // KOI8U is the KOI8-U encoding. -var KOI8U encoding.Encoding = &koi8U +var KOI8U *Charmap = &koi8U -var koi8U = charmap{ +var koi8U = Charmap{ name: "KOI8-U", mib: identifier.KOI8U, asciiSuperset: true, @@ -4383,9 +5083,9 @@ var koi8U = charmap{ } // Macintosh is the Macintosh encoding. -var Macintosh encoding.Encoding = &macintosh +var Macintosh *Charmap = &macintosh -var macintosh = charmap{ +var macintosh = Charmap{ name: "Macintosh", mib: identifier.Macintosh, asciiSuperset: true, @@ -4558,9 +5258,9 @@ var macintosh = charmap{ } // MacintoshCyrillic is the Macintosh Cyrillic encoding. -var MacintoshCyrillic encoding.Encoding = &macintoshCyrillic +var MacintoshCyrillic *Charmap = &macintoshCyrillic -var macintoshCyrillic = charmap{ +var macintoshCyrillic = Charmap{ name: "Macintosh Cyrillic", mib: identifier.MacintoshCyrillic, asciiSuperset: true, @@ -4733,9 +5433,9 @@ var macintoshCyrillic = charmap{ } // Windows874 is the Windows 874 encoding. -var Windows874 encoding.Encoding = &windows874 +var Windows874 *Charmap = &windows874 -var windows874 = charmap{ +var windows874 = Charmap{ name: "Windows 874", mib: identifier.Windows874, asciiSuperset: true, @@ -4908,9 +5608,9 @@ var windows874 = charmap{ } // Windows1250 is the Windows 1250 encoding. -var Windows1250 encoding.Encoding = &windows1250 +var Windows1250 *Charmap = &windows1250 -var windows1250 = charmap{ +var windows1250 = Charmap{ name: "Windows 1250", mib: identifier.Windows1250, asciiSuperset: true, @@ -5083,9 +5783,9 @@ var windows1250 = charmap{ } // Windows1251 is the Windows 1251 encoding. -var Windows1251 encoding.Encoding = &windows1251 +var Windows1251 *Charmap = &windows1251 -var windows1251 = charmap{ +var windows1251 = Charmap{ name: "Windows 1251", mib: identifier.Windows1251, asciiSuperset: true, @@ -5258,9 +5958,9 @@ var windows1251 = charmap{ } // Windows1252 is the Windows 1252 encoding. -var Windows1252 encoding.Encoding = &windows1252 +var Windows1252 *Charmap = &windows1252 -var windows1252 = charmap{ +var windows1252 = Charmap{ name: "Windows 1252", mib: identifier.Windows1252, asciiSuperset: true, @@ -5433,9 +6133,9 @@ var windows1252 = charmap{ } // Windows1253 is the Windows 1253 encoding. -var Windows1253 encoding.Encoding = &windows1253 +var Windows1253 *Charmap = &windows1253 -var windows1253 = charmap{ +var windows1253 = Charmap{ name: "Windows 1253", mib: identifier.Windows1253, asciiSuperset: true, @@ -5608,9 +6308,9 @@ var windows1253 = charmap{ } // Windows1254 is the Windows 1254 encoding. -var Windows1254 encoding.Encoding = &windows1254 +var Windows1254 *Charmap = &windows1254 -var windows1254 = charmap{ +var windows1254 = Charmap{ name: "Windows 1254", mib: identifier.Windows1254, asciiSuperset: true, @@ -5783,9 +6483,9 @@ var windows1254 = charmap{ } // Windows1255 is the Windows 1255 encoding. -var Windows1255 encoding.Encoding = &windows1255 +var Windows1255 *Charmap = &windows1255 -var windows1255 = charmap{ +var windows1255 = Charmap{ name: "Windows 1255", mib: identifier.Windows1255, asciiSuperset: true, @@ -5893,7 +6593,7 @@ var windows1255 = charmap{ {2, [3]byte{0xd6, 0xb4, 0x00}}, {2, [3]byte{0xd6, 0xb5, 0x00}}, {2, [3]byte{0xd6, 0xb6, 0x00}}, {2, [3]byte{0xd6, 0xb7, 0x00}}, {2, [3]byte{0xd6, 0xb8, 0x00}}, {2, [3]byte{0xd6, 0xb9, 0x00}}, - {3, [3]byte{0xef, 0xbf, 0xbd}}, {2, [3]byte{0xd6, 0xbb, 0x00}}, + {2, [3]byte{0xd6, 0xba, 0x00}}, {2, [3]byte{0xd6, 0xbb, 0x00}}, {2, [3]byte{0xd6, 0xbc, 0x00}}, {2, [3]byte{0xd6, 0xbd, 0x00}}, {2, [3]byte{0xd6, 0xbe, 0x00}}, {2, [3]byte{0xd6, 0xbf, 0x00}}, {2, [3]byte{0xd7, 0x80, 0x00}}, {2, [3]byte{0xd7, 0x81, 0x00}}, @@ -5943,24 +6643,24 @@ var windows1255 = charmap{ 0xb20000b2, 0xb30000b3, 0xb40000b4, 0xb50000b5, 0xb60000b6, 0xb70000b7, 0xb80000b8, 0xb90000b9, 0xbb0000bb, 0xbc0000bc, 0xbd0000bd, 0xbe0000be, 0xbf0000bf, 0xaa0000d7, 0xba0000f7, 0x83000192, 0x880002c6, 0x980002dc, 0xc00005b0, 0xc10005b1, 0xc20005b2, 0xc30005b3, 0xc40005b4, 0xc50005b5, - 0xc60005b6, 0xc70005b7, 0xc80005b8, 0xc90005b9, 0xcb0005bb, 0xcc0005bc, 0xcd0005bd, 0xce0005be, - 0xcf0005bf, 0xd00005c0, 0xd10005c1, 0xd20005c2, 0xd30005c3, 0xe00005d0, 0xe10005d1, 0xe20005d2, - 0xe30005d3, 0xe40005d4, 0xe50005d5, 0xe60005d6, 0xe70005d7, 0xe80005d8, 0xe90005d9, 0xea0005da, - 0xeb0005db, 0xec0005dc, 0xed0005dd, 0xee0005de, 0xef0005df, 0xf00005e0, 0xf10005e1, 0xf20005e2, - 0xf30005e3, 0xf40005e4, 0xf50005e5, 0xf60005e6, 0xf70005e7, 0xf80005e8, 0xf90005e9, 0xfa0005ea, - 0xd40005f0, 0xd50005f1, 0xd60005f2, 0xd70005f3, 0xd80005f4, 0xfd00200e, 0xfe00200f, 0x96002013, - 0x97002014, 0x91002018, 0x92002019, 0x8200201a, 0x9300201c, 0x9400201d, 0x8400201e, 0x86002020, - 0x87002021, 0x95002022, 0x85002026, 0x89002030, 0x8b002039, 0x9b00203a, 0xa40020aa, 0x800020ac, - 0x99002122, 0x99002122, 0x99002122, 0x99002122, 0x99002122, 0x99002122, 0x99002122, 0x99002122, + 0xc60005b6, 0xc70005b7, 0xc80005b8, 0xc90005b9, 0xca0005ba, 0xcb0005bb, 0xcc0005bc, 0xcd0005bd, + 0xce0005be, 0xcf0005bf, 0xd00005c0, 0xd10005c1, 0xd20005c2, 0xd30005c3, 0xe00005d0, 0xe10005d1, + 0xe20005d2, 0xe30005d3, 0xe40005d4, 0xe50005d5, 0xe60005d6, 0xe70005d7, 0xe80005d8, 0xe90005d9, + 0xea0005da, 0xeb0005db, 0xec0005dc, 0xed0005dd, 0xee0005de, 0xef0005df, 0xf00005e0, 0xf10005e1, + 0xf20005e2, 0xf30005e3, 0xf40005e4, 0xf50005e5, 0xf60005e6, 0xf70005e7, 0xf80005e8, 0xf90005e9, + 0xfa0005ea, 0xd40005f0, 0xd50005f1, 0xd60005f2, 0xd70005f3, 0xd80005f4, 0xfd00200e, 0xfe00200f, + 0x96002013, 0x97002014, 0x91002018, 0x92002019, 0x8200201a, 0x9300201c, 0x9400201d, 0x8400201e, + 0x86002020, 0x87002021, 0x95002022, 0x85002026, 0x89002030, 0x8b002039, 0x9b00203a, 0xa40020aa, + 0x800020ac, 0x99002122, 0x99002122, 0x99002122, 0x99002122, 0x99002122, 0x99002122, 0x99002122, 0x99002122, 0x99002122, 0x99002122, 0x99002122, 0x99002122, 0x99002122, 0x99002122, 0x99002122, 0x99002122, 0x99002122, 0x99002122, 0x99002122, 0x99002122, 0x99002122, 0x99002122, 0x99002122, }, } // Windows1256 is the Windows 1256 encoding. -var Windows1256 encoding.Encoding = &windows1256 +var Windows1256 *Charmap = &windows1256 -var windows1256 = charmap{ +var windows1256 = Charmap{ name: "Windows 1256", mib: identifier.Windows1256, asciiSuperset: true, @@ -6133,9 +6833,9 @@ var windows1256 = charmap{ } // Windows1257 is the Windows 1257 encoding. -var Windows1257 encoding.Encoding = &windows1257 +var Windows1257 *Charmap = &windows1257 -var windows1257 = charmap{ +var windows1257 = Charmap{ name: "Windows 1257", mib: identifier.Windows1257, asciiSuperset: true, @@ -6308,9 +7008,9 @@ var windows1257 = charmap{ } // Windows1258 is the Windows 1258 encoding. -var Windows1258 encoding.Encoding = &windows1258 +var Windows1258 *Charmap = &windows1258 -var windows1258 = charmap{ +var windows1258 = Charmap{ name: "Windows 1258", mib: identifier.Windows1258, asciiSuperset: true, @@ -6485,9 +7185,9 @@ var windows1258 = charmap{ // XUserDefined is the X-User-Defined encoding. // // It is defined at http://encoding.spec.whatwg.org/#x-user-defined -var XUserDefined encoding.Encoding = &xUserDefined +var XUserDefined *Charmap = &xUserDefined -var xUserDefined = charmap{ +var xUserDefined = Charmap{ name: "X-User-Defined", mib: identifier.XUserDefined, asciiSuperset: true, @@ -6659,6 +7359,7 @@ var xUserDefined = charmap{ }, } var listAll = []encoding.Encoding{ + CodePage037, CodePage437, CodePage850, CodePage852, @@ -6669,6 +7370,8 @@ var listAll = []encoding.Encoding{ CodePage863, CodePage865, CodePage866, + CodePage1047, + CodePage1140, ISO8859_1, ISO8859_2, ISO8859_3, @@ -6681,6 +7384,7 @@ var listAll = []encoding.Encoding{ ISO8859_8, ISO8859_8E, ISO8859_8I, + ISO8859_9, ISO8859_10, ISO8859_13, ISO8859_14, @@ -6703,4 +7407,4 @@ var listAll = []encoding.Encoding{ XUserDefined, } -// Total table size 78736 bytes (76KiB); checksum: 811C9DC5 +// Total table size 87024 bytes (84KiB); checksum: 811C9DC5 diff --git a/vendor/golang.org/x/text/encoding/htmlindex/gen.go b/vendor/golang.org/x/text/encoding/htmlindex/gen.go deleted file mode 100644 index d10e5e09f..000000000 --- a/vendor/golang.org/x/text/encoding/htmlindex/gen.go +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "log" - "strings" - - "golang.org/x/text/internal/gen" -) - -type group struct { - Encodings []struct { - Labels []string - Name string - } -} - -func main() { - gen.Init() - - r := gen.Open("http://www.w3.org/TR", "w3", "encoding/indexes/encodings.json") - var groups []group - if err := json.NewDecoder(r).Decode(&groups); err != nil { - log.Fatalf("Error reading encodings.json: %v", err) - } - - w := &bytes.Buffer{} - fmt.Fprintln(w, "type htmlEncoding byte") - fmt.Fprintln(w, "const (") - for i, g := range groups { - for _, e := range g.Encodings { - name := consts[e.Name] - if name == "" { - log.Fatalf("No const defined for %s.", e.Name) - } - if i == 0 { - fmt.Fprintf(w, "%s htmlEncoding = iota\n", name) - } else { - fmt.Fprintf(w, "%s\n", name) - } - } - } - fmt.Fprintln(w, "numEncodings") - fmt.Fprint(w, ")\n\n") - - fmt.Fprintln(w, "var canonical = [numEncodings]string{") - for _, g := range groups { - for _, e := range g.Encodings { - fmt.Fprintf(w, "%q,\n", e.Name) - } - } - fmt.Fprint(w, "}\n\n") - - fmt.Fprintln(w, "var nameMap = map[string]htmlEncoding{") - for _, g := range groups { - for _, e := range g.Encodings { - for _, l := range e.Labels { - fmt.Fprintf(w, "%q: %s,\n", l, consts[e.Name]) - } - } - } - fmt.Fprint(w, "}\n\n") - - var tags []string - fmt.Fprintln(w, "var localeMap = []htmlEncoding{") - for _, loc := range locales { - tags = append(tags, loc.tag) - fmt.Fprintf(w, "%s, // %s \n", consts[loc.name], loc.tag) - } - fmt.Fprint(w, "}\n\n") - - fmt.Fprintf(w, "const locales = %q\n", strings.Join(tags, " ")) - - gen.WriteGoFile("tables.go", "htmlindex", w.Bytes()) -} - -// consts maps canonical encoding name to internal constant. -var consts = map[string]string{ - "utf-8": "utf8", - "ibm866": "ibm866", - "iso-8859-2": "iso8859_2", - "iso-8859-3": "iso8859_3", - "iso-8859-4": "iso8859_4", - "iso-8859-5": "iso8859_5", - "iso-8859-6": "iso8859_6", - "iso-8859-7": "iso8859_7", - "iso-8859-8": "iso8859_8", - "iso-8859-8-i": "iso8859_8I", - "iso-8859-10": "iso8859_10", - "iso-8859-13": "iso8859_13", - "iso-8859-14": "iso8859_14", - "iso-8859-15": "iso8859_15", - "iso-8859-16": "iso8859_16", - "koi8-r": "koi8r", - "koi8-u": "koi8u", - "macintosh": "macintosh", - "windows-874": "windows874", - "windows-1250": "windows1250", - "windows-1251": "windows1251", - "windows-1252": "windows1252", - "windows-1253": "windows1253", - "windows-1254": "windows1254", - "windows-1255": "windows1255", - "windows-1256": "windows1256", - "windows-1257": "windows1257", - "windows-1258": "windows1258", - "x-mac-cyrillic": "macintoshCyrillic", - "gbk": "gbk", - "gb18030": "gb18030", - // "hz-gb-2312": "hzgb2312", // Was removed from WhatWG - "big5": "big5", - "euc-jp": "eucjp", - "iso-2022-jp": "iso2022jp", - "shift_jis": "shiftJIS", - "euc-kr": "euckr", - "replacement": "replacement", - "utf-16be": "utf16be", - "utf-16le": "utf16le", - "x-user-defined": "xUserDefined", -} - -// locales is taken from -// https://html.spec.whatwg.org/multipage/syntax.html#encoding-sniffing-algorithm. -var locales = []struct{ tag, name string }{ - {"und", "windows-1252"}, // The default value. - {"ar", "windows-1256"}, - {"ba", "windows-1251"}, - {"be", "windows-1251"}, - {"bg", "windows-1251"}, - {"cs", "windows-1250"}, - {"el", "iso-8859-7"}, - {"et", "windows-1257"}, - {"fa", "windows-1256"}, - {"he", "windows-1255"}, - {"hr", "windows-1250"}, - {"hu", "iso-8859-2"}, - {"ja", "shift_jis"}, - {"kk", "windows-1251"}, - {"ko", "euc-kr"}, - {"ku", "windows-1254"}, - {"ky", "windows-1251"}, - {"lt", "windows-1257"}, - {"lv", "windows-1257"}, - {"mk", "windows-1251"}, - {"pl", "iso-8859-2"}, - {"ru", "windows-1251"}, - {"sah", "windows-1251"}, - {"sk", "windows-1250"}, - {"sl", "iso-8859-2"}, - {"sr", "windows-1251"}, - {"tg", "windows-1251"}, - {"th", "windows-874"}, - {"tr", "windows-1254"}, - {"tt", "windows-1251"}, - {"uk", "windows-1251"}, - {"vi", "windows-1258"}, - {"zh-hans", "gb18030"}, - {"zh-hant", "big5"}, -} diff --git a/vendor/golang.org/x/text/encoding/htmlindex/tables.go b/vendor/golang.org/x/text/encoding/htmlindex/tables.go index 78950d3c3..cbf4ba923 100644 --- a/vendor/golang.org/x/text/encoding/htmlindex/tables.go +++ b/vendor/golang.org/x/text/encoding/htmlindex/tables.go @@ -1,4 +1,4 @@ -// This file was generated by go generate; DO NOT EDIT +// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. package htmlindex diff --git a/vendor/golang.org/x/text/encoding/internal/identifier/gen.go b/vendor/golang.org/x/text/encoding/internal/identifier/gen.go deleted file mode 100644 index 0c8eba7e5..000000000 --- a/vendor/golang.org/x/text/encoding/internal/identifier/gen.go +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -package main - -import ( - "bytes" - "encoding/xml" - "fmt" - "io" - "log" - "strings" - - "golang.org/x/text/internal/gen" -) - -type registry struct { - XMLName xml.Name `xml:"registry"` - Updated string `xml:"updated"` - Registry []struct { - ID string `xml:"id,attr"` - Record []struct { - Name string `xml:"name"` - Xref []struct { - Type string `xml:"type,attr"` - Data string `xml:"data,attr"` - } `xml:"xref"` - Desc struct { - Data string `xml:",innerxml"` - // Any []struct { - // Data string `xml:",chardata"` - // } `xml:",any"` - // Data string `xml:",chardata"` - } `xml:"description,"` - MIB string `xml:"value"` - Alias []string `xml:"alias"` - MIME string `xml:"preferred_alias"` - } `xml:"record"` - } `xml:"registry"` -} - -func main() { - r := gen.OpenIANAFile("assignments/character-sets/character-sets.xml") - reg := ®istry{} - if err := xml.NewDecoder(r).Decode(®); err != nil && err != io.EOF { - log.Fatalf("Error decoding charset registry: %v", err) - } - if len(reg.Registry) == 0 || reg.Registry[0].ID != "character-sets-1" { - log.Fatalf("Unexpected ID %s", reg.Registry[0].ID) - } - - w := &bytes.Buffer{} - fmt.Fprintf(w, "const (\n") - for _, rec := range reg.Registry[0].Record { - constName := "" - for _, a := range rec.Alias { - if strings.HasPrefix(a, "cs") && strings.IndexByte(a, '-') == -1 { - // Some of the constant definitions have comments in them. Strip those. - constName = strings.Title(strings.SplitN(a[2:], "\n", 2)[0]) - } - } - if constName == "" { - switch rec.MIB { - case "2085": - constName = "HZGB2312" // Not listed as alias for some reason. - default: - log.Fatalf("No cs alias defined for %s.", rec.MIB) - } - } - if rec.MIME != "" { - rec.MIME = fmt.Sprintf(" (MIME: %s)", rec.MIME) - } - fmt.Fprintf(w, "// %s is the MIB identifier with IANA name %s%s.\n//\n", constName, rec.Name, rec.MIME) - if len(rec.Desc.Data) > 0 { - fmt.Fprint(w, "// ") - d := xml.NewDecoder(strings.NewReader(rec.Desc.Data)) - inElem := true - attr := "" - for { - t, err := d.Token() - if err != nil { - if err != io.EOF { - log.Fatal(err) - } - break - } - switch x := t.(type) { - case xml.CharData: - attr = "" // Don't need attribute info. - a := bytes.Split([]byte(x), []byte("\n")) - for i, b := range a { - if b = bytes.TrimSpace(b); len(b) != 0 { - if !inElem && i > 0 { - fmt.Fprint(w, "\n// ") - } - inElem = false - fmt.Fprintf(w, "%s ", string(b)) - } - } - case xml.StartElement: - if x.Name.Local == "xref" { - inElem = true - use := false - for _, a := range x.Attr { - if a.Name.Local == "type" { - use = use || a.Value != "person" - } - if a.Name.Local == "data" && use { - attr = a.Value + " " - } - } - } - case xml.EndElement: - inElem = false - fmt.Fprint(w, attr) - } - } - fmt.Fprint(w, "\n") - } - for _, x := range rec.Xref { - switch x.Type { - case "rfc": - fmt.Fprintf(w, "// Reference: %s\n", strings.ToUpper(x.Data)) - case "uri": - fmt.Fprintf(w, "// Reference: %s\n", x.Data) - } - } - fmt.Fprintf(w, "%s MIB = %s\n", constName, rec.MIB) - fmt.Fprintln(w) - } - fmt.Fprintln(w, ")") - - gen.WriteGoFile("mib.go", "identifier", w.Bytes()) -} diff --git a/vendor/golang.org/x/text/encoding/internal/identifier/identifier.go b/vendor/golang.org/x/text/encoding/internal/identifier/identifier.go index 2a2da0ef2..7351b4ef8 100644 --- a/vendor/golang.org/x/text/encoding/internal/identifier/identifier.go +++ b/vendor/golang.org/x/text/encoding/internal/identifier/identifier.go @@ -36,8 +36,8 @@ package identifier // - http://www.ietf.org/rfc/rfc2978.txt // - http://www.unicode.org/reports/tr22/ // - http://www.w3.org/TR/encoding/ -// - http://www.w3.org/TR/encoding/indexes/encodings.json // - https://encoding.spec.whatwg.org/ +// - https://encoding.spec.whatwg.org/encodings.json // - https://tools.ietf.org/html/rfc6657#section-5 // Interface can be implemented by Encodings to define the CCS or CES for which diff --git a/vendor/golang.org/x/text/encoding/internal/identifier/mib.go b/vendor/golang.org/x/text/encoding/internal/identifier/mib.go index 915abfa29..768842b0a 100644 --- a/vendor/golang.org/x/text/encoding/internal/identifier/mib.go +++ b/vendor/golang.org/x/text/encoding/internal/identifier/mib.go @@ -1,4 +1,4 @@ -// This file was generated by go generate; DO NOT EDIT +// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. package identifier diff --git a/vendor/golang.org/x/text/encoding/japanese/eucjp.go b/vendor/golang.org/x/text/encoding/japanese/eucjp.go index 40f9b05f3..79313fa58 100644 --- a/vendor/golang.org/x/text/encoding/japanese/eucjp.go +++ b/vendor/golang.org/x/text/encoding/japanese/eucjp.go @@ -5,7 +5,6 @@ package japanese import ( - "errors" "unicode/utf8" "golang.org/x/text/encoding" @@ -23,10 +22,9 @@ var eucJP = internal.Encoding{ identifier.EUCPkdFmtJapanese, } -var errInvalidEUCJP = errors.New("japanese: invalid EUC-JP encoding") - type eucJPDecoder struct{ transform.NopResetter } +// See https://encoding.spec.whatwg.org/#euc-jp-decoder. func (eucJPDecoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { r, size := rune(0), 0 loop: @@ -37,60 +35,79 @@ loop: case c0 == 0x8e: if nSrc+1 >= len(src) { - err = transform.ErrShortSrc - break loop + if !atEOF { + err = transform.ErrShortSrc + break loop + } + r, size = utf8.RuneError, 1 + break } c1 := src[nSrc+1] - if c1 < 0xa1 || 0xdf < c1 { - err = errInvalidEUCJP - break loop + switch { + case c1 < 0xa1: + r, size = utf8.RuneError, 1 + case c1 > 0xdf: + r, size = utf8.RuneError, 2 + if c1 == 0xff { + size = 1 + } + default: + r, size = rune(c1)+(0xff61-0xa1), 2 } - r, size = rune(c1)+(0xff61-0xa1), 2 - case c0 == 0x8f: if nSrc+2 >= len(src) { - err = transform.ErrShortSrc - break loop + if !atEOF { + err = transform.ErrShortSrc + break loop + } + r, size = utf8.RuneError, 1 + if p := nSrc + 1; p < len(src) && 0xa1 <= src[p] && src[p] < 0xfe { + size = 2 + } + break } c1 := src[nSrc+1] if c1 < 0xa1 || 0xfe < c1 { - err = errInvalidEUCJP - break loop + r, size = utf8.RuneError, 1 + break } c2 := src[nSrc+2] if c2 < 0xa1 || 0xfe < c2 { - err = errInvalidEUCJP - break loop + r, size = utf8.RuneError, 2 + break } - r, size = '\ufffd', 3 + r, size = utf8.RuneError, 3 if i := int(c1-0xa1)*94 + int(c2-0xa1); i < len(jis0212Decode) { r = rune(jis0212Decode[i]) if r == 0 { - r = '\ufffd' + r = utf8.RuneError } } case 0xa1 <= c0 && c0 <= 0xfe: if nSrc+1 >= len(src) { - err = transform.ErrShortSrc - break loop + if !atEOF { + err = transform.ErrShortSrc + break loop + } + r, size = utf8.RuneError, 1 + break } c1 := src[nSrc+1] if c1 < 0xa1 || 0xfe < c1 { - err = errInvalidEUCJP - break loop + r, size = utf8.RuneError, 1 + break } - r, size = '\ufffd', 2 + r, size = utf8.RuneError, 2 if i := int(c0-0xa1)*94 + int(c1-0xa1); i < len(jis0208Decode) { r = rune(jis0208Decode[i]) if r == 0 { - r = '\ufffd' + r = utf8.RuneError } } default: - err = errInvalidEUCJP - break loop + r, size = utf8.RuneError, 1 } if nDst+utf8.RuneLen(r) > len(dst) { @@ -99,9 +116,6 @@ loop: } nDst += utf8.EncodeRune(dst[nDst:], r) } - if atEOF && err == transform.ErrShortSrc { - err = errInvalidEUCJP - } return nDst, nSrc, err } diff --git a/vendor/golang.org/x/text/encoding/japanese/iso2022jp.go b/vendor/golang.org/x/text/encoding/japanese/iso2022jp.go index b63e7d5d8..613226df5 100644 --- a/vendor/golang.org/x/text/encoding/japanese/iso2022jp.go +++ b/vendor/golang.org/x/text/encoding/japanese/iso2022jp.go @@ -5,7 +5,6 @@ package japanese import ( - "errors" "unicode/utf8" "golang.org/x/text/encoding" @@ -31,8 +30,6 @@ func iso2022JPNewEncoder() transform.Transformer { return new(iso2022JPEncoder) } -var errInvalidISO2022JP = errors.New("japanese: invalid ISO-2022-JP encoding") - const ( asciiState = iota katakanaState @@ -50,45 +47,51 @@ func (d *iso2022JPDecoder) Reset() { func (d *iso2022JPDecoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { r, size := rune(0), 0 -loop: for ; nSrc < len(src); nSrc += size { c0 := src[nSrc] if c0 >= utf8.RuneSelf { - err = errInvalidISO2022JP - break loop + r, size = '\ufffd', 1 + goto write } if c0 == asciiEsc { if nSrc+2 >= len(src) { - err = transform.ErrShortSrc - break loop + if !atEOF { + return nDst, nSrc, transform.ErrShortSrc + } + // TODO: is it correct to only skip 1?? + r, size = '\ufffd', 1 + goto write } size = 3 c1 := src[nSrc+1] c2 := src[nSrc+2] switch { - case c1 == '$' && (c2 == '@' || c2 == 'B'): + case c1 == '$' && (c2 == '@' || c2 == 'B'): // 0x24 {0x40, 0x42} *d = jis0208State continue - case c1 == '$' && c2 == '(': + case c1 == '$' && c2 == '(': // 0x24 0x28 if nSrc+3 >= len(src) { - err = transform.ErrShortSrc - break loop + if !atEOF { + return nDst, nSrc, transform.ErrShortSrc + } + r, size = '\ufffd', 1 + goto write } size = 4 - if src[nSrc]+3 == 'D' { + if src[nSrc+3] == 'D' { *d = jis0212State continue } - case c1 == '(' && (c2 == 'B' || c2 == 'J'): + case c1 == '(' && (c2 == 'B' || c2 == 'J'): // 0x28 {0x42, 0x4A} *d = asciiState continue - case c1 == '(' && c2 == 'I': + case c1 == '(' && c2 == 'I': // 0x28 0x49 *d = katakanaState continue } - err = errInvalidISO2022JP - break loop + r, size = '\ufffd', 1 + goto write } switch *d { @@ -97,8 +100,8 @@ loop: case katakanaState: if c0 < 0x21 || 0x60 <= c0 { - err = errInvalidISO2022JP - break loop + r, size = '\ufffd', 1 + goto write } r, size = rune(c0)+(0xff61-0x21), 1 @@ -106,11 +109,14 @@ loop: if c0 == 0x0a { *d = asciiState r, size = rune(c0), 1 - break + goto write } if nSrc+1 >= len(src) { - err = transform.ErrShortSrc - break loop + if !atEOF { + return nDst, nSrc, transform.ErrShortSrc + } + r, size = '\ufffd', 1 + goto write } size = 2 c1 := src[nSrc+1] @@ -121,22 +127,19 @@ loop: r = rune(jis0212Decode[i]) } else { r = '\ufffd' - break + goto write } if r == 0 { r = '\ufffd' } } + write: if nDst+utf8.RuneLen(r) > len(dst) { - err = transform.ErrShortDst - break loop + return nDst, nSrc, transform.ErrShortDst } nDst += utf8.EncodeRune(dst[nDst:], r) } - if atEOF && err == transform.ErrShortSrc { - err = errInvalidISO2022JP - } return nDst, nSrc, err } diff --git a/vendor/golang.org/x/text/encoding/japanese/maketables.go b/vendor/golang.org/x/text/encoding/japanese/maketables.go deleted file mode 100644 index d6c10deb0..000000000 --- a/vendor/golang.org/x/text/encoding/japanese/maketables.go +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -package main - -// This program generates tables.go: -// go run maketables.go | gofmt > tables.go - -// TODO: Emoji extensions? -// http://www.unicode.org/faq/emoji_dingbats.html -// http://www.unicode.org/Public/UNIDATA/EmojiSources.txt - -import ( - "bufio" - "fmt" - "log" - "net/http" - "sort" - "strings" -) - -type entry struct { - jisCode, table int -} - -func main() { - fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n") - fmt.Printf("// Package japanese provides Japanese encodings such as EUC-JP and Shift JIS.\n") - fmt.Printf(`package japanese // import "golang.org/x/text/encoding/japanese"` + "\n\n") - - reverse := [65536]entry{} - for i := range reverse { - reverse[i].table = -1 - } - - tables := []struct { - url string - name string - }{ - {"http://encoding.spec.whatwg.org/index-jis0208.txt", "0208"}, - {"http://encoding.spec.whatwg.org/index-jis0212.txt", "0212"}, - } - for i, table := range tables { - res, err := http.Get(table.url) - if err != nil { - log.Fatalf("%q: Get: %v", table.url, err) - } - defer res.Body.Close() - - mapping := [65536]uint16{} - - scanner := bufio.NewScanner(res.Body) - for scanner.Scan() { - s := strings.TrimSpace(scanner.Text()) - if s == "" || s[0] == '#' { - continue - } - x, y := 0, uint16(0) - if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil { - log.Fatalf("%q: could not parse %q", table.url, s) - } - if x < 0 || 120*94 <= x { - log.Fatalf("%q: JIS code %d is out of range", table.url, x) - } - mapping[x] = y - if reverse[y].table == -1 { - reverse[y] = entry{jisCode: x, table: i} - } - } - if err := scanner.Err(); err != nil { - log.Fatalf("%q: scanner error: %v", table.url, err) - } - - fmt.Printf("// jis%sDecode is the decoding table from JIS %s code to Unicode.\n// It is defined at %s\n", - table.name, table.name, table.url) - fmt.Printf("var jis%sDecode = [...]uint16{\n", table.name) - for i, m := range mapping { - if m != 0 { - fmt.Printf("\t%d: 0x%04X,\n", i, m) - } - } - fmt.Printf("}\n\n") - } - - // Any run of at least separation continuous zero entries in the reverse map will - // be a separate encode table. - const separation = 1024 - - intervals := []interval(nil) - low, high := -1, -1 - for i, v := range reverse { - if v.table == -1 { - continue - } - if low < 0 { - low = i - } else if i-high >= separation { - if high >= 0 { - intervals = append(intervals, interval{low, high}) - } - low = i - } - high = i + 1 - } - if high >= 0 { - intervals = append(intervals, interval{low, high}) - } - sort.Sort(byDecreasingLength(intervals)) - - fmt.Printf("const (\n") - fmt.Printf("\tjis0208 = 1\n") - fmt.Printf("\tjis0212 = 2\n") - fmt.Printf("\tcodeMask = 0x7f\n") - fmt.Printf("\tcodeShift = 7\n") - fmt.Printf("\ttableShift = 14\n") - fmt.Printf(")\n\n") - - fmt.Printf("const numEncodeTables = %d\n\n", len(intervals)) - fmt.Printf("// encodeX are the encoding tables from Unicode to JIS code,\n") - fmt.Printf("// sorted by decreasing length.\n") - for i, v := range intervals { - fmt.Printf("// encode%d: %5d entries for runes in [%5d, %5d).\n", i, v.len(), v.low, v.high) - } - fmt.Printf("//\n") - fmt.Printf("// The high two bits of the value record whether the JIS code comes from the\n") - fmt.Printf("// JIS0208 table (high bits == 1) or the JIS0212 table (high bits == 2).\n") - fmt.Printf("// The low 14 bits are two 7-bit unsigned integers j1 and j2 that form the\n") - fmt.Printf("// JIS code (94*j1 + j2) within that table.\n") - fmt.Printf("\n") - - for i, v := range intervals { - fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high) - fmt.Printf("var encode%d = [...]uint16{\n", i) - for j := v.low; j < v.high; j++ { - x := reverse[j] - if x.table == -1 { - continue - } - fmt.Printf("\t%d - %d: jis%s<<14 | 0x%02X<<7 | 0x%02X,\n", - j, v.low, tables[x.table].name, x.jisCode/94, x.jisCode%94) - } - fmt.Printf("}\n\n") - } -} - -// interval is a half-open interval [low, high). -type interval struct { - low, high int -} - -func (i interval) len() int { return i.high - i.low } - -// byDecreasingLength sorts intervals by decreasing length. -type byDecreasingLength []interval - -func (b byDecreasingLength) Len() int { return len(b) } -func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() } -func (b byDecreasingLength) Swap(i, j int) { b[i], b[j] = b[j], b[i] } diff --git a/vendor/golang.org/x/text/encoding/japanese/shiftjis.go b/vendor/golang.org/x/text/encoding/japanese/shiftjis.go index 099aecc31..16fd8a6e3 100644 --- a/vendor/golang.org/x/text/encoding/japanese/shiftjis.go +++ b/vendor/golang.org/x/text/encoding/japanese/shiftjis.go @@ -5,7 +5,6 @@ package japanese import ( - "errors" "unicode/utf8" "golang.org/x/text/encoding" @@ -24,8 +23,6 @@ var shiftJIS = internal.Encoding{ identifier.ShiftJIS, } -var errInvalidShiftJIS = errors.New("japanese: invalid Shift JIS encoding") - type shiftJISDecoder struct{ transform.NopResetter } func (shiftJISDecoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { @@ -48,28 +45,32 @@ loop: c0 = 2*c0 - 0x21 if nSrc+1 >= len(src) { - err = transform.ErrShortSrc - break loop + if !atEOF { + err = transform.ErrShortSrc + break loop + } + r, size = '\ufffd', 1 + goto write } c1 := src[nSrc+1] switch { case c1 < 0x40: - err = errInvalidShiftJIS - break loop + r, size = '\ufffd', 1 // c1 is ASCII so output on next round + goto write case c1 < 0x7f: c0-- c1 -= 0x40 case c1 == 0x7f: - err = errInvalidShiftJIS - break loop + r, size = '\ufffd', 1 // c1 is ASCII so output on next round + goto write case c1 < 0x9f: c0-- c1 -= 0x41 case c1 < 0xfd: c1 -= 0x9f default: - err = errInvalidShiftJIS - break loop + r, size = '\ufffd', 2 + goto write } r, size = '\ufffd', 2 if i := int(c0)*94 + int(c1); i < len(jis0208Decode) { @@ -79,20 +80,19 @@ loop: } } - default: - err = errInvalidShiftJIS - break loop - } + case c0 == 0x80: + r, size = 0x80, 1 + default: + r, size = '\ufffd', 1 + } + write: if nDst+utf8.RuneLen(r) > len(dst) { err = transform.ErrShortDst break loop } nDst += utf8.EncodeRune(dst[nDst:], r) } - if atEOF && err == transform.ErrShortSrc { - err = errInvalidShiftJIS - } return nDst, nSrc, err } diff --git a/vendor/golang.org/x/text/encoding/korean/euckr.go b/vendor/golang.org/x/text/encoding/korean/euckr.go index a4b9ff178..034337f5d 100644 --- a/vendor/golang.org/x/text/encoding/korean/euckr.go +++ b/vendor/golang.org/x/text/encoding/korean/euckr.go @@ -5,7 +5,6 @@ package korean import ( - "errors" "unicode/utf8" "golang.org/x/text/encoding" @@ -26,8 +25,6 @@ var eucKR = internal.Encoding{ identifier.EUCKR, } -var errInvalidEUCKR = errors.New("korean: invalid EUC-KR encoding") - type eucKRDecoder struct{ transform.NopResetter } func (eucKRDecoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { @@ -40,10 +37,15 @@ loop: case 0x81 <= c0 && c0 < 0xff: if nSrc+1 >= len(src) { - err = transform.ErrShortSrc - break loop + if !atEOF { + err = transform.ErrShortSrc + break loop + } + r, size = utf8.RuneError, 1 + break } c1 := src[nSrc+1] + size = 2 if c0 < 0xc7 { r = 178 * rune(c0-0x81) switch { @@ -54,39 +56,36 @@ loop: case 0x81 <= c1 && c1 < 0xff: r += rune(c1) - (0x81 - 2*26) default: - err = errInvalidEUCKR - break loop + goto decError } } else if 0xa1 <= c1 && c1 < 0xff { r = 178*(0xc7-0x81) + rune(c0-0xc7)*94 + rune(c1-0xa1) } else { - err = errInvalidEUCKR - break loop + goto decError } if int(r) < len(decode) { r = rune(decode[r]) - if r == 0 { - r = '\ufffd' + if r != 0 { + break } - } else { - r = '\ufffd' } - size = 2 + decError: + r = utf8.RuneError + if c1 < utf8.RuneSelf { + size = 1 + } default: - err = errInvalidEUCKR - break loop + r, size = utf8.RuneError, 1 + break } if nDst+utf8.RuneLen(r) > len(dst) { err = transform.ErrShortDst - break loop + break } nDst += utf8.EncodeRune(dst[nDst:], r) } - if atEOF && err == transform.ErrShortSrc { - err = errInvalidEUCKR - } return nDst, nSrc, err } diff --git a/vendor/golang.org/x/text/encoding/korean/maketables.go b/vendor/golang.org/x/text/encoding/korean/maketables.go deleted file mode 100644 index c84034fb6..000000000 --- a/vendor/golang.org/x/text/encoding/korean/maketables.go +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -package main - -// This program generates tables.go: -// go run maketables.go | gofmt > tables.go - -import ( - "bufio" - "fmt" - "log" - "net/http" - "sort" - "strings" -) - -func main() { - fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n") - fmt.Printf("// Package korean provides Korean encodings such as EUC-KR.\n") - fmt.Printf(`package korean // import "golang.org/x/text/encoding/korean"` + "\n\n") - - res, err := http.Get("http://encoding.spec.whatwg.org/index-euc-kr.txt") - if err != nil { - log.Fatalf("Get: %v", err) - } - defer res.Body.Close() - - mapping := [65536]uint16{} - reverse := [65536]uint16{} - - scanner := bufio.NewScanner(res.Body) - for scanner.Scan() { - s := strings.TrimSpace(scanner.Text()) - if s == "" || s[0] == '#' { - continue - } - x, y := uint16(0), uint16(0) - if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil { - log.Fatalf("could not parse %q", s) - } - if x < 0 || 178*(0xc7-0x81)+(0xfe-0xc7)*94+(0xff-0xa1) <= x { - log.Fatalf("EUC-KR code %d is out of range", x) - } - mapping[x] = y - if reverse[y] == 0 { - c0, c1 := uint16(0), uint16(0) - if x < 178*(0xc7-0x81) { - c0 = uint16(x/178) + 0x81 - c1 = uint16(x % 178) - switch { - case c1 < 1*26: - c1 += 0x41 - case c1 < 2*26: - c1 += 0x47 - default: - c1 += 0x4d - } - } else { - x -= 178 * (0xc7 - 0x81) - c0 = uint16(x/94) + 0xc7 - c1 = uint16(x%94) + 0xa1 - } - reverse[y] = c0<<8 | c1 - } - } - if err := scanner.Err(); err != nil { - log.Fatalf("scanner error: %v", err) - } - - fmt.Printf("// decode is the decoding table from EUC-KR code to Unicode.\n") - fmt.Printf("// It is defined at http://encoding.spec.whatwg.org/index-euc-kr.txt\n") - fmt.Printf("var decode = [...]uint16{\n") - for i, v := range mapping { - if v != 0 { - fmt.Printf("\t%d: 0x%04X,\n", i, v) - } - } - fmt.Printf("}\n\n") - - // Any run of at least separation continuous zero entries in the reverse map will - // be a separate encode table. - const separation = 1024 - - intervals := []interval(nil) - low, high := -1, -1 - for i, v := range reverse { - if v == 0 { - continue - } - if low < 0 { - low = i - } else if i-high >= separation { - if high >= 0 { - intervals = append(intervals, interval{low, high}) - } - low = i - } - high = i + 1 - } - if high >= 0 { - intervals = append(intervals, interval{low, high}) - } - sort.Sort(byDecreasingLength(intervals)) - - fmt.Printf("const numEncodeTables = %d\n\n", len(intervals)) - fmt.Printf("// encodeX are the encoding tables from Unicode to EUC-KR code,\n") - fmt.Printf("// sorted by decreasing length.\n") - for i, v := range intervals { - fmt.Printf("// encode%d: %5d entries for runes in [%5d, %5d).\n", i, v.len(), v.low, v.high) - } - fmt.Printf("\n") - - for i, v := range intervals { - fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high) - fmt.Printf("var encode%d = [...]uint16{\n", i) - for j := v.low; j < v.high; j++ { - x := reverse[j] - if x == 0 { - continue - } - fmt.Printf("\t%d-%d: 0x%04X,\n", j, v.low, x) - } - fmt.Printf("}\n\n") - } -} - -// interval is a half-open interval [low, high). -type interval struct { - low, high int -} - -func (i interval) len() int { return i.high - i.low } - -// byDecreasingLength sorts intervals by decreasing length. -type byDecreasingLength []interval - -func (b byDecreasingLength) Len() int { return len(b) } -func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() } -func (b byDecreasingLength) Swap(i, j int) { b[i], b[j] = b[j], b[i] } diff --git a/vendor/golang.org/x/text/encoding/simplifiedchinese/gbk.go b/vendor/golang.org/x/text/encoding/simplifiedchinese/gbk.go index e0b15bbcc..b89c45b03 100644 --- a/vendor/golang.org/x/text/encoding/simplifiedchinese/gbk.go +++ b/vendor/golang.org/x/text/encoding/simplifiedchinese/gbk.go @@ -5,7 +5,6 @@ package simplifiedchinese import ( - "errors" "unicode/utf8" "golang.org/x/text/encoding" @@ -40,11 +39,6 @@ var gbk18030 = internal.Encoding{ identifier.GB18030, } -var ( - errInvalidGB18030 = errors.New("simplifiedchinese: invalid GB18030 encoding") - errInvalidGBK = errors.New("simplifiedchinese: invalid GBK encoding") -) - type gbkDecoder struct { transform.NopResetter gb18030 bool @@ -66,8 +60,12 @@ loop: case c0 < 0xff: if nSrc+1 >= len(src) { - err = transform.ErrShortSrc - break loop + if !atEOF { + err = transform.ErrShortSrc + break loop + } + r, size = utf8.RuneError, 1 + goto write } c1 := src[nSrc+1] switch { @@ -77,18 +75,24 @@ loop: c1 -= 0x41 case d.gb18030 && 0x30 <= c1 && c1 < 0x40: if nSrc+3 >= len(src) { - err = transform.ErrShortSrc - break loop + if !atEOF { + err = transform.ErrShortSrc + break loop + } + // The second byte here is always ASCII, so we can set size + // to 1 in all cases. + r, size = utf8.RuneError, 1 + goto write } c2 := src[nSrc+2] if c2 < 0x81 || 0xff <= c2 { - err = errInvalidGB18030 - break loop + r, size = utf8.RuneError, 1 + goto write } c3 := src[nSrc+3] if c3 < 0x30 || 0x3a <= c3 { - err = errInvalidGB18030 - break loop + r, size = utf8.RuneError, 1 + goto write } size = 4 r = ((rune(c0-0x81)*10+rune(c1-0x30))*126+rune(c2-0x81))*10 + rune(c3-0x30) @@ -109,17 +113,13 @@ loop: r -= 189000 if 0 <= r && r < 0x100000 { r += 0x10000 - goto write - } - err = errInvalidGB18030 - break loop - default: - if d.gb18030 { - err = errInvalidGB18030 } else { - err = errInvalidGBK + r, size = utf8.RuneError, 1 } - break loop + goto write + default: + r, size = utf8.RuneError, 1 + goto write } r, size = '\ufffd', 2 if i := int(c0-0x81)*190 + int(c1); i < len(decode) { @@ -130,12 +130,7 @@ loop: } default: - if d.gb18030 { - err = errInvalidGB18030 - } else { - err = errInvalidGBK - } - break loop + r, size = utf8.RuneError, 1 } write: @@ -145,13 +140,6 @@ loop: } nDst += utf8.EncodeRune(dst[nDst:], r) } - if atEOF && err == transform.ErrShortSrc { - if d.gb18030 { - err = errInvalidGB18030 - } else { - err = errInvalidGBK - } - } return nDst, nSrc, err } diff --git a/vendor/golang.org/x/text/encoding/simplifiedchinese/hzgb2312.go b/vendor/golang.org/x/text/encoding/simplifiedchinese/hzgb2312.go index 85de6b1e6..eb3157f0b 100644 --- a/vendor/golang.org/x/text/encoding/simplifiedchinese/hzgb2312.go +++ b/vendor/golang.org/x/text/encoding/simplifiedchinese/hzgb2312.go @@ -5,7 +5,6 @@ package simplifiedchinese import ( - "errors" "unicode/utf8" "golang.org/x/text/encoding" @@ -31,8 +30,6 @@ func hzGB2312NewEncoder() transform.Transformer { return new(hzGB2312Encoder) } -var errInvalidHZGB2312 = errors.New("simplifiedchinese: invalid HZ-GB2312 encoding") - const ( asciiState = iota gbState @@ -50,14 +47,18 @@ loop: for ; nSrc < len(src); nSrc += size { c0 := src[nSrc] if c0 >= utf8.RuneSelf { - err = errInvalidHZGB2312 - break loop + r, size = utf8.RuneError, 1 + goto write } if c0 == '~' { if nSrc+1 >= len(src) { - err = transform.ErrShortSrc - break loop + if !atEOF { + err = transform.ErrShortSrc + break loop + } + r = utf8.RuneError + goto write } size = 2 switch src[nSrc+1] { @@ -78,8 +79,8 @@ loop: case '\n': continue default: - err = errInvalidHZGB2312 - break loop + r = utf8.RuneError + goto write } } @@ -87,33 +88,37 @@ loop: r, size = rune(c0), 1 } else { if nSrc+1 >= len(src) { - err = transform.ErrShortSrc - break loop + if !atEOF { + err = transform.ErrShortSrc + break loop + } + r, size = utf8.RuneError, 1 + goto write } + size = 2 c1 := src[nSrc+1] if c0 < 0x21 || 0x7e <= c0 || c1 < 0x21 || 0x7f <= c1 { - err = errInvalidHZGB2312 - break loop - } - - r, size = '\ufffd', 2 - if i := int(c0-0x01)*190 + int(c1+0x3f); i < len(decode) { + // error + } else if i := int(c0-0x01)*190 + int(c1+0x3f); i < len(decode) { r = rune(decode[i]) - if r == 0 { - r = '\ufffd' + if r != 0 { + goto write } } + if c1 > utf8.RuneSelf { + // Be consistent and always treat non-ASCII as a single error. + size = 1 + } + r = utf8.RuneError } + write: if nDst+utf8.RuneLen(r) > len(dst) { err = transform.ErrShortDst break loop } nDst += utf8.EncodeRune(dst[nDst:], r) } - if atEOF && err == transform.ErrShortSrc { - err = errInvalidHZGB2312 - } return nDst, nSrc, err } diff --git a/vendor/golang.org/x/text/encoding/simplifiedchinese/maketables.go b/vendor/golang.org/x/text/encoding/simplifiedchinese/maketables.go deleted file mode 100644 index 55016c786..000000000 --- a/vendor/golang.org/x/text/encoding/simplifiedchinese/maketables.go +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -package main - -// This program generates tables.go: -// go run maketables.go | gofmt > tables.go - -import ( - "bufio" - "fmt" - "log" - "net/http" - "sort" - "strings" -) - -func main() { - fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n") - fmt.Printf("// Package simplifiedchinese provides Simplified Chinese encodings such as GBK.\n") - fmt.Printf(`package simplifiedchinese // import "golang.org/x/text/encoding/simplifiedchinese"` + "\n\n") - - printGB18030() - printGBK() -} - -func printGB18030() { - res, err := http.Get("http://encoding.spec.whatwg.org/index-gb18030.txt") - if err != nil { - log.Fatalf("Get: %v", err) - } - defer res.Body.Close() - - fmt.Printf("// gb18030 is the table from http://encoding.spec.whatwg.org/index-gb18030.txt\n") - fmt.Printf("var gb18030 = [...][2]uint16{\n") - scanner := bufio.NewScanner(res.Body) - for scanner.Scan() { - s := strings.TrimSpace(scanner.Text()) - if s == "" || s[0] == '#' { - continue - } - x, y := uint32(0), uint32(0) - if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil { - log.Fatalf("could not parse %q", s) - } - if x < 0x10000 && y < 0x10000 { - fmt.Printf("\t{0x%04x, 0x%04x},\n", x, y) - } - } - fmt.Printf("}\n\n") -} - -func printGBK() { - res, err := http.Get("http://encoding.spec.whatwg.org/index-gbk.txt") - if err != nil { - log.Fatalf("Get: %v", err) - } - defer res.Body.Close() - - mapping := [65536]uint16{} - reverse := [65536]uint16{} - - scanner := bufio.NewScanner(res.Body) - for scanner.Scan() { - s := strings.TrimSpace(scanner.Text()) - if s == "" || s[0] == '#' { - continue - } - x, y := uint16(0), uint16(0) - if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil { - log.Fatalf("could not parse %q", s) - } - if x < 0 || 126*190 <= x { - log.Fatalf("GBK code %d is out of range", x) - } - mapping[x] = y - if reverse[y] == 0 { - c0, c1 := x/190, x%190 - if c1 >= 0x3f { - c1++ - } - reverse[y] = (0x81+c0)<<8 | (0x40 + c1) - } - } - if err := scanner.Err(); err != nil { - log.Fatalf("scanner error: %v", err) - } - - fmt.Printf("// decode is the decoding table from GBK code to Unicode.\n") - fmt.Printf("// It is defined at http://encoding.spec.whatwg.org/index-gbk.txt\n") - fmt.Printf("var decode = [...]uint16{\n") - for i, v := range mapping { - if v != 0 { - fmt.Printf("\t%d: 0x%04X,\n", i, v) - } - } - fmt.Printf("}\n\n") - - // Any run of at least separation continuous zero entries in the reverse map will - // be a separate encode table. - const separation = 1024 - - intervals := []interval(nil) - low, high := -1, -1 - for i, v := range reverse { - if v == 0 { - continue - } - if low < 0 { - low = i - } else if i-high >= separation { - if high >= 0 { - intervals = append(intervals, interval{low, high}) - } - low = i - } - high = i + 1 - } - if high >= 0 { - intervals = append(intervals, interval{low, high}) - } - sort.Sort(byDecreasingLength(intervals)) - - fmt.Printf("const numEncodeTables = %d\n\n", len(intervals)) - fmt.Printf("// encodeX are the encoding tables from Unicode to GBK code,\n") - fmt.Printf("// sorted by decreasing length.\n") - for i, v := range intervals { - fmt.Printf("// encode%d: %5d entries for runes in [%5d, %5d).\n", i, v.len(), v.low, v.high) - } - fmt.Printf("\n") - - for i, v := range intervals { - fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high) - fmt.Printf("var encode%d = [...]uint16{\n", i) - for j := v.low; j < v.high; j++ { - x := reverse[j] - if x == 0 { - continue - } - fmt.Printf("\t%d-%d: 0x%04X,\n", j, v.low, x) - } - fmt.Printf("}\n\n") - } -} - -// interval is a half-open interval [low, high). -type interval struct { - low, high int -} - -func (i interval) len() int { return i.high - i.low } - -// byDecreasingLength sorts intervals by decreasing length. -type byDecreasingLength []interval - -func (b byDecreasingLength) Len() int { return len(b) } -func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() } -func (b byDecreasingLength) Swap(i, j int) { b[i], b[j] = b[j], b[i] } diff --git a/vendor/golang.org/x/text/encoding/traditionalchinese/big5.go b/vendor/golang.org/x/text/encoding/traditionalchinese/big5.go index 275821f5d..1fcddde08 100644 --- a/vendor/golang.org/x/text/encoding/traditionalchinese/big5.go +++ b/vendor/golang.org/x/text/encoding/traditionalchinese/big5.go @@ -5,7 +5,6 @@ package traditionalchinese import ( - "errors" "unicode/utf8" "golang.org/x/text/encoding" @@ -26,8 +25,6 @@ var big5 = internal.Encoding{ identifier.Big5, } -var errInvalidBig5 = errors.New("traditionalchinese: invalid Big5 encoding") - type big5Decoder struct{ transform.NopResetter } func (big5Decoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { @@ -40,8 +37,12 @@ loop: case 0x81 <= c0 && c0 < 0xff: if nSrc+1 >= len(src) { - err = transform.ErrShortSrc - break loop + if !atEOF { + err = transform.ErrShortSrc + break loop + } + r, size = utf8.RuneError, 1 + goto write } c1 := src[nSrc+1] switch { @@ -49,9 +50,12 @@ loop: c1 -= 0x40 case 0xa1 <= c1 && c1 < 0xff: c1 -= 0x62 + case c1 < 0x40: + r, size = utf8.RuneError, 1 + goto write default: - err = errInvalidBig5 - break loop + r, size = utf8.RuneError, 2 + goto write } r, size = '\ufffd', 2 if i := int(c0-0x81)*157 + int(c1); i < len(decode) { @@ -80,10 +84,10 @@ loop: } default: - err = errInvalidBig5 - break loop + r, size = utf8.RuneError, 1 } + write: if nDst+utf8.RuneLen(r) > len(dst) { err = transform.ErrShortDst break loop @@ -99,9 +103,6 @@ loop: nDst += copy(dst[nDst:], s) continue loop } - if atEOF && err == transform.ErrShortSrc { - err = errInvalidBig5 - } return nDst, nSrc, err } diff --git a/vendor/golang.org/x/text/encoding/traditionalchinese/maketables.go b/vendor/golang.org/x/text/encoding/traditionalchinese/maketables.go deleted file mode 100644 index cf7fdb31a..000000000 --- a/vendor/golang.org/x/text/encoding/traditionalchinese/maketables.go +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -package main - -// This program generates tables.go: -// go run maketables.go | gofmt > tables.go - -import ( - "bufio" - "fmt" - "log" - "net/http" - "sort" - "strings" -) - -func main() { - fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n") - fmt.Printf("// Package traditionalchinese provides Traditional Chinese encodings such as Big5.\n") - fmt.Printf(`package traditionalchinese // import "golang.org/x/text/encoding/traditionalchinese"` + "\n\n") - - res, err := http.Get("http://encoding.spec.whatwg.org/index-big5.txt") - if err != nil { - log.Fatalf("Get: %v", err) - } - defer res.Body.Close() - - mapping := [65536]uint32{} - reverse := [65536 * 4]uint16{} - - scanner := bufio.NewScanner(res.Body) - for scanner.Scan() { - s := strings.TrimSpace(scanner.Text()) - if s == "" || s[0] == '#' { - continue - } - x, y := uint16(0), uint32(0) - if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil { - log.Fatalf("could not parse %q", s) - } - if x < 0 || 126*157 <= x { - log.Fatalf("Big5 code %d is out of range", x) - } - mapping[x] = y - - // The WHATWG spec http://encoding.spec.whatwg.org/#indexes says that - // "The index pointer for code point in index is the first pointer - // corresponding to code point in index", which would normally mean - // that the code below should be guarded by "if reverse[y] == 0", but - // last instead of first seems to match the behavior of - // "iconv -f UTF-8 -t BIG5". For example, U+8005 者 occurs twice in - // http://encoding.spec.whatwg.org/index-big5.txt, as index 2148 - // (encoded as "\x8e\xcd") and index 6543 (encoded as "\xaa\xcc") - // and "echo 者 | iconv -f UTF-8 -t BIG5 | xxd" gives "\xaa\xcc". - c0, c1 := x/157, x%157 - if c1 < 0x3f { - c1 += 0x40 - } else { - c1 += 0x62 - } - reverse[y] = (0x81+c0)<<8 | c1 - } - if err := scanner.Err(); err != nil { - log.Fatalf("scanner error: %v", err) - } - - fmt.Printf("// decode is the decoding table from Big5 code to Unicode.\n") - fmt.Printf("// It is defined at http://encoding.spec.whatwg.org/index-big5.txt\n") - fmt.Printf("var decode = [...]uint32{\n") - for i, v := range mapping { - if v != 0 { - fmt.Printf("\t%d: 0x%08X,\n", i, v) - } - } - fmt.Printf("}\n\n") - - // Any run of at least separation continuous zero entries in the reverse map will - // be a separate encode table. - const separation = 1024 - - intervals := []interval(nil) - low, high := -1, -1 - for i, v := range reverse { - if v == 0 { - continue - } - if low < 0 { - low = i - } else if i-high >= separation { - if high >= 0 { - intervals = append(intervals, interval{low, high}) - } - low = i - } - high = i + 1 - } - if high >= 0 { - intervals = append(intervals, interval{low, high}) - } - sort.Sort(byDecreasingLength(intervals)) - - fmt.Printf("const numEncodeTables = %d\n\n", len(intervals)) - fmt.Printf("// encodeX are the encoding tables from Unicode to Big5 code,\n") - fmt.Printf("// sorted by decreasing length.\n") - for i, v := range intervals { - fmt.Printf("// encode%d: %5d entries for runes in [%6d, %6d).\n", i, v.len(), v.low, v.high) - } - fmt.Printf("\n") - - for i, v := range intervals { - fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high) - fmt.Printf("var encode%d = [...]uint16{\n", i) - for j := v.low; j < v.high; j++ { - x := reverse[j] - if x == 0 { - continue - } - fmt.Printf("\t%d-%d: 0x%04X,\n", j, v.low, x) - } - fmt.Printf("}\n\n") - } -} - -// interval is a half-open interval [low, high). -type interval struct { - low, high int -} - -func (i interval) len() int { return i.high - i.low } - -// byDecreasingLength sorts intervals by decreasing length. -type byDecreasingLength []interval - -func (b byDecreasingLength) Len() int { return len(b) } -func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() } -func (b byDecreasingLength) Swap(i, j int) { b[i], b[j] = b[j], b[i] } diff --git a/vendor/golang.org/x/text/language/common.go b/vendor/golang.org/x/text/language/common.go index a255bb0a5..9d86e1855 100644 --- a/vendor/golang.org/x/text/language/common.go +++ b/vendor/golang.org/x/text/language/common.go @@ -1,4 +1,4 @@ -// This file was generated by go generate; DO NOT EDIT +// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. package language diff --git a/vendor/golang.org/x/text/language/gen_common.go b/vendor/golang.org/x/text/language/gen_common.go deleted file mode 100644 index 83ce18013..000000000 --- a/vendor/golang.org/x/text/language/gen_common.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -package main - -// This file contains code common to the maketables.go and the package code. - -// langAliasType is the type of an alias in langAliasMap. -type langAliasType int8 - -const ( - langDeprecated langAliasType = iota - langMacro - langLegacy - - langAliasTypeUnknown langAliasType = -1 -) diff --git a/vendor/golang.org/x/text/language/gen_index.go b/vendor/golang.org/x/text/language/gen_index.go deleted file mode 100644 index eef555cd3..000000000 --- a/vendor/golang.org/x/text/language/gen_index.go +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -package main - -// This file generates derivative tables based on the language package itself. - -import ( - "bytes" - "flag" - "fmt" - "io/ioutil" - "log" - "reflect" - "sort" - "strings" - - "golang.org/x/text/internal/gen" - "golang.org/x/text/language" - "golang.org/x/text/unicode/cldr" -) - -var ( - test = flag.Bool("test", false, - "test existing tables; can be used to compare web data with package data.") - - draft = flag.String("draft", - "contributed", - `Minimal draft requirements (approved, contributed, provisional, unconfirmed).`) -) - -func main() { - gen.Init() - - // Read the CLDR zip file. - r := gen.OpenCLDRCoreZip() - defer r.Close() - - d := &cldr.Decoder{} - data, err := d.DecodeZip(r) - if err != nil { - log.Fatalf("DecodeZip: %v", err) - } - - w := gen.NewCodeWriter() - defer func() { - buf := &bytes.Buffer{} - - if _, err = w.WriteGo(buf, "language"); err != nil { - log.Fatalf("Error formatting file index.go: %v", err) - } - - // Since we're generating a table for our own package we need to rewrite - // doing the equivalent of go fmt -r 'language.b -> b'. Using - // bytes.Replace will do. - out := bytes.Replace(buf.Bytes(), []byte("language."), nil, -1) - if err := ioutil.WriteFile("index.go", out, 0600); err != nil { - log.Fatalf("Could not create file index.go: %v", err) - } - }() - - m := map[language.Tag]bool{} - for _, lang := range data.Locales() { - // We include all locales unconditionally to be consistent with en_US. - // We want en_US, even though it has no data associated with it. - - // TODO: put any of the languages for which no data exists at the end - // of the index. This allows all components based on ICU to use that - // as the cutoff point. - // if x := data.RawLDML(lang); false || - // x.LocaleDisplayNames != nil || - // x.Characters != nil || - // x.Delimiters != nil || - // x.Measurement != nil || - // x.Dates != nil || - // x.Numbers != nil || - // x.Units != nil || - // x.ListPatterns != nil || - // x.Collations != nil || - // x.Segmentations != nil || - // x.Rbnf != nil || - // x.Annotations != nil || - // x.Metadata != nil { - - // TODO: support POSIX natively, albeit non-standard. - tag := language.Make(strings.Replace(lang, "_POSIX", "-u-va-posix", 1)) - m[tag] = true - // } - } - // Include locales for plural rules, which uses a different structure. - for _, plurals := range data.Supplemental().Plurals { - for _, rules := range plurals.PluralRules { - for _, lang := range strings.Split(rules.Locales, " ") { - m[language.Make(lang)] = true - } - } - } - - var core, special []language.Tag - - for t := range m { - if x := t.Extensions(); len(x) != 0 && fmt.Sprint(x) != "[u-va-posix]" { - log.Fatalf("Unexpected extension %v in %v", x, t) - } - if len(t.Variants()) == 0 && len(t.Extensions()) == 0 { - core = append(core, t) - } else { - special = append(special, t) - } - } - - w.WriteComment(` - NumCompactTags is the number of common tags. The maximum tag is - NumCompactTags-1.`) - w.WriteConst("NumCompactTags", len(core)+len(special)) - - sort.Sort(byAlpha(special)) - w.WriteVar("specialTags", special) - - // TODO: order by frequency? - sort.Sort(byAlpha(core)) - - // Size computations are just an estimate. - w.Size += int(reflect.TypeOf(map[uint32]uint16{}).Size()) - w.Size += len(core) * 6 // size of uint32 and uint16 - - fmt.Fprintln(w) - fmt.Fprintln(w, "var coreTags = map[uint32]uint16{") - fmt.Fprintln(w, "0x0: 0, // und") - i := len(special) + 1 // Und and special tags already written. - for _, t := range core { - if t == language.Und { - continue - } - fmt.Fprint(w.Hash, t, i) - b, s, r := t.Raw() - fmt.Fprintf(w, "0x%s%s%s: %d, // %s\n", - getIndex(b, 3), // 3 is enough as it is guaranteed to be a compact number - getIndex(s, 2), - getIndex(r, 3), - i, t) - i++ - } - fmt.Fprintln(w, "}") -} - -// getIndex prints the subtag type and extracts its index of size nibble. -// If the index is less than n nibbles, the result is prefixed with 0s. -func getIndex(x interface{}, n int) string { - s := fmt.Sprintf("%#v", x) // s is of form Type{typeID: 0x00} - s = s[strings.Index(s, "0x")+2 : len(s)-1] - return strings.Repeat("0", n-len(s)) + s -} - -type byAlpha []language.Tag - -func (a byAlpha) Len() int { return len(a) } -func (a byAlpha) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a byAlpha) Less(i, j int) bool { return a[i].String() < a[j].String() } diff --git a/vendor/golang.org/x/text/language/index.go b/vendor/golang.org/x/text/language/index.go index 50c752186..973db9fd5 100644 --- a/vendor/golang.org/x/text/language/index.go +++ b/vendor/golang.org/x/text/language/index.go @@ -1,767 +1,769 @@ -// This file was generated by go generate; DO NOT EDIT +// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. package language // NumCompactTags is the number of common tags. The maximum tag is // NumCompactTags-1. -const NumCompactTags = 752 +const NumCompactTags = 754 var specialTags = []Tag{ // 2 elements - 0: {lang: 0xd5, region: 0x6d, script: 0x0, pVariant: 0x5, pExt: 0xe, str: "ca-ES-valencia"}, - 1: {lang: 0x134, region: 0x134, script: 0x0, pVariant: 0x5, pExt: 0x5, str: "en-US-u-va-posix"}, + 0: {lang: 0xd7, region: 0x6d, script: 0x0, pVariant: 0x5, pExt: 0xe, str: "ca-ES-valencia"}, + 1: {lang: 0x138, region: 0x134, script: 0x0, pVariant: 0x5, pExt: 0x5, str: "en-US-u-va-posix"}, } // Size: 72 bytes var coreTags = map[uint32]uint16{ 0x0: 0, // und - 0x01500000: 3, // af - 0x015000d1: 4, // af-NA - 0x01500160: 5, // af-ZA - 0x01b00000: 6, // agq - 0x01b00051: 7, // agq-CM - 0x02000000: 8, // ak - 0x0200007f: 9, // ak-GH - 0x02600000: 10, // am - 0x0260006e: 11, // am-ET - 0x03900000: 12, // ar - 0x03900001: 13, // ar-001 - 0x03900022: 14, // ar-AE - 0x03900038: 15, // ar-BH - 0x03900061: 16, // ar-DJ - 0x03900066: 17, // ar-DZ - 0x0390006a: 18, // ar-EG - 0x0390006b: 19, // ar-EH - 0x0390006c: 20, // ar-ER - 0x03900096: 21, // ar-IL - 0x0390009a: 22, // ar-IQ - 0x039000a0: 23, // ar-JO - 0x039000a7: 24, // ar-KM - 0x039000ab: 25, // ar-KW - 0x039000af: 26, // ar-LB - 0x039000b8: 27, // ar-LY - 0x039000b9: 28, // ar-MA - 0x039000c8: 29, // ar-MR - 0x039000e0: 30, // ar-OM - 0x039000ec: 31, // ar-PS - 0x039000f2: 32, // ar-QA - 0x03900107: 33, // ar-SA - 0x0390010a: 34, // ar-SD - 0x03900114: 35, // ar-SO - 0x03900116: 36, // ar-SS - 0x0390011b: 37, // ar-SY - 0x0390011f: 38, // ar-TD - 0x03900127: 39, // ar-TN - 0x0390015d: 40, // ar-YE - 0x03f00000: 41, // ars - 0x04200000: 42, // as - 0x04200098: 43, // as-IN - 0x04300000: 44, // asa - 0x0430012e: 45, // asa-TZ - 0x04700000: 46, // ast - 0x0470006d: 47, // ast-ES - 0x05700000: 48, // az - 0x0571e000: 49, // az-Cyrl - 0x0571e031: 50, // az-Cyrl-AZ - 0x05752000: 51, // az-Latn - 0x05752031: 52, // az-Latn-AZ - 0x05d00000: 53, // bas - 0x05d00051: 54, // bas-CM - 0x07000000: 55, // be - 0x07000046: 56, // be-BY - 0x07400000: 57, // bem - 0x07400161: 58, // bem-ZM - 0x07800000: 59, // bez - 0x0780012e: 60, // bez-TZ - 0x07d00000: 61, // bg - 0x07d00037: 62, // bg-BG - 0x08100000: 63, // bh - 0x09e00000: 64, // bm - 0x09e000c2: 65, // bm-ML - 0x0a300000: 66, // bn - 0x0a300034: 67, // bn-BD - 0x0a300098: 68, // bn-IN - 0x0a700000: 69, // bo - 0x0a700052: 70, // bo-CN - 0x0a700098: 71, // bo-IN - 0x0b000000: 72, // br - 0x0b000077: 73, // br-FR - 0x0b300000: 74, // brx - 0x0b300098: 75, // brx-IN - 0x0b500000: 76, // bs - 0x0b51e000: 77, // bs-Cyrl - 0x0b51e032: 78, // bs-Cyrl-BA - 0x0b552000: 79, // bs-Latn - 0x0b552032: 80, // bs-Latn-BA - 0x0d500000: 81, // ca - 0x0d500021: 82, // ca-AD - 0x0d50006d: 83, // ca-ES - 0x0d500077: 84, // ca-FR - 0x0d50009d: 85, // ca-IT - 0x0da00000: 86, // ce - 0x0da00105: 87, // ce-RU - 0x0dd00000: 88, // cgg - 0x0dd00130: 89, // cgg-UG - 0x0e300000: 90, // chr - 0x0e300134: 91, // chr-US - 0x0e700000: 92, // ckb - 0x0e70009a: 93, // ckb-IQ - 0x0e70009b: 94, // ckb-IR - 0x0f600000: 95, // cs - 0x0f60005d: 96, // cs-CZ - 0x0fa00000: 97, // cu - 0x0fa00105: 98, // cu-RU - 0x0fc00000: 99, // cy - 0x0fc0007a: 100, // cy-GB - 0x0fd00000: 101, // da - 0x0fd00062: 102, // da-DK - 0x0fd00081: 103, // da-GL - 0x10400000: 104, // dav - 0x104000a3: 105, // dav-KE - 0x10900000: 106, // de - 0x1090002d: 107, // de-AT - 0x10900035: 108, // de-BE - 0x1090004d: 109, // de-CH - 0x1090005f: 110, // de-DE - 0x1090009d: 111, // de-IT - 0x109000b1: 112, // de-LI - 0x109000b6: 113, // de-LU - 0x11300000: 114, // dje - 0x113000d3: 115, // dje-NE - 0x11b00000: 116, // dsb - 0x11b0005f: 117, // dsb-DE - 0x12000000: 118, // dua - 0x12000051: 119, // dua-CM - 0x12400000: 120, // dv - 0x12700000: 121, // dyo - 0x12700113: 122, // dyo-SN - 0x12900000: 123, // dz - 0x12900042: 124, // dz-BT - 0x12b00000: 125, // ebu - 0x12b000a3: 126, // ebu-KE - 0x12c00000: 127, // ee - 0x12c0007f: 128, // ee-GH - 0x12c00121: 129, // ee-TG - 0x13100000: 130, // el - 0x1310005c: 131, // el-CY - 0x13100086: 132, // el-GR - 0x13400000: 133, // en - 0x13400001: 134, // en-001 - 0x1340001a: 135, // en-150 - 0x13400024: 136, // en-AG - 0x13400025: 137, // en-AI - 0x1340002c: 138, // en-AS - 0x1340002d: 139, // en-AT - 0x1340002e: 140, // en-AU - 0x13400033: 141, // en-BB - 0x13400035: 142, // en-BE - 0x13400039: 143, // en-BI - 0x1340003c: 144, // en-BM - 0x13400041: 145, // en-BS - 0x13400045: 146, // en-BW - 0x13400047: 147, // en-BZ - 0x13400048: 148, // en-CA - 0x13400049: 149, // en-CC - 0x1340004d: 150, // en-CH - 0x1340004f: 151, // en-CK - 0x13400051: 152, // en-CM - 0x1340005b: 153, // en-CX - 0x1340005c: 154, // en-CY - 0x1340005f: 155, // en-DE - 0x13400060: 156, // en-DG - 0x13400062: 157, // en-DK - 0x13400063: 158, // en-DM - 0x1340006c: 159, // en-ER - 0x13400071: 160, // en-FI - 0x13400072: 161, // en-FJ - 0x13400073: 162, // en-FK - 0x13400074: 163, // en-FM - 0x1340007a: 164, // en-GB - 0x1340007b: 165, // en-GD - 0x1340007e: 166, // en-GG - 0x1340007f: 167, // en-GH - 0x13400080: 168, // en-GI - 0x13400082: 169, // en-GM - 0x13400089: 170, // en-GU - 0x1340008b: 171, // en-GY - 0x1340008c: 172, // en-HK - 0x13400095: 173, // en-IE - 0x13400096: 174, // en-IL - 0x13400097: 175, // en-IM - 0x13400098: 176, // en-IN - 0x13400099: 177, // en-IO - 0x1340009e: 178, // en-JE - 0x1340009f: 179, // en-JM - 0x134000a3: 180, // en-KE - 0x134000a6: 181, // en-KI - 0x134000a8: 182, // en-KN - 0x134000ac: 183, // en-KY - 0x134000b0: 184, // en-LC - 0x134000b3: 185, // en-LR - 0x134000b4: 186, // en-LS - 0x134000be: 187, // en-MG - 0x134000bf: 188, // en-MH - 0x134000c5: 189, // en-MO - 0x134000c6: 190, // en-MP - 0x134000c9: 191, // en-MS - 0x134000ca: 192, // en-MT - 0x134000cb: 193, // en-MU - 0x134000cd: 194, // en-MW - 0x134000cf: 195, // en-MY - 0x134000d1: 196, // en-NA - 0x134000d4: 197, // en-NF - 0x134000d5: 198, // en-NG - 0x134000d8: 199, // en-NL - 0x134000dc: 200, // en-NR - 0x134000de: 201, // en-NU - 0x134000df: 202, // en-NZ - 0x134000e5: 203, // en-PG - 0x134000e6: 204, // en-PH - 0x134000e7: 205, // en-PK - 0x134000ea: 206, // en-PN - 0x134000eb: 207, // en-PR - 0x134000ef: 208, // en-PW - 0x13400106: 209, // en-RW - 0x13400108: 210, // en-SB - 0x13400109: 211, // en-SC - 0x1340010a: 212, // en-SD - 0x1340010b: 213, // en-SE - 0x1340010c: 214, // en-SG - 0x1340010d: 215, // en-SH - 0x1340010e: 216, // en-SI - 0x13400111: 217, // en-SL - 0x13400116: 218, // en-SS - 0x1340011a: 219, // en-SX - 0x1340011c: 220, // en-SZ - 0x1340011e: 221, // en-TC - 0x13400124: 222, // en-TK - 0x13400128: 223, // en-TO - 0x1340012b: 224, // en-TT - 0x1340012c: 225, // en-TV - 0x1340012e: 226, // en-TZ - 0x13400130: 227, // en-UG - 0x13400132: 228, // en-UM - 0x13400134: 229, // en-US - 0x13400138: 230, // en-VC - 0x1340013b: 231, // en-VG - 0x1340013c: 232, // en-VI - 0x1340013e: 233, // en-VU - 0x13400141: 234, // en-WS - 0x13400160: 235, // en-ZA - 0x13400161: 236, // en-ZM - 0x13400163: 237, // en-ZW - 0x13700000: 238, // eo - 0x13700001: 239, // eo-001 - 0x13900000: 240, // es - 0x1390001e: 241, // es-419 - 0x1390002b: 242, // es-AR - 0x1390003e: 243, // es-BO - 0x13900040: 244, // es-BR - 0x13900050: 245, // es-CL - 0x13900053: 246, // es-CO - 0x13900055: 247, // es-CR - 0x13900058: 248, // es-CU - 0x13900064: 249, // es-DO - 0x13900067: 250, // es-EA - 0x13900068: 251, // es-EC - 0x1390006d: 252, // es-ES - 0x13900085: 253, // es-GQ - 0x13900088: 254, // es-GT - 0x1390008e: 255, // es-HN - 0x13900093: 256, // es-IC - 0x139000ce: 257, // es-MX - 0x139000d7: 258, // es-NI - 0x139000e1: 259, // es-PA - 0x139000e3: 260, // es-PE - 0x139000e6: 261, // es-PH - 0x139000eb: 262, // es-PR - 0x139000f0: 263, // es-PY - 0x13900119: 264, // es-SV - 0x13900134: 265, // es-US - 0x13900135: 266, // es-UY - 0x1390013a: 267, // es-VE - 0x13b00000: 268, // et - 0x13b00069: 269, // et-EE - 0x14000000: 270, // eu - 0x1400006d: 271, // eu-ES - 0x14100000: 272, // ewo - 0x14100051: 273, // ewo-CM - 0x14300000: 274, // fa - 0x14300023: 275, // fa-AF - 0x1430009b: 276, // fa-IR - 0x14900000: 277, // ff - 0x14900051: 278, // ff-CM - 0x14900083: 279, // ff-GN - 0x149000c8: 280, // ff-MR - 0x14900113: 281, // ff-SN - 0x14c00000: 282, // fi - 0x14c00071: 283, // fi-FI - 0x14e00000: 284, // fil - 0x14e000e6: 285, // fil-PH - 0x15300000: 286, // fo - 0x15300062: 287, // fo-DK - 0x15300075: 288, // fo-FO - 0x15900000: 289, // fr - 0x15900035: 290, // fr-BE - 0x15900036: 291, // fr-BF - 0x15900039: 292, // fr-BI - 0x1590003a: 293, // fr-BJ - 0x1590003b: 294, // fr-BL - 0x15900048: 295, // fr-CA - 0x1590004a: 296, // fr-CD - 0x1590004b: 297, // fr-CF - 0x1590004c: 298, // fr-CG - 0x1590004d: 299, // fr-CH - 0x1590004e: 300, // fr-CI - 0x15900051: 301, // fr-CM - 0x15900061: 302, // fr-DJ - 0x15900066: 303, // fr-DZ - 0x15900077: 304, // fr-FR - 0x15900079: 305, // fr-GA - 0x1590007d: 306, // fr-GF - 0x15900083: 307, // fr-GN - 0x15900084: 308, // fr-GP - 0x15900085: 309, // fr-GQ - 0x15900090: 310, // fr-HT - 0x159000a7: 311, // fr-KM - 0x159000b6: 312, // fr-LU - 0x159000b9: 313, // fr-MA - 0x159000ba: 314, // fr-MC - 0x159000bd: 315, // fr-MF - 0x159000be: 316, // fr-MG - 0x159000c2: 317, // fr-ML - 0x159000c7: 318, // fr-MQ - 0x159000c8: 319, // fr-MR - 0x159000cb: 320, // fr-MU - 0x159000d2: 321, // fr-NC - 0x159000d3: 322, // fr-NE - 0x159000e4: 323, // fr-PF - 0x159000e9: 324, // fr-PM - 0x15900101: 325, // fr-RE - 0x15900106: 326, // fr-RW - 0x15900109: 327, // fr-SC - 0x15900113: 328, // fr-SN - 0x1590011b: 329, // fr-SY - 0x1590011f: 330, // fr-TD - 0x15900121: 331, // fr-TG - 0x15900127: 332, // fr-TN - 0x1590013e: 333, // fr-VU - 0x1590013f: 334, // fr-WF - 0x1590015e: 335, // fr-YT - 0x16400000: 336, // fur - 0x1640009d: 337, // fur-IT - 0x16800000: 338, // fy - 0x168000d8: 339, // fy-NL - 0x16900000: 340, // ga - 0x16900095: 341, // ga-IE - 0x17800000: 342, // gd - 0x1780007a: 343, // gd-GB - 0x18a00000: 344, // gl - 0x18a0006d: 345, // gl-ES - 0x19c00000: 346, // gsw - 0x19c0004d: 347, // gsw-CH - 0x19c00077: 348, // gsw-FR - 0x19c000b1: 349, // gsw-LI - 0x19d00000: 350, // gu - 0x19d00098: 351, // gu-IN - 0x1a200000: 352, // guw - 0x1a400000: 353, // guz - 0x1a4000a3: 354, // guz-KE - 0x1a500000: 355, // gv - 0x1a500097: 356, // gv-IM - 0x1ad00000: 357, // ha - 0x1ad0007f: 358, // ha-GH - 0x1ad000d3: 359, // ha-NE - 0x1ad000d5: 360, // ha-NG - 0x1b100000: 361, // haw - 0x1b100134: 362, // haw-US - 0x1b500000: 363, // he - 0x1b500096: 364, // he-IL - 0x1b700000: 365, // hi - 0x1b700098: 366, // hi-IN - 0x1ca00000: 367, // hr - 0x1ca00032: 368, // hr-BA - 0x1ca0008f: 369, // hr-HR - 0x1cb00000: 370, // hsb - 0x1cb0005f: 371, // hsb-DE - 0x1ce00000: 372, // hu - 0x1ce00091: 373, // hu-HU - 0x1d000000: 374, // hy - 0x1d000027: 375, // hy-AM - 0x1da00000: 376, // id - 0x1da00094: 377, // id-ID - 0x1df00000: 378, // ig - 0x1df000d5: 379, // ig-NG - 0x1e200000: 380, // ii - 0x1e200052: 381, // ii-CN - 0x1f000000: 382, // is - 0x1f00009c: 383, // is-IS - 0x1f100000: 384, // it - 0x1f10004d: 385, // it-CH - 0x1f10009d: 386, // it-IT - 0x1f100112: 387, // it-SM - 0x1f200000: 388, // iu - 0x1f800000: 389, // ja - 0x1f8000a1: 390, // ja-JP - 0x1fb00000: 391, // jbo - 0x1ff00000: 392, // jgo - 0x1ff00051: 393, // jgo-CM - 0x20200000: 394, // jmc - 0x2020012e: 395, // jmc-TZ - 0x20600000: 396, // jv - 0x20800000: 397, // ka - 0x2080007c: 398, // ka-GE - 0x20a00000: 399, // kab - 0x20a00066: 400, // kab-DZ - 0x20e00000: 401, // kaj - 0x20f00000: 402, // kam - 0x20f000a3: 403, // kam-KE - 0x21700000: 404, // kcg - 0x21b00000: 405, // kde - 0x21b0012e: 406, // kde-TZ - 0x21f00000: 407, // kea - 0x21f00059: 408, // kea-CV - 0x22c00000: 409, // khq - 0x22c000c2: 410, // khq-ML - 0x23100000: 411, // ki - 0x231000a3: 412, // ki-KE - 0x23a00000: 413, // kk - 0x23a000ad: 414, // kk-KZ - 0x23c00000: 415, // kkj - 0x23c00051: 416, // kkj-CM - 0x23d00000: 417, // kl - 0x23d00081: 418, // kl-GL - 0x23e00000: 419, // kln - 0x23e000a3: 420, // kln-KE - 0x24200000: 421, // km - 0x242000a5: 422, // km-KH - 0x24900000: 423, // kn - 0x24900098: 424, // kn-IN - 0x24b00000: 425, // ko - 0x24b000a9: 426, // ko-KP - 0x24b000aa: 427, // ko-KR - 0x24d00000: 428, // kok - 0x24d00098: 429, // kok-IN - 0x26100000: 430, // ks - 0x26100098: 431, // ks-IN - 0x26200000: 432, // ksb - 0x2620012e: 433, // ksb-TZ - 0x26400000: 434, // ksf - 0x26400051: 435, // ksf-CM - 0x26500000: 436, // ksh - 0x2650005f: 437, // ksh-DE - 0x26b00000: 438, // ku - 0x27800000: 439, // kw - 0x2780007a: 440, // kw-GB - 0x28100000: 441, // ky - 0x281000a4: 442, // ky-KG - 0x28800000: 443, // lag - 0x2880012e: 444, // lag-TZ - 0x28c00000: 445, // lb - 0x28c000b6: 446, // lb-LU - 0x29a00000: 447, // lg - 0x29a00130: 448, // lg-UG - 0x2a600000: 449, // lkt - 0x2a600134: 450, // lkt-US - 0x2ac00000: 451, // ln - 0x2ac00029: 452, // ln-AO - 0x2ac0004a: 453, // ln-CD - 0x2ac0004b: 454, // ln-CF - 0x2ac0004c: 455, // ln-CG - 0x2af00000: 456, // lo - 0x2af000ae: 457, // lo-LA - 0x2b600000: 458, // lrc - 0x2b60009a: 459, // lrc-IQ - 0x2b60009b: 460, // lrc-IR - 0x2b700000: 461, // lt - 0x2b7000b5: 462, // lt-LT - 0x2b900000: 463, // lu - 0x2b90004a: 464, // lu-CD - 0x2bb00000: 465, // luo - 0x2bb000a3: 466, // luo-KE - 0x2bc00000: 467, // luy - 0x2bc000a3: 468, // luy-KE - 0x2be00000: 469, // lv - 0x2be000b7: 470, // lv-LV - 0x2c800000: 471, // mas - 0x2c8000a3: 472, // mas-KE - 0x2c80012e: 473, // mas-TZ - 0x2e000000: 474, // mer - 0x2e0000a3: 475, // mer-KE - 0x2e400000: 476, // mfe - 0x2e4000cb: 477, // mfe-MU - 0x2e800000: 478, // mg - 0x2e8000be: 479, // mg-MG - 0x2e900000: 480, // mgh - 0x2e9000d0: 481, // mgh-MZ - 0x2eb00000: 482, // mgo - 0x2eb00051: 483, // mgo-CM - 0x2f600000: 484, // mk - 0x2f6000c1: 485, // mk-MK - 0x2fb00000: 486, // ml - 0x2fb00098: 487, // ml-IN - 0x30200000: 488, // mn - 0x302000c4: 489, // mn-MN - 0x31200000: 490, // mr - 0x31200098: 491, // mr-IN - 0x31600000: 492, // ms - 0x3160003d: 493, // ms-BN - 0x316000cf: 494, // ms-MY - 0x3160010c: 495, // ms-SG - 0x31700000: 496, // mt - 0x317000ca: 497, // mt-MT - 0x31c00000: 498, // mua - 0x31c00051: 499, // mua-CM - 0x32800000: 500, // my - 0x328000c3: 501, // my-MM - 0x33100000: 502, // mzn - 0x3310009b: 503, // mzn-IR - 0x33800000: 504, // nah - 0x33c00000: 505, // naq - 0x33c000d1: 506, // naq-NA - 0x33e00000: 507, // nb - 0x33e000d9: 508, // nb-NO - 0x33e0010f: 509, // nb-SJ - 0x34500000: 510, // nd - 0x34500163: 511, // nd-ZW - 0x34700000: 512, // nds - 0x3470005f: 513, // nds-DE - 0x347000d8: 514, // nds-NL - 0x34800000: 515, // ne - 0x34800098: 516, // ne-IN - 0x348000da: 517, // ne-NP - 0x35e00000: 518, // nl - 0x35e0002f: 519, // nl-AW - 0x35e00035: 520, // nl-BE - 0x35e0003f: 521, // nl-BQ - 0x35e0005a: 522, // nl-CW - 0x35e000d8: 523, // nl-NL - 0x35e00115: 524, // nl-SR - 0x35e0011a: 525, // nl-SX - 0x35f00000: 526, // nmg - 0x35f00051: 527, // nmg-CM - 0x36100000: 528, // nn - 0x361000d9: 529, // nn-NO - 0x36300000: 530, // nnh - 0x36300051: 531, // nnh-CM - 0x36600000: 532, // no - 0x36c00000: 533, // nqo - 0x36d00000: 534, // nr - 0x37100000: 535, // nso - 0x37700000: 536, // nus - 0x37700116: 537, // nus-SS - 0x37e00000: 538, // ny - 0x38000000: 539, // nyn - 0x38000130: 540, // nyn-UG - 0x38700000: 541, // om - 0x3870006e: 542, // om-ET - 0x387000a3: 543, // om-KE - 0x38c00000: 544, // or - 0x38c00098: 545, // or-IN - 0x38f00000: 546, // os - 0x38f0007c: 547, // os-GE - 0x38f00105: 548, // os-RU - 0x39400000: 549, // pa - 0x39405000: 550, // pa-Arab - 0x394050e7: 551, // pa-Arab-PK - 0x3942f000: 552, // pa-Guru - 0x3942f098: 553, // pa-Guru-IN - 0x39800000: 554, // pap - 0x3aa00000: 555, // pl - 0x3aa000e8: 556, // pl-PL - 0x3b400000: 557, // prg - 0x3b400001: 558, // prg-001 - 0x3b500000: 559, // ps - 0x3b500023: 560, // ps-AF - 0x3b700000: 561, // pt - 0x3b700029: 562, // pt-AO - 0x3b700040: 563, // pt-BR - 0x3b70004d: 564, // pt-CH - 0x3b700059: 565, // pt-CV - 0x3b700085: 566, // pt-GQ - 0x3b70008a: 567, // pt-GW - 0x3b7000b6: 568, // pt-LU - 0x3b7000c5: 569, // pt-MO - 0x3b7000d0: 570, // pt-MZ - 0x3b7000ed: 571, // pt-PT - 0x3b700117: 572, // pt-ST - 0x3b700125: 573, // pt-TL - 0x3bb00000: 574, // qu - 0x3bb0003e: 575, // qu-BO - 0x3bb00068: 576, // qu-EC - 0x3bb000e3: 577, // qu-PE - 0x3cb00000: 578, // rm - 0x3cb0004d: 579, // rm-CH - 0x3d000000: 580, // rn - 0x3d000039: 581, // rn-BI - 0x3d300000: 582, // ro - 0x3d3000bb: 583, // ro-MD - 0x3d300103: 584, // ro-RO - 0x3d500000: 585, // rof - 0x3d50012e: 586, // rof-TZ - 0x3d900000: 587, // ru - 0x3d900046: 588, // ru-BY - 0x3d9000a4: 589, // ru-KG - 0x3d9000ad: 590, // ru-KZ - 0x3d9000bb: 591, // ru-MD - 0x3d900105: 592, // ru-RU - 0x3d90012f: 593, // ru-UA - 0x3dc00000: 594, // rw - 0x3dc00106: 595, // rw-RW - 0x3dd00000: 596, // rwk - 0x3dd0012e: 597, // rwk-TZ - 0x3e200000: 598, // sah - 0x3e200105: 599, // sah-RU - 0x3e300000: 600, // saq - 0x3e3000a3: 601, // saq-KE - 0x3e900000: 602, // sbp - 0x3e90012e: 603, // sbp-TZ - 0x3f200000: 604, // sdh - 0x3f300000: 605, // se - 0x3f300071: 606, // se-FI - 0x3f3000d9: 607, // se-NO - 0x3f30010b: 608, // se-SE - 0x3f500000: 609, // seh - 0x3f5000d0: 610, // seh-MZ - 0x3f700000: 611, // ses - 0x3f7000c2: 612, // ses-ML - 0x3f800000: 613, // sg - 0x3f80004b: 614, // sg-CF - 0x3fe00000: 615, // shi - 0x3fe52000: 616, // shi-Latn - 0x3fe520b9: 617, // shi-Latn-MA - 0x3fed2000: 618, // shi-Tfng - 0x3fed20b9: 619, // shi-Tfng-MA - 0x40200000: 620, // si - 0x402000b2: 621, // si-LK - 0x40800000: 622, // sk - 0x40800110: 623, // sk-SK - 0x40c00000: 624, // sl - 0x40c0010e: 625, // sl-SI - 0x41200000: 626, // sma - 0x41300000: 627, // smi - 0x41400000: 628, // smj - 0x41500000: 629, // smn - 0x41500071: 630, // smn-FI - 0x41800000: 631, // sms - 0x41900000: 632, // sn - 0x41900163: 633, // sn-ZW - 0x41f00000: 634, // so - 0x41f00061: 635, // so-DJ - 0x41f0006e: 636, // so-ET - 0x41f000a3: 637, // so-KE - 0x41f00114: 638, // so-SO - 0x42700000: 639, // sq - 0x42700026: 640, // sq-AL - 0x427000c1: 641, // sq-MK - 0x4270014c: 642, // sq-XK - 0x42800000: 643, // sr - 0x4281e000: 644, // sr-Cyrl - 0x4281e032: 645, // sr-Cyrl-BA - 0x4281e0bc: 646, // sr-Cyrl-ME - 0x4281e104: 647, // sr-Cyrl-RS - 0x4281e14c: 648, // sr-Cyrl-XK - 0x42852000: 649, // sr-Latn - 0x42852032: 650, // sr-Latn-BA - 0x428520bc: 651, // sr-Latn-ME - 0x42852104: 652, // sr-Latn-RS - 0x4285214c: 653, // sr-Latn-XK - 0x42d00000: 654, // ss - 0x43000000: 655, // ssy - 0x43100000: 656, // st - 0x43a00000: 657, // sv - 0x43a00030: 658, // sv-AX - 0x43a00071: 659, // sv-FI - 0x43a0010b: 660, // sv-SE - 0x43b00000: 661, // sw - 0x43b0004a: 662, // sw-CD - 0x43b000a3: 663, // sw-KE - 0x43b0012e: 664, // sw-TZ - 0x43b00130: 665, // sw-UG - 0x44400000: 666, // syr - 0x44600000: 667, // ta - 0x44600098: 668, // ta-IN - 0x446000b2: 669, // ta-LK - 0x446000cf: 670, // ta-MY - 0x4460010c: 671, // ta-SG - 0x45700000: 672, // te - 0x45700098: 673, // te-IN - 0x45a00000: 674, // teo - 0x45a000a3: 675, // teo-KE - 0x45a00130: 676, // teo-UG - 0x46100000: 677, // th - 0x46100122: 678, // th-TH - 0x46500000: 679, // ti - 0x4650006c: 680, // ti-ER - 0x4650006e: 681, // ti-ET - 0x46700000: 682, // tig - 0x46c00000: 683, // tk - 0x46c00126: 684, // tk-TM - 0x47600000: 685, // tn - 0x47800000: 686, // to - 0x47800128: 687, // to-TO - 0x48000000: 688, // tr - 0x4800005c: 689, // tr-CY - 0x4800012a: 690, // tr-TR - 0x48400000: 691, // ts - 0x49a00000: 692, // twq - 0x49a000d3: 693, // twq-NE - 0x49f00000: 694, // tzm - 0x49f000b9: 695, // tzm-MA - 0x4a200000: 696, // ug - 0x4a200052: 697, // ug-CN - 0x4a400000: 698, // uk - 0x4a40012f: 699, // uk-UA - 0x4aa00000: 700, // ur - 0x4aa00098: 701, // ur-IN - 0x4aa000e7: 702, // ur-PK - 0x4b200000: 703, // uz - 0x4b205000: 704, // uz-Arab - 0x4b205023: 705, // uz-Arab-AF - 0x4b21e000: 706, // uz-Cyrl - 0x4b21e136: 707, // uz-Cyrl-UZ - 0x4b252000: 708, // uz-Latn - 0x4b252136: 709, // uz-Latn-UZ - 0x4b400000: 710, // vai - 0x4b452000: 711, // vai-Latn - 0x4b4520b3: 712, // vai-Latn-LR - 0x4b4d9000: 713, // vai-Vaii - 0x4b4d90b3: 714, // vai-Vaii-LR - 0x4b600000: 715, // ve - 0x4b900000: 716, // vi - 0x4b90013d: 717, // vi-VN - 0x4bf00000: 718, // vo - 0x4bf00001: 719, // vo-001 - 0x4c200000: 720, // vun - 0x4c20012e: 721, // vun-TZ - 0x4c400000: 722, // wa - 0x4c500000: 723, // wae - 0x4c50004d: 724, // wae-CH - 0x4db00000: 725, // wo - 0x4e800000: 726, // xh - 0x4f100000: 727, // xog - 0x4f100130: 728, // xog-UG - 0x4ff00000: 729, // yav - 0x4ff00051: 730, // yav-CM - 0x50800000: 731, // yi - 0x50800001: 732, // yi-001 - 0x50e00000: 733, // yo - 0x50e0003a: 734, // yo-BJ - 0x50e000d5: 735, // yo-NG - 0x51500000: 736, // yue - 0x5150008c: 737, // yue-HK - 0x51e00000: 738, // zgh - 0x51e000b9: 739, // zgh-MA - 0x51f00000: 740, // zh - 0x51f34000: 741, // zh-Hans - 0x51f34052: 742, // zh-Hans-CN - 0x51f3408c: 743, // zh-Hans-HK - 0x51f340c5: 744, // zh-Hans-MO - 0x51f3410c: 745, // zh-Hans-SG - 0x51f35000: 746, // zh-Hant - 0x51f3508c: 747, // zh-Hant-HK - 0x51f350c5: 748, // zh-Hant-MO - 0x51f3512d: 749, // zh-Hant-TW - 0x52400000: 750, // zu - 0x52400160: 751, // zu-ZA + 0x01600000: 3, // af + 0x016000d1: 4, // af-NA + 0x01600160: 5, // af-ZA + 0x01c00000: 6, // agq + 0x01c00051: 7, // agq-CM + 0x02100000: 8, // ak + 0x0210007f: 9, // ak-GH + 0x02700000: 10, // am + 0x0270006e: 11, // am-ET + 0x03a00000: 12, // ar + 0x03a00001: 13, // ar-001 + 0x03a00022: 14, // ar-AE + 0x03a00038: 15, // ar-BH + 0x03a00061: 16, // ar-DJ + 0x03a00066: 17, // ar-DZ + 0x03a0006a: 18, // ar-EG + 0x03a0006b: 19, // ar-EH + 0x03a0006c: 20, // ar-ER + 0x03a00096: 21, // ar-IL + 0x03a0009a: 22, // ar-IQ + 0x03a000a0: 23, // ar-JO + 0x03a000a7: 24, // ar-KM + 0x03a000ab: 25, // ar-KW + 0x03a000af: 26, // ar-LB + 0x03a000b8: 27, // ar-LY + 0x03a000b9: 28, // ar-MA + 0x03a000c8: 29, // ar-MR + 0x03a000e0: 30, // ar-OM + 0x03a000ec: 31, // ar-PS + 0x03a000f2: 32, // ar-QA + 0x03a00107: 33, // ar-SA + 0x03a0010a: 34, // ar-SD + 0x03a00114: 35, // ar-SO + 0x03a00116: 36, // ar-SS + 0x03a0011b: 37, // ar-SY + 0x03a0011f: 38, // ar-TD + 0x03a00127: 39, // ar-TN + 0x03a0015d: 40, // ar-YE + 0x04000000: 41, // ars + 0x04300000: 42, // as + 0x04300098: 43, // as-IN + 0x04400000: 44, // asa + 0x0440012e: 45, // asa-TZ + 0x04800000: 46, // ast + 0x0480006d: 47, // ast-ES + 0x05800000: 48, // az + 0x0581e000: 49, // az-Cyrl + 0x0581e031: 50, // az-Cyrl-AZ + 0x05852000: 51, // az-Latn + 0x05852031: 52, // az-Latn-AZ + 0x05e00000: 53, // bas + 0x05e00051: 54, // bas-CM + 0x07100000: 55, // be + 0x07100046: 56, // be-BY + 0x07500000: 57, // bem + 0x07500161: 58, // bem-ZM + 0x07900000: 59, // bez + 0x0790012e: 60, // bez-TZ + 0x07e00000: 61, // bg + 0x07e00037: 62, // bg-BG + 0x08200000: 63, // bh + 0x0a000000: 64, // bm + 0x0a0000c2: 65, // bm-ML + 0x0a500000: 66, // bn + 0x0a500034: 67, // bn-BD + 0x0a500098: 68, // bn-IN + 0x0a900000: 69, // bo + 0x0a900052: 70, // bo-CN + 0x0a900098: 71, // bo-IN + 0x0b200000: 72, // br + 0x0b200077: 73, // br-FR + 0x0b500000: 74, // brx + 0x0b500098: 75, // brx-IN + 0x0b700000: 76, // bs + 0x0b71e000: 77, // bs-Cyrl + 0x0b71e032: 78, // bs-Cyrl-BA + 0x0b752000: 79, // bs-Latn + 0x0b752032: 80, // bs-Latn-BA + 0x0d700000: 81, // ca + 0x0d700021: 82, // ca-AD + 0x0d70006d: 83, // ca-ES + 0x0d700077: 84, // ca-FR + 0x0d70009d: 85, // ca-IT + 0x0dc00000: 86, // ce + 0x0dc00105: 87, // ce-RU + 0x0df00000: 88, // cgg + 0x0df00130: 89, // cgg-UG + 0x0e500000: 90, // chr + 0x0e500134: 91, // chr-US + 0x0e900000: 92, // ckb + 0x0e90009a: 93, // ckb-IQ + 0x0e90009b: 94, // ckb-IR + 0x0f900000: 95, // cs + 0x0f90005d: 96, // cs-CZ + 0x0fd00000: 97, // cu + 0x0fd00105: 98, // cu-RU + 0x0ff00000: 99, // cy + 0x0ff0007a: 100, // cy-GB + 0x10000000: 101, // da + 0x10000062: 102, // da-DK + 0x10000081: 103, // da-GL + 0x10700000: 104, // dav + 0x107000a3: 105, // dav-KE + 0x10c00000: 106, // de + 0x10c0002d: 107, // de-AT + 0x10c00035: 108, // de-BE + 0x10c0004d: 109, // de-CH + 0x10c0005f: 110, // de-DE + 0x10c0009d: 111, // de-IT + 0x10c000b1: 112, // de-LI + 0x10c000b6: 113, // de-LU + 0x11600000: 114, // dje + 0x116000d3: 115, // dje-NE + 0x11e00000: 116, // dsb + 0x11e0005f: 117, // dsb-DE + 0x12300000: 118, // dua + 0x12300051: 119, // dua-CM + 0x12700000: 120, // dv + 0x12a00000: 121, // dyo + 0x12a00113: 122, // dyo-SN + 0x12c00000: 123, // dz + 0x12c00042: 124, // dz-BT + 0x12e00000: 125, // ebu + 0x12e000a3: 126, // ebu-KE + 0x12f00000: 127, // ee + 0x12f0007f: 128, // ee-GH + 0x12f00121: 129, // ee-TG + 0x13500000: 130, // el + 0x1350005c: 131, // el-CY + 0x13500086: 132, // el-GR + 0x13800000: 133, // en + 0x13800001: 134, // en-001 + 0x1380001a: 135, // en-150 + 0x13800024: 136, // en-AG + 0x13800025: 137, // en-AI + 0x1380002c: 138, // en-AS + 0x1380002d: 139, // en-AT + 0x1380002e: 140, // en-AU + 0x13800033: 141, // en-BB + 0x13800035: 142, // en-BE + 0x13800039: 143, // en-BI + 0x1380003c: 144, // en-BM + 0x13800041: 145, // en-BS + 0x13800045: 146, // en-BW + 0x13800047: 147, // en-BZ + 0x13800048: 148, // en-CA + 0x13800049: 149, // en-CC + 0x1380004d: 150, // en-CH + 0x1380004f: 151, // en-CK + 0x13800051: 152, // en-CM + 0x1380005b: 153, // en-CX + 0x1380005c: 154, // en-CY + 0x1380005f: 155, // en-DE + 0x13800060: 156, // en-DG + 0x13800062: 157, // en-DK + 0x13800063: 158, // en-DM + 0x1380006c: 159, // en-ER + 0x13800071: 160, // en-FI + 0x13800072: 161, // en-FJ + 0x13800073: 162, // en-FK + 0x13800074: 163, // en-FM + 0x1380007a: 164, // en-GB + 0x1380007b: 165, // en-GD + 0x1380007e: 166, // en-GG + 0x1380007f: 167, // en-GH + 0x13800080: 168, // en-GI + 0x13800082: 169, // en-GM + 0x13800089: 170, // en-GU + 0x1380008b: 171, // en-GY + 0x1380008c: 172, // en-HK + 0x13800095: 173, // en-IE + 0x13800096: 174, // en-IL + 0x13800097: 175, // en-IM + 0x13800098: 176, // en-IN + 0x13800099: 177, // en-IO + 0x1380009e: 178, // en-JE + 0x1380009f: 179, // en-JM + 0x138000a3: 180, // en-KE + 0x138000a6: 181, // en-KI + 0x138000a8: 182, // en-KN + 0x138000ac: 183, // en-KY + 0x138000b0: 184, // en-LC + 0x138000b3: 185, // en-LR + 0x138000b4: 186, // en-LS + 0x138000be: 187, // en-MG + 0x138000bf: 188, // en-MH + 0x138000c5: 189, // en-MO + 0x138000c6: 190, // en-MP + 0x138000c9: 191, // en-MS + 0x138000ca: 192, // en-MT + 0x138000cb: 193, // en-MU + 0x138000cd: 194, // en-MW + 0x138000cf: 195, // en-MY + 0x138000d1: 196, // en-NA + 0x138000d4: 197, // en-NF + 0x138000d5: 198, // en-NG + 0x138000d8: 199, // en-NL + 0x138000dc: 200, // en-NR + 0x138000de: 201, // en-NU + 0x138000df: 202, // en-NZ + 0x138000e5: 203, // en-PG + 0x138000e6: 204, // en-PH + 0x138000e7: 205, // en-PK + 0x138000ea: 206, // en-PN + 0x138000eb: 207, // en-PR + 0x138000ef: 208, // en-PW + 0x13800106: 209, // en-RW + 0x13800108: 210, // en-SB + 0x13800109: 211, // en-SC + 0x1380010a: 212, // en-SD + 0x1380010b: 213, // en-SE + 0x1380010c: 214, // en-SG + 0x1380010d: 215, // en-SH + 0x1380010e: 216, // en-SI + 0x13800111: 217, // en-SL + 0x13800116: 218, // en-SS + 0x1380011a: 219, // en-SX + 0x1380011c: 220, // en-SZ + 0x1380011e: 221, // en-TC + 0x13800124: 222, // en-TK + 0x13800128: 223, // en-TO + 0x1380012b: 224, // en-TT + 0x1380012c: 225, // en-TV + 0x1380012e: 226, // en-TZ + 0x13800130: 227, // en-UG + 0x13800132: 228, // en-UM + 0x13800134: 229, // en-US + 0x13800138: 230, // en-VC + 0x1380013b: 231, // en-VG + 0x1380013c: 232, // en-VI + 0x1380013e: 233, // en-VU + 0x13800141: 234, // en-WS + 0x13800160: 235, // en-ZA + 0x13800161: 236, // en-ZM + 0x13800163: 237, // en-ZW + 0x13b00000: 238, // eo + 0x13b00001: 239, // eo-001 + 0x13d00000: 240, // es + 0x13d0001e: 241, // es-419 + 0x13d0002b: 242, // es-AR + 0x13d0003e: 243, // es-BO + 0x13d00040: 244, // es-BR + 0x13d00047: 245, // es-BZ + 0x13d00050: 246, // es-CL + 0x13d00053: 247, // es-CO + 0x13d00055: 248, // es-CR + 0x13d00058: 249, // es-CU + 0x13d00064: 250, // es-DO + 0x13d00067: 251, // es-EA + 0x13d00068: 252, // es-EC + 0x13d0006d: 253, // es-ES + 0x13d00085: 254, // es-GQ + 0x13d00088: 255, // es-GT + 0x13d0008e: 256, // es-HN + 0x13d00093: 257, // es-IC + 0x13d000ce: 258, // es-MX + 0x13d000d7: 259, // es-NI + 0x13d000e1: 260, // es-PA + 0x13d000e3: 261, // es-PE + 0x13d000e6: 262, // es-PH + 0x13d000eb: 263, // es-PR + 0x13d000f0: 264, // es-PY + 0x13d00119: 265, // es-SV + 0x13d00134: 266, // es-US + 0x13d00135: 267, // es-UY + 0x13d0013a: 268, // es-VE + 0x13f00000: 269, // et + 0x13f00069: 270, // et-EE + 0x14400000: 271, // eu + 0x1440006d: 272, // eu-ES + 0x14500000: 273, // ewo + 0x14500051: 274, // ewo-CM + 0x14700000: 275, // fa + 0x14700023: 276, // fa-AF + 0x1470009b: 277, // fa-IR + 0x14d00000: 278, // ff + 0x14d00051: 279, // ff-CM + 0x14d00083: 280, // ff-GN + 0x14d000c8: 281, // ff-MR + 0x14d00113: 282, // ff-SN + 0x15000000: 283, // fi + 0x15000071: 284, // fi-FI + 0x15200000: 285, // fil + 0x152000e6: 286, // fil-PH + 0x15700000: 287, // fo + 0x15700062: 288, // fo-DK + 0x15700075: 289, // fo-FO + 0x15d00000: 290, // fr + 0x15d00035: 291, // fr-BE + 0x15d00036: 292, // fr-BF + 0x15d00039: 293, // fr-BI + 0x15d0003a: 294, // fr-BJ + 0x15d0003b: 295, // fr-BL + 0x15d00048: 296, // fr-CA + 0x15d0004a: 297, // fr-CD + 0x15d0004b: 298, // fr-CF + 0x15d0004c: 299, // fr-CG + 0x15d0004d: 300, // fr-CH + 0x15d0004e: 301, // fr-CI + 0x15d00051: 302, // fr-CM + 0x15d00061: 303, // fr-DJ + 0x15d00066: 304, // fr-DZ + 0x15d00077: 305, // fr-FR + 0x15d00079: 306, // fr-GA + 0x15d0007d: 307, // fr-GF + 0x15d00083: 308, // fr-GN + 0x15d00084: 309, // fr-GP + 0x15d00085: 310, // fr-GQ + 0x15d00090: 311, // fr-HT + 0x15d000a7: 312, // fr-KM + 0x15d000b6: 313, // fr-LU + 0x15d000b9: 314, // fr-MA + 0x15d000ba: 315, // fr-MC + 0x15d000bd: 316, // fr-MF + 0x15d000be: 317, // fr-MG + 0x15d000c2: 318, // fr-ML + 0x15d000c7: 319, // fr-MQ + 0x15d000c8: 320, // fr-MR + 0x15d000cb: 321, // fr-MU + 0x15d000d2: 322, // fr-NC + 0x15d000d3: 323, // fr-NE + 0x15d000e4: 324, // fr-PF + 0x15d000e9: 325, // fr-PM + 0x15d00101: 326, // fr-RE + 0x15d00106: 327, // fr-RW + 0x15d00109: 328, // fr-SC + 0x15d00113: 329, // fr-SN + 0x15d0011b: 330, // fr-SY + 0x15d0011f: 331, // fr-TD + 0x15d00121: 332, // fr-TG + 0x15d00127: 333, // fr-TN + 0x15d0013e: 334, // fr-VU + 0x15d0013f: 335, // fr-WF + 0x15d0015e: 336, // fr-YT + 0x16800000: 337, // fur + 0x1680009d: 338, // fur-IT + 0x16c00000: 339, // fy + 0x16c000d8: 340, // fy-NL + 0x16d00000: 341, // ga + 0x16d00095: 342, // ga-IE + 0x17c00000: 343, // gd + 0x17c0007a: 344, // gd-GB + 0x18e00000: 345, // gl + 0x18e0006d: 346, // gl-ES + 0x1a100000: 347, // gsw + 0x1a10004d: 348, // gsw-CH + 0x1a100077: 349, // gsw-FR + 0x1a1000b1: 350, // gsw-LI + 0x1a200000: 351, // gu + 0x1a200098: 352, // gu-IN + 0x1a700000: 353, // guw + 0x1a900000: 354, // guz + 0x1a9000a3: 355, // guz-KE + 0x1aa00000: 356, // gv + 0x1aa00097: 357, // gv-IM + 0x1b200000: 358, // ha + 0x1b20007f: 359, // ha-GH + 0x1b2000d3: 360, // ha-NE + 0x1b2000d5: 361, // ha-NG + 0x1b600000: 362, // haw + 0x1b600134: 363, // haw-US + 0x1ba00000: 364, // he + 0x1ba00096: 365, // he-IL + 0x1bc00000: 366, // hi + 0x1bc00098: 367, // hi-IN + 0x1cf00000: 368, // hr + 0x1cf00032: 369, // hr-BA + 0x1cf0008f: 370, // hr-HR + 0x1d000000: 371, // hsb + 0x1d00005f: 372, // hsb-DE + 0x1d300000: 373, // hu + 0x1d300091: 374, // hu-HU + 0x1d500000: 375, // hy + 0x1d500027: 376, // hy-AM + 0x1df00000: 377, // id + 0x1df00094: 378, // id-ID + 0x1e500000: 379, // ig + 0x1e5000d5: 380, // ig-NG + 0x1e800000: 381, // ii + 0x1e800052: 382, // ii-CN + 0x1f600000: 383, // is + 0x1f60009c: 384, // is-IS + 0x1f700000: 385, // it + 0x1f70004d: 386, // it-CH + 0x1f70009d: 387, // it-IT + 0x1f700112: 388, // it-SM + 0x1f700137: 389, // it-VA + 0x1f800000: 390, // iu + 0x1fe00000: 391, // ja + 0x1fe000a1: 392, // ja-JP + 0x20100000: 393, // jbo + 0x20500000: 394, // jgo + 0x20500051: 395, // jgo-CM + 0x20800000: 396, // jmc + 0x2080012e: 397, // jmc-TZ + 0x20c00000: 398, // jv + 0x20e00000: 399, // ka + 0x20e0007c: 400, // ka-GE + 0x21000000: 401, // kab + 0x21000066: 402, // kab-DZ + 0x21400000: 403, // kaj + 0x21500000: 404, // kam + 0x215000a3: 405, // kam-KE + 0x21d00000: 406, // kcg + 0x22100000: 407, // kde + 0x2210012e: 408, // kde-TZ + 0x22500000: 409, // kea + 0x22500059: 410, // kea-CV + 0x23200000: 411, // khq + 0x232000c2: 412, // khq-ML + 0x23700000: 413, // ki + 0x237000a3: 414, // ki-KE + 0x24000000: 415, // kk + 0x240000ad: 416, // kk-KZ + 0x24200000: 417, // kkj + 0x24200051: 418, // kkj-CM + 0x24300000: 419, // kl + 0x24300081: 420, // kl-GL + 0x24400000: 421, // kln + 0x244000a3: 422, // kln-KE + 0x24800000: 423, // km + 0x248000a5: 424, // km-KH + 0x24f00000: 425, // kn + 0x24f00098: 426, // kn-IN + 0x25200000: 427, // ko + 0x252000a9: 428, // ko-KP + 0x252000aa: 429, // ko-KR + 0x25400000: 430, // kok + 0x25400098: 431, // kok-IN + 0x26800000: 432, // ks + 0x26800098: 433, // ks-IN + 0x26900000: 434, // ksb + 0x2690012e: 435, // ksb-TZ + 0x26b00000: 436, // ksf + 0x26b00051: 437, // ksf-CM + 0x26c00000: 438, // ksh + 0x26c0005f: 439, // ksh-DE + 0x27200000: 440, // ku + 0x27f00000: 441, // kw + 0x27f0007a: 442, // kw-GB + 0x28800000: 443, // ky + 0x288000a4: 444, // ky-KG + 0x28f00000: 445, // lag + 0x28f0012e: 446, // lag-TZ + 0x29300000: 447, // lb + 0x293000b6: 448, // lb-LU + 0x2a100000: 449, // lg + 0x2a100130: 450, // lg-UG + 0x2ad00000: 451, // lkt + 0x2ad00134: 452, // lkt-US + 0x2b300000: 453, // ln + 0x2b300029: 454, // ln-AO + 0x2b30004a: 455, // ln-CD + 0x2b30004b: 456, // ln-CF + 0x2b30004c: 457, // ln-CG + 0x2b600000: 458, // lo + 0x2b6000ae: 459, // lo-LA + 0x2bd00000: 460, // lrc + 0x2bd0009a: 461, // lrc-IQ + 0x2bd0009b: 462, // lrc-IR + 0x2be00000: 463, // lt + 0x2be000b5: 464, // lt-LT + 0x2c000000: 465, // lu + 0x2c00004a: 466, // lu-CD + 0x2c200000: 467, // luo + 0x2c2000a3: 468, // luo-KE + 0x2c300000: 469, // luy + 0x2c3000a3: 470, // luy-KE + 0x2c500000: 471, // lv + 0x2c5000b7: 472, // lv-LV + 0x2cf00000: 473, // mas + 0x2cf000a3: 474, // mas-KE + 0x2cf0012e: 475, // mas-TZ + 0x2e700000: 476, // mer + 0x2e7000a3: 477, // mer-KE + 0x2eb00000: 478, // mfe + 0x2eb000cb: 479, // mfe-MU + 0x2ef00000: 480, // mg + 0x2ef000be: 481, // mg-MG + 0x2f000000: 482, // mgh + 0x2f0000d0: 483, // mgh-MZ + 0x2f200000: 484, // mgo + 0x2f200051: 485, // mgo-CM + 0x2fd00000: 486, // mk + 0x2fd000c1: 487, // mk-MK + 0x30200000: 488, // ml + 0x30200098: 489, // ml-IN + 0x30900000: 490, // mn + 0x309000c4: 491, // mn-MN + 0x31900000: 492, // mr + 0x31900098: 493, // mr-IN + 0x31d00000: 494, // ms + 0x31d0003d: 495, // ms-BN + 0x31d000cf: 496, // ms-MY + 0x31d0010c: 497, // ms-SG + 0x31e00000: 498, // mt + 0x31e000ca: 499, // mt-MT + 0x32300000: 500, // mua + 0x32300051: 501, // mua-CM + 0x32f00000: 502, // my + 0x32f000c3: 503, // my-MM + 0x33800000: 504, // mzn + 0x3380009b: 505, // mzn-IR + 0x33f00000: 506, // nah + 0x34300000: 507, // naq + 0x343000d1: 508, // naq-NA + 0x34500000: 509, // nb + 0x345000d9: 510, // nb-NO + 0x3450010f: 511, // nb-SJ + 0x34c00000: 512, // nd + 0x34c00163: 513, // nd-ZW + 0x34e00000: 514, // nds + 0x34e0005f: 515, // nds-DE + 0x34e000d8: 516, // nds-NL + 0x34f00000: 517, // ne + 0x34f00098: 518, // ne-IN + 0x34f000da: 519, // ne-NP + 0x36500000: 520, // nl + 0x3650002f: 521, // nl-AW + 0x36500035: 522, // nl-BE + 0x3650003f: 523, // nl-BQ + 0x3650005a: 524, // nl-CW + 0x365000d8: 525, // nl-NL + 0x36500115: 526, // nl-SR + 0x3650011a: 527, // nl-SX + 0x36600000: 528, // nmg + 0x36600051: 529, // nmg-CM + 0x36800000: 530, // nn + 0x368000d9: 531, // nn-NO + 0x36a00000: 532, // nnh + 0x36a00051: 533, // nnh-CM + 0x36d00000: 534, // no + 0x37300000: 535, // nqo + 0x37400000: 536, // nr + 0x37800000: 537, // nso + 0x37e00000: 538, // nus + 0x37e00116: 539, // nus-SS + 0x38500000: 540, // ny + 0x38700000: 541, // nyn + 0x38700130: 542, // nyn-UG + 0x38e00000: 543, // om + 0x38e0006e: 544, // om-ET + 0x38e000a3: 545, // om-KE + 0x39300000: 546, // or + 0x39300098: 547, // or-IN + 0x39600000: 548, // os + 0x3960007c: 549, // os-GE + 0x39600105: 550, // os-RU + 0x39b00000: 551, // pa + 0x39b05000: 552, // pa-Arab + 0x39b050e7: 553, // pa-Arab-PK + 0x39b2f000: 554, // pa-Guru + 0x39b2f098: 555, // pa-Guru-IN + 0x39f00000: 556, // pap + 0x3b100000: 557, // pl + 0x3b1000e8: 558, // pl-PL + 0x3bb00000: 559, // prg + 0x3bb00001: 560, // prg-001 + 0x3bc00000: 561, // ps + 0x3bc00023: 562, // ps-AF + 0x3be00000: 563, // pt + 0x3be00029: 564, // pt-AO + 0x3be00040: 565, // pt-BR + 0x3be0004d: 566, // pt-CH + 0x3be00059: 567, // pt-CV + 0x3be00085: 568, // pt-GQ + 0x3be0008a: 569, // pt-GW + 0x3be000b6: 570, // pt-LU + 0x3be000c5: 571, // pt-MO + 0x3be000d0: 572, // pt-MZ + 0x3be000ed: 573, // pt-PT + 0x3be00117: 574, // pt-ST + 0x3be00125: 575, // pt-TL + 0x3c200000: 576, // qu + 0x3c20003e: 577, // qu-BO + 0x3c200068: 578, // qu-EC + 0x3c2000e3: 579, // qu-PE + 0x3d200000: 580, // rm + 0x3d20004d: 581, // rm-CH + 0x3d700000: 582, // rn + 0x3d700039: 583, // rn-BI + 0x3da00000: 584, // ro + 0x3da000bb: 585, // ro-MD + 0x3da00103: 586, // ro-RO + 0x3dc00000: 587, // rof + 0x3dc0012e: 588, // rof-TZ + 0x3e000000: 589, // ru + 0x3e000046: 590, // ru-BY + 0x3e0000a4: 591, // ru-KG + 0x3e0000ad: 592, // ru-KZ + 0x3e0000bb: 593, // ru-MD + 0x3e000105: 594, // ru-RU + 0x3e00012f: 595, // ru-UA + 0x3e300000: 596, // rw + 0x3e300106: 597, // rw-RW + 0x3e400000: 598, // rwk + 0x3e40012e: 599, // rwk-TZ + 0x3e900000: 600, // sah + 0x3e900105: 601, // sah-RU + 0x3ea00000: 602, // saq + 0x3ea000a3: 603, // saq-KE + 0x3f100000: 604, // sbp + 0x3f10012e: 605, // sbp-TZ + 0x3fa00000: 606, // sdh + 0x3fb00000: 607, // se + 0x3fb00071: 608, // se-FI + 0x3fb000d9: 609, // se-NO + 0x3fb0010b: 610, // se-SE + 0x3fd00000: 611, // seh + 0x3fd000d0: 612, // seh-MZ + 0x3ff00000: 613, // ses + 0x3ff000c2: 614, // ses-ML + 0x40000000: 615, // sg + 0x4000004b: 616, // sg-CF + 0x40600000: 617, // shi + 0x40652000: 618, // shi-Latn + 0x406520b9: 619, // shi-Latn-MA + 0x406d2000: 620, // shi-Tfng + 0x406d20b9: 621, // shi-Tfng-MA + 0x40a00000: 622, // si + 0x40a000b2: 623, // si-LK + 0x41000000: 624, // sk + 0x41000110: 625, // sk-SK + 0x41400000: 626, // sl + 0x4140010e: 627, // sl-SI + 0x41a00000: 628, // sma + 0x41b00000: 629, // smi + 0x41c00000: 630, // smj + 0x41d00000: 631, // smn + 0x41d00071: 632, // smn-FI + 0x42000000: 633, // sms + 0x42100000: 634, // sn + 0x42100163: 635, // sn-ZW + 0x42700000: 636, // so + 0x42700061: 637, // so-DJ + 0x4270006e: 638, // so-ET + 0x427000a3: 639, // so-KE + 0x42700114: 640, // so-SO + 0x42f00000: 641, // sq + 0x42f00026: 642, // sq-AL + 0x42f000c1: 643, // sq-MK + 0x42f0014c: 644, // sq-XK + 0x43000000: 645, // sr + 0x4301e000: 646, // sr-Cyrl + 0x4301e032: 647, // sr-Cyrl-BA + 0x4301e0bc: 648, // sr-Cyrl-ME + 0x4301e104: 649, // sr-Cyrl-RS + 0x4301e14c: 650, // sr-Cyrl-XK + 0x43052000: 651, // sr-Latn + 0x43052032: 652, // sr-Latn-BA + 0x430520bc: 653, // sr-Latn-ME + 0x43052104: 654, // sr-Latn-RS + 0x4305214c: 655, // sr-Latn-XK + 0x43500000: 656, // ss + 0x43800000: 657, // ssy + 0x43900000: 658, // st + 0x44200000: 659, // sv + 0x44200030: 660, // sv-AX + 0x44200071: 661, // sv-FI + 0x4420010b: 662, // sv-SE + 0x44300000: 663, // sw + 0x4430004a: 664, // sw-CD + 0x443000a3: 665, // sw-KE + 0x4430012e: 666, // sw-TZ + 0x44300130: 667, // sw-UG + 0x44c00000: 668, // syr + 0x44e00000: 669, // ta + 0x44e00098: 670, // ta-IN + 0x44e000b2: 671, // ta-LK + 0x44e000cf: 672, // ta-MY + 0x44e0010c: 673, // ta-SG + 0x45f00000: 674, // te + 0x45f00098: 675, // te-IN + 0x46200000: 676, // teo + 0x462000a3: 677, // teo-KE + 0x46200130: 678, // teo-UG + 0x46900000: 679, // th + 0x46900122: 680, // th-TH + 0x46d00000: 681, // ti + 0x46d0006c: 682, // ti-ER + 0x46d0006e: 683, // ti-ET + 0x46f00000: 684, // tig + 0x47400000: 685, // tk + 0x47400126: 686, // tk-TM + 0x47e00000: 687, // tn + 0x48000000: 688, // to + 0x48000128: 689, // to-TO + 0x48800000: 690, // tr + 0x4880005c: 691, // tr-CY + 0x4880012a: 692, // tr-TR + 0x48c00000: 693, // ts + 0x4a200000: 694, // twq + 0x4a2000d3: 695, // twq-NE + 0x4a700000: 696, // tzm + 0x4a7000b9: 697, // tzm-MA + 0x4aa00000: 698, // ug + 0x4aa00052: 699, // ug-CN + 0x4ac00000: 700, // uk + 0x4ac0012f: 701, // uk-UA + 0x4b200000: 702, // ur + 0x4b200098: 703, // ur-IN + 0x4b2000e7: 704, // ur-PK + 0x4ba00000: 705, // uz + 0x4ba05000: 706, // uz-Arab + 0x4ba05023: 707, // uz-Arab-AF + 0x4ba1e000: 708, // uz-Cyrl + 0x4ba1e136: 709, // uz-Cyrl-UZ + 0x4ba52000: 710, // uz-Latn + 0x4ba52136: 711, // uz-Latn-UZ + 0x4bc00000: 712, // vai + 0x4bc52000: 713, // vai-Latn + 0x4bc520b3: 714, // vai-Latn-LR + 0x4bcd9000: 715, // vai-Vaii + 0x4bcd90b3: 716, // vai-Vaii-LR + 0x4be00000: 717, // ve + 0x4c100000: 718, // vi + 0x4c10013d: 719, // vi-VN + 0x4c700000: 720, // vo + 0x4c700001: 721, // vo-001 + 0x4ca00000: 722, // vun + 0x4ca0012e: 723, // vun-TZ + 0x4cc00000: 724, // wa + 0x4cd00000: 725, // wae + 0x4cd0004d: 726, // wae-CH + 0x4e300000: 727, // wo + 0x4f000000: 728, // xh + 0x4f900000: 729, // xog + 0x4f900130: 730, // xog-UG + 0x50700000: 731, // yav + 0x50700051: 732, // yav-CM + 0x51000000: 733, // yi + 0x51000001: 734, // yi-001 + 0x51600000: 735, // yo + 0x5160003a: 736, // yo-BJ + 0x516000d5: 737, // yo-NG + 0x51d00000: 738, // yue + 0x51d0008c: 739, // yue-HK + 0x52600000: 740, // zgh + 0x526000b9: 741, // zgh-MA + 0x52700000: 742, // zh + 0x52734000: 743, // zh-Hans + 0x52734052: 744, // zh-Hans-CN + 0x5273408c: 745, // zh-Hans-HK + 0x527340c5: 746, // zh-Hans-MO + 0x5273410c: 747, // zh-Hans-SG + 0x52735000: 748, // zh-Hant + 0x5273508c: 749, // zh-Hant-HK + 0x527350c5: 750, // zh-Hant-MO + 0x5273512d: 751, // zh-Hant-TW + 0x52c00000: 752, // zu + 0x52c00160: 753, // zu-ZA } -// Total table size 4580 bytes (4KiB); checksum: A7F72A2A +// Total table size 4592 bytes (4KiB); checksum: C25F8AFF diff --git a/vendor/golang.org/x/text/language/maketables.go b/vendor/golang.org/x/text/language/maketables.go deleted file mode 100644 index 107f99254..000000000 --- a/vendor/golang.org/x/text/language/maketables.go +++ /dev/null @@ -1,1648 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -// Language tag table generator. -// Data read from the web. - -package main - -import ( - "bufio" - "flag" - "fmt" - "io" - "io/ioutil" - "log" - "math" - "reflect" - "regexp" - "sort" - "strconv" - "strings" - - "golang.org/x/text/internal/gen" - "golang.org/x/text/internal/tag" - "golang.org/x/text/unicode/cldr" -) - -var ( - test = flag.Bool("test", - false, - "test existing tables; can be used to compare web data with package data.") - outputFile = flag.String("output", - "tables.go", - "output file for generated tables") -) - -var comment = []string{ - ` -lang holds an alphabetically sorted list of ISO-639 language identifiers. -All entries are 4 bytes. The index of the identifier (divided by 4) is the language tag. -For 2-byte language identifiers, the two successive bytes have the following meaning: - - if the first letter of the 2- and 3-letter ISO codes are the same: - the second and third letter of the 3-letter ISO code. - - otherwise: a 0 and a by 2 bits right-shifted index into altLangISO3. -For 3-byte language identifiers the 4th byte is 0.`, - ` -langNoIndex is a bit vector of all 3-letter language codes that are not used as an index -in lookup tables. The language ids for these language codes are derived directly -from the letters and are not consecutive.`, - ` -altLangISO3 holds an alphabetically sorted list of 3-letter language code alternatives -to 2-letter language codes that cannot be derived using the method described above. -Each 3-letter code is followed by its 1-byte langID.`, - ` -altLangIndex is used to convert indexes in altLangISO3 to langIDs.`, - ` -langAliasMap maps langIDs to their suggested replacements.`, - ` -script is an alphabetically sorted list of ISO 15924 codes. The index -of the script in the string, divided by 4, is the internal scriptID.`, - ` -isoRegionOffset needs to be added to the index of regionISO to obtain the regionID -for 2-letter ISO codes. (The first isoRegionOffset regionIDs are reserved for -the UN.M49 codes used for groups.)`, - ` -regionISO holds a list of alphabetically sorted 2-letter ISO region codes. -Each 2-letter codes is followed by two bytes with the following meaning: - - [A-Z}{2}: the first letter of the 2-letter code plus these two - letters form the 3-letter ISO code. - - 0, n: index into altRegionISO3.`, - ` -regionTypes defines the status of a region for various standards.`, - ` -m49 maps regionIDs to UN.M49 codes. The first isoRegionOffset entries are -codes indicating collections of regions.`, - ` -m49Index gives indexes into fromM49 based on the three most significant bits -of a 10-bit UN.M49 code. To search an UN.M49 code in fromM49, search in - fromM49[m49Index[msb39(code)]:m49Index[msb3(code)+1]] -for an entry where the first 7 bits match the 7 lsb of the UN.M49 code. -The region code is stored in the 9 lsb of the indexed value.`, - ` -fromM49 contains entries to map UN.M49 codes to regions. See m49Index for details.`, - ` -altRegionISO3 holds a list of 3-letter region codes that cannot be -mapped to 2-letter codes using the default algorithm. This is a short list.`, - ` -altRegionIDs holds a list of regionIDs the positions of which match those -of the 3-letter ISO codes in altRegionISO3.`, - ` -variantNumSpecialized is the number of specialized variants in variants.`, - ` -suppressScript is an index from langID to the dominant script for that language, -if it exists. If a script is given, it should be suppressed from the language tag.`, - ` -likelyLang is a lookup table, indexed by langID, for the most likely -scripts and regions given incomplete information. If more entries exist for a -given language, region and script are the index and size respectively -of the list in likelyLangList.`, - ` -likelyLangList holds lists info associated with likelyLang.`, - ` -likelyRegion is a lookup table, indexed by regionID, for the most likely -languages and scripts given incomplete information. If more entries exist -for a given regionID, lang and script are the index and size respectively -of the list in likelyRegionList. -TODO: exclude containers and user-definable regions from the list.`, - ` -likelyRegionList holds lists info associated with likelyRegion.`, - ` -likelyScript is a lookup table, indexed by scriptID, for the most likely -languages and regions given a script.`, - ` -matchLang holds pairs of langIDs of base languages that are typically -mutually intelligible. Each pair is associated with a confidence and -whether the intelligibility goes one or both ways.`, - ` -matchScript holds pairs of scriptIDs where readers of one script -can typically also read the other. Each is associated with a confidence.`, - ` -nRegionGroups is the number of region groups.`, - ` -regionInclusion maps region identifiers to sets of regions in regionInclusionBits, -where each set holds all groupings that are directly connected in a region -containment graph.`, - ` -regionInclusionBits is an array of bit vectors where every vector represents -a set of region groupings. These sets are used to compute the distance -between two regions for the purpose of language matching.`, - ` -regionInclusionNext marks, for each entry in regionInclusionBits, the set of -all groups that are reachable from the groups set in the respective entry.`, -} - -// TODO: consider changing some of these structures to tries. This can reduce -// memory, but may increase the need for memory allocations. This could be -// mitigated if we can piggyback on language tags for common cases. - -func failOnError(e error) { - if e != nil { - log.Panic(e) - } -} - -type setType int - -const ( - Indexed setType = 1 + iota // all elements must be of same size - Linear -) - -type stringSet struct { - s []string - sorted, frozen bool - - // We often need to update values after the creation of an index is completed. - // We include a convenience map for keeping track of this. - update map[string]string - typ setType // used for checking. -} - -func (ss *stringSet) clone() stringSet { - c := *ss - c.s = append([]string(nil), c.s...) - return c -} - -func (ss *stringSet) setType(t setType) { - if ss.typ != t && ss.typ != 0 { - log.Panicf("type %d cannot be assigned as it was already %d", t, ss.typ) - } -} - -// parse parses a whitespace-separated string and initializes ss with its -// components. -func (ss *stringSet) parse(s string) { - scan := bufio.NewScanner(strings.NewReader(s)) - scan.Split(bufio.ScanWords) - for scan.Scan() { - ss.add(scan.Text()) - } -} - -func (ss *stringSet) assertChangeable() { - if ss.frozen { - log.Panic("attempt to modify a frozen stringSet") - } -} - -func (ss *stringSet) add(s string) { - ss.assertChangeable() - ss.s = append(ss.s, s) - ss.sorted = ss.frozen -} - -func (ss *stringSet) freeze() { - ss.compact() - ss.frozen = true -} - -func (ss *stringSet) compact() { - if ss.sorted { - return - } - a := ss.s - sort.Strings(a) - k := 0 - for i := 1; i < len(a); i++ { - if a[k] != a[i] { - a[k+1] = a[i] - k++ - } - } - ss.s = a[:k+1] - ss.sorted = ss.frozen -} - -type funcSorter struct { - fn func(a, b string) bool - sort.StringSlice -} - -func (s funcSorter) Less(i, j int) bool { - return s.fn(s.StringSlice[i], s.StringSlice[j]) -} - -func (ss *stringSet) sortFunc(f func(a, b string) bool) { - ss.compact() - sort.Sort(funcSorter{f, sort.StringSlice(ss.s)}) -} - -func (ss *stringSet) remove(s string) { - ss.assertChangeable() - if i, ok := ss.find(s); ok { - copy(ss.s[i:], ss.s[i+1:]) - ss.s = ss.s[:len(ss.s)-1] - } -} - -func (ss *stringSet) replace(ol, nu string) { - ss.s[ss.index(ol)] = nu - ss.sorted = ss.frozen -} - -func (ss *stringSet) index(s string) int { - ss.setType(Indexed) - i, ok := ss.find(s) - if !ok { - if i < len(ss.s) { - log.Panicf("find: item %q is not in list. Closest match is %q.", s, ss.s[i]) - } - log.Panicf("find: item %q is not in list", s) - - } - return i -} - -func (ss *stringSet) find(s string) (int, bool) { - ss.compact() - i := sort.SearchStrings(ss.s, s) - return i, i != len(ss.s) && ss.s[i] == s -} - -func (ss *stringSet) slice() []string { - ss.compact() - return ss.s -} - -func (ss *stringSet) updateLater(v, key string) { - if ss.update == nil { - ss.update = map[string]string{} - } - ss.update[v] = key -} - -// join joins the string and ensures that all entries are of the same length. -func (ss *stringSet) join() string { - ss.setType(Indexed) - n := len(ss.s[0]) - for _, s := range ss.s { - if len(s) != n { - log.Panicf("join: not all entries are of the same length: %q", s) - } - } - ss.s = append(ss.s, strings.Repeat("\xff", n)) - return strings.Join(ss.s, "") -} - -// ianaEntry holds information for an entry in the IANA Language Subtag Repository. -// All types use the same entry. -// See http://tools.ietf.org/html/bcp47#section-5.1 for a description of the various -// fields. -type ianaEntry struct { - typ string - description []string - scope string - added string - preferred string - deprecated string - suppressScript string - macro string - prefix []string -} - -type builder struct { - w *gen.CodeWriter - hw io.Writer // MultiWriter for w and w.Hash - data *cldr.CLDR - supp *cldr.SupplementalData - - // indices - locale stringSet // common locales - lang stringSet // canonical language ids (2 or 3 letter ISO codes) with data - langNoIndex stringSet // 3-letter ISO codes with no associated data - script stringSet // 4-letter ISO codes - region stringSet // 2-letter ISO or 3-digit UN M49 codes - variant stringSet // 4-8-alphanumeric variant code. - - // Region codes that are groups with their corresponding group IDs. - groups map[int]index - - // langInfo - registry map[string]*ianaEntry -} - -type index uint - -func newBuilder(w *gen.CodeWriter) *builder { - r := gen.OpenCLDRCoreZip() - defer r.Close() - d := &cldr.Decoder{} - data, err := d.DecodeZip(r) - failOnError(err) - b := builder{ - w: w, - hw: io.MultiWriter(w, w.Hash), - data: data, - supp: data.Supplemental(), - } - b.parseRegistry() - return &b -} - -func (b *builder) parseRegistry() { - r := gen.OpenIANAFile("assignments/language-subtag-registry") - defer r.Close() - b.registry = make(map[string]*ianaEntry) - - scan := bufio.NewScanner(r) - scan.Split(bufio.ScanWords) - var record *ianaEntry - for more := scan.Scan(); more; { - key := scan.Text() - more = scan.Scan() - value := scan.Text() - switch key { - case "Type:": - record = &ianaEntry{typ: value} - case "Subtag:", "Tag:": - if s := strings.SplitN(value, "..", 2); len(s) > 1 { - for a := s[0]; a <= s[1]; a = inc(a) { - b.addToRegistry(a, record) - } - } else { - b.addToRegistry(value, record) - } - case "Suppress-Script:": - record.suppressScript = value - case "Added:": - record.added = value - case "Deprecated:": - record.deprecated = value - case "Macrolanguage:": - record.macro = value - case "Preferred-Value:": - record.preferred = value - case "Prefix:": - record.prefix = append(record.prefix, value) - case "Scope:": - record.scope = value - case "Description:": - buf := []byte(value) - for more = scan.Scan(); more; more = scan.Scan() { - b := scan.Bytes() - if b[0] == '%' || b[len(b)-1] == ':' { - break - } - buf = append(buf, ' ') - buf = append(buf, b...) - } - record.description = append(record.description, string(buf)) - continue - default: - continue - } - more = scan.Scan() - } - if scan.Err() != nil { - log.Panic(scan.Err()) - } -} - -func (b *builder) addToRegistry(key string, entry *ianaEntry) { - if info, ok := b.registry[key]; ok { - if info.typ != "language" || entry.typ != "extlang" { - log.Fatalf("parseRegistry: tag %q already exists", key) - } - } else { - b.registry[key] = entry - } -} - -var commentIndex = make(map[string]string) - -func init() { - for _, s := range comment { - key := strings.TrimSpace(strings.SplitN(s, " ", 2)[0]) - commentIndex[key] = s - } -} - -func (b *builder) comment(name string) { - if s := commentIndex[name]; len(s) > 0 { - b.w.WriteComment(s) - } else { - fmt.Fprintln(b.w) - } -} - -func (b *builder) pf(f string, x ...interface{}) { - fmt.Fprintf(b.hw, f, x...) - fmt.Fprint(b.hw, "\n") -} - -func (b *builder) p(x ...interface{}) { - fmt.Fprintln(b.hw, x...) -} - -func (b *builder) addSize(s int) { - b.w.Size += s - b.pf("// Size: %d bytes", s) -} - -func (b *builder) writeConst(name string, x interface{}) { - b.comment(name) - b.w.WriteConst(name, x) -} - -// writeConsts computes f(v) for all v in values and writes the results -// as constants named _v to a single constant block. -func (b *builder) writeConsts(f func(string) int, values ...string) { - b.pf("const (") - for _, v := range values { - b.pf("\t_%s = %v", v, f(v)) - } - b.pf(")") -} - -// writeType writes the type of the given value, which must be a struct. -func (b *builder) writeType(value interface{}) { - b.comment(reflect.TypeOf(value).Name()) - b.w.WriteType(value) -} - -func (b *builder) writeSlice(name string, ss interface{}) { - b.writeSliceAddSize(name, 0, ss) -} - -func (b *builder) writeSliceAddSize(name string, extraSize int, ss interface{}) { - b.comment(name) - b.w.Size += extraSize - v := reflect.ValueOf(ss) - t := v.Type().Elem() - b.pf("// Size: %d bytes, %d elements", v.Len()*int(t.Size())+extraSize, v.Len()) - - fmt.Fprintf(b.w, "var %s = ", name) - b.w.WriteArray(ss) - b.p() -} - -type fromTo struct { - from, to uint16 -} - -func (b *builder) writeSortedMap(name string, ss *stringSet, index func(s string) uint16) { - ss.sortFunc(func(a, b string) bool { - return index(a) < index(b) - }) - m := []fromTo{} - for _, s := range ss.s { - m = append(m, fromTo{index(s), index(ss.update[s])}) - } - b.writeSlice(name, m) -} - -const base = 'z' - 'a' + 1 - -func strToInt(s string) uint { - v := uint(0) - for i := 0; i < len(s); i++ { - v *= base - v += uint(s[i] - 'a') - } - return v -} - -// converts the given integer to the original ASCII string passed to strToInt. -// len(s) must match the number of characters obtained. -func intToStr(v uint, s []byte) { - for i := len(s) - 1; i >= 0; i-- { - s[i] = byte(v%base) + 'a' - v /= base - } -} - -func (b *builder) writeBitVector(name string, ss []string) { - vec := make([]uint8, int(math.Ceil(math.Pow(base, float64(len(ss[0])))/8))) - for _, s := range ss { - v := strToInt(s) - vec[v/8] |= 1 << (v % 8) - } - b.writeSlice(name, vec) -} - -// TODO: convert this type into a list or two-stage trie. -func (b *builder) writeMapFunc(name string, m map[string]string, f func(string) uint16) { - b.comment(name) - v := reflect.ValueOf(m) - sz := v.Len() * (2 + int(v.Type().Key().Size())) - for _, k := range m { - sz += len(k) - } - b.addSize(sz) - keys := []string{} - b.pf(`var %s = map[string]uint16{`, name) - for k := range m { - keys = append(keys, k) - } - sort.Strings(keys) - for _, k := range keys { - b.pf("\t%q: %v,", k, f(m[k])) - } - b.p("}") -} - -func (b *builder) writeMap(name string, m interface{}) { - b.comment(name) - v := reflect.ValueOf(m) - sz := v.Len() * (2 + int(v.Type().Key().Size()) + int(v.Type().Elem().Size())) - b.addSize(sz) - f := strings.FieldsFunc(fmt.Sprintf("%#v", m), func(r rune) bool { - return strings.IndexRune("{}, ", r) != -1 - }) - sort.Strings(f[1:]) - b.pf(`var %s = %s{`, name, f[0]) - for _, kv := range f[1:] { - b.pf("\t%s,", kv) - } - b.p("}") -} - -func (b *builder) langIndex(s string) uint16 { - if s == "und" { - return 0 - } - if i, ok := b.lang.find(s); ok { - return uint16(i) - } - return uint16(strToInt(s)) + uint16(len(b.lang.s)) -} - -// inc advances the string to its lexicographical successor. -func inc(s string) string { - const maxTagLength = 4 - var buf [maxTagLength]byte - intToStr(strToInt(strings.ToLower(s))+1, buf[:len(s)]) - for i := 0; i < len(s); i++ { - if s[i] <= 'Z' { - buf[i] -= 'a' - 'A' - } - } - return string(buf[:len(s)]) -} - -func (b *builder) parseIndices() { - meta := b.supp.Metadata - - for k, v := range b.registry { - var ss *stringSet - switch v.typ { - case "language": - if len(k) == 2 || v.suppressScript != "" || v.scope == "special" { - b.lang.add(k) - continue - } else { - ss = &b.langNoIndex - } - case "region": - ss = &b.region - case "script": - ss = &b.script - case "variant": - ss = &b.variant - default: - continue - } - ss.add(k) - } - // Include any language for which there is data. - for _, lang := range b.data.Locales() { - if x := b.data.RawLDML(lang); false || - x.LocaleDisplayNames != nil || - x.Characters != nil || - x.Delimiters != nil || - x.Measurement != nil || - x.Dates != nil || - x.Numbers != nil || - x.Units != nil || - x.ListPatterns != nil || - x.Collations != nil || - x.Segmentations != nil || - x.Rbnf != nil || - x.Annotations != nil || - x.Metadata != nil { - - from := strings.Split(lang, "_") - if lang := from[0]; lang != "root" { - b.lang.add(lang) - } - } - } - // Include locales for plural rules, which uses a different structure. - for _, plurals := range b.data.Supplemental().Plurals { - for _, rules := range plurals.PluralRules { - for _, lang := range strings.Split(rules.Locales, " ") { - if lang = strings.Split(lang, "_")[0]; lang != "root" { - b.lang.add(lang) - } - } - } - } - // Include languages in likely subtags. - for _, m := range b.supp.LikelySubtags.LikelySubtag { - from := strings.Split(m.From, "_") - b.lang.add(from[0]) - } - // Include ISO-639 alpha-3 bibliographic entries. - for _, a := range meta.Alias.LanguageAlias { - if a.Reason == "bibliographic" { - b.langNoIndex.add(a.Type) - } - } - // Include regions in territoryAlias (not all are in the IANA registry!) - for _, reg := range b.supp.Metadata.Alias.TerritoryAlias { - if len(reg.Type) == 2 { - b.region.add(reg.Type) - } - } - - for _, s := range b.lang.s { - if len(s) == 3 { - b.langNoIndex.remove(s) - } - } - b.writeConst("numLanguages", len(b.lang.slice())+len(b.langNoIndex.slice())) - b.writeConst("numScripts", len(b.script.slice())) - b.writeConst("numRegions", len(b.region.slice())) - - // Add dummy codes at the start of each list to represent "unspecified". - b.lang.add("---") - b.script.add("----") - b.region.add("---") - - // common locales - b.locale.parse(meta.DefaultContent.Locales) -} - -// TODO: region inclusion data will probably not be use used in future matchers. - -func (b *builder) computeRegionGroups() { - b.groups = make(map[int]index) - - // Create group indices. - for i := 1; b.region.s[i][0] < 'A'; i++ { // Base M49 indices on regionID. - b.groups[i] = index(len(b.groups)) - } - for _, g := range b.supp.TerritoryContainment.Group { - // Skip UN and EURO zone as they are flattening the containment - // relationship. - if g.Type == "EZ" || g.Type == "UN" { - continue - } - group := b.region.index(g.Type) - if _, ok := b.groups[group]; !ok { - b.groups[group] = index(len(b.groups)) - } - } - if len(b.groups) > 32 { - log.Fatalf("only 32 groups supported, found %d", len(b.groups)) - } - b.writeConst("nRegionGroups", len(b.groups)) -} - -var langConsts = []string{ - "af", "am", "ar", "az", "bg", "bn", "ca", "cs", "da", "de", "el", "en", "es", - "et", "fa", "fi", "fil", "fr", "gu", "he", "hi", "hr", "hu", "hy", "id", "is", - "it", "ja", "ka", "kk", "km", "kn", "ko", "ky", "lo", "lt", "lv", "mk", "ml", - "mn", "mo", "mr", "ms", "mul", "my", "nb", "ne", "nl", "no", "pa", "pl", "pt", - "ro", "ru", "sh", "si", "sk", "sl", "sq", "sr", "sv", "sw", "ta", "te", "th", - "tl", "tn", "tr", "uk", "ur", "uz", "vi", "zh", "zu", - - // constants for grandfathered tags (if not already defined) - "jbo", "ami", "bnn", "hak", "tlh", "lb", "nv", "pwn", "tao", "tay", "tsu", - "nn", "sfb", "vgt", "sgg", "cmn", "nan", "hsn", -} - -// writeLanguage generates all tables needed for language canonicalization. -func (b *builder) writeLanguage() { - meta := b.supp.Metadata - - b.writeConst("nonCanonicalUnd", b.lang.index("und")) - b.writeConsts(func(s string) int { return int(b.langIndex(s)) }, langConsts...) - b.writeConst("langPrivateStart", b.langIndex("qaa")) - b.writeConst("langPrivateEnd", b.langIndex("qtz")) - - // Get language codes that need to be mapped (overlong 3-letter codes, - // deprecated 2-letter codes, legacy and grandfathered tags.) - langAliasMap := stringSet{} - aliasTypeMap := map[string]langAliasType{} - - // altLangISO3 get the alternative ISO3 names that need to be mapped. - altLangISO3 := stringSet{} - // Add dummy start to avoid the use of index 0. - altLangISO3.add("---") - altLangISO3.updateLater("---", "aa") - - lang := b.lang.clone() - for _, a := range meta.Alias.LanguageAlias { - if a.Replacement == "" { - a.Replacement = "und" - } - // TODO: support mapping to tags - repl := strings.SplitN(a.Replacement, "_", 2)[0] - if a.Reason == "overlong" { - if len(a.Replacement) == 2 && len(a.Type) == 3 { - lang.updateLater(a.Replacement, a.Type) - } - } else if len(a.Type) <= 3 { - switch a.Reason { - case "macrolanguage": - aliasTypeMap[a.Type] = langMacro - case "deprecated": - // handled elsewhere - continue - case "bibliographic", "legacy": - if a.Type == "no" { - continue - } - aliasTypeMap[a.Type] = langLegacy - default: - log.Fatalf("new %s alias: %s", a.Reason, a.Type) - } - langAliasMap.add(a.Type) - langAliasMap.updateLater(a.Type, repl) - } - } - // Manually add the mapping of "nb" (Norwegian) to its macro language. - // This can be removed if CLDR adopts this change. - langAliasMap.add("nb") - langAliasMap.updateLater("nb", "no") - aliasTypeMap["nb"] = langMacro - - for k, v := range b.registry { - // Also add deprecated values for 3-letter ISO codes, which CLDR omits. - if v.typ == "language" && v.deprecated != "" && v.preferred != "" { - langAliasMap.add(k) - langAliasMap.updateLater(k, v.preferred) - aliasTypeMap[k] = langDeprecated - } - } - // Fix CLDR mappings. - lang.updateLater("tl", "tgl") - lang.updateLater("sh", "hbs") - lang.updateLater("mo", "mol") - lang.updateLater("no", "nor") - lang.updateLater("tw", "twi") - lang.updateLater("nb", "nob") - lang.updateLater("ak", "aka") - lang.updateLater("bh", "bih") - - // Ensure that each 2-letter code is matched with a 3-letter code. - for _, v := range lang.s[1:] { - s, ok := lang.update[v] - if !ok { - if s, ok = lang.update[langAliasMap.update[v]]; !ok { - continue - } - lang.update[v] = s - } - if v[0] != s[0] { - altLangISO3.add(s) - altLangISO3.updateLater(s, v) - } - } - - // Complete canonialized language tags. - lang.freeze() - for i, v := range lang.s { - // We can avoid these manual entries by using the IANI registry directly. - // Seems easier to update the list manually, as changes are rare. - // The panic in this loop will trigger if we miss an entry. - add := "" - if s, ok := lang.update[v]; ok { - if s[0] == v[0] { - add = s[1:] - } else { - add = string([]byte{0, byte(altLangISO3.index(s))}) - } - } else if len(v) == 3 { - add = "\x00" - } else { - log.Panicf("no data for long form of %q", v) - } - lang.s[i] += add - } - b.writeConst("lang", tag.Index(lang.join())) - - b.writeConst("langNoIndexOffset", len(b.lang.s)) - - // space of all valid 3-letter language identifiers. - b.writeBitVector("langNoIndex", b.langNoIndex.slice()) - - altLangIndex := []uint16{} - for i, s := range altLangISO3.slice() { - altLangISO3.s[i] += string([]byte{byte(len(altLangIndex))}) - if i > 0 { - idx := b.lang.index(altLangISO3.update[s]) - altLangIndex = append(altLangIndex, uint16(idx)) - } - } - b.writeConst("altLangISO3", tag.Index(altLangISO3.join())) - b.writeSlice("altLangIndex", altLangIndex) - - b.writeSortedMap("langAliasMap", &langAliasMap, b.langIndex) - types := make([]langAliasType, len(langAliasMap.s)) - for i, s := range langAliasMap.s { - types[i] = aliasTypeMap[s] - } - b.writeSlice("langAliasTypes", types) -} - -var scriptConsts = []string{ - "Latn", "Hani", "Hans", "Hant", "Qaaa", "Qaai", "Qabx", "Zinh", "Zyyy", - "Zzzz", -} - -func (b *builder) writeScript() { - b.writeConsts(b.script.index, scriptConsts...) - b.writeConst("script", tag.Index(b.script.join())) - - supp := make([]uint8, len(b.lang.slice())) - for i, v := range b.lang.slice()[1:] { - if sc := b.registry[v].suppressScript; sc != "" { - supp[i+1] = uint8(b.script.index(sc)) - } - } - b.writeSlice("suppressScript", supp) - - // There is only one deprecated script in CLDR. This value is hard-coded. - // We check here if the code must be updated. - for _, a := range b.supp.Metadata.Alias.ScriptAlias { - if a.Type != "Qaai" { - log.Panicf("unexpected deprecated stript %q", a.Type) - } - } -} - -func parseM49(s string) int16 { - if len(s) == 0 { - return 0 - } - v, err := strconv.ParseUint(s, 10, 10) - failOnError(err) - return int16(v) -} - -var regionConsts = []string{ - "001", "419", "BR", "CA", "ES", "GB", "MD", "PT", "UK", "US", - "ZZ", "XA", "XC", "XK", // Unofficial tag for Kosovo. -} - -func (b *builder) writeRegion() { - b.writeConsts(b.region.index, regionConsts...) - - isoOffset := b.region.index("AA") - m49map := make([]int16, len(b.region.slice())) - fromM49map := make(map[int16]int) - altRegionISO3 := "" - altRegionIDs := []uint16{} - - b.writeConst("isoRegionOffset", isoOffset) - - // 2-letter region lookup and mapping to numeric codes. - regionISO := b.region.clone() - regionISO.s = regionISO.s[isoOffset:] - regionISO.sorted = false - - regionTypes := make([]byte, len(b.region.s)) - - // Is the region valid BCP 47? - for s, e := range b.registry { - if len(s) == 2 && s == strings.ToUpper(s) { - i := b.region.index(s) - for _, d := range e.description { - if strings.Contains(d, "Private use") { - regionTypes[i] = iso3166UserAssgined - } - } - regionTypes[i] |= bcp47Region - } - } - - // Is the region a valid ccTLD? - r := gen.OpenIANAFile("domains/root/db") - defer r.Close() - - buf, err := ioutil.ReadAll(r) - failOnError(err) - re := regexp.MustCompile(`"/domains/root/db/([a-z]{2}).html"`) - for _, m := range re.FindAllSubmatch(buf, -1) { - i := b.region.index(strings.ToUpper(string(m[1]))) - regionTypes[i] |= ccTLD - } - - b.writeSlice("regionTypes", regionTypes) - - iso3Set := make(map[string]int) - update := func(iso2, iso3 string) { - i := regionISO.index(iso2) - if j, ok := iso3Set[iso3]; !ok && iso3[0] == iso2[0] { - regionISO.s[i] += iso3[1:] - iso3Set[iso3] = -1 - } else { - if ok && j >= 0 { - regionISO.s[i] += string([]byte{0, byte(j)}) - } else { - iso3Set[iso3] = len(altRegionISO3) - regionISO.s[i] += string([]byte{0, byte(len(altRegionISO3))}) - altRegionISO3 += iso3 - altRegionIDs = append(altRegionIDs, uint16(isoOffset+i)) - } - } - } - for _, tc := range b.supp.CodeMappings.TerritoryCodes { - i := regionISO.index(tc.Type) + isoOffset - if d := m49map[i]; d != 0 { - log.Panicf("%s found as a duplicate UN.M49 code of %03d", tc.Numeric, d) - } - m49 := parseM49(tc.Numeric) - m49map[i] = m49 - if r := fromM49map[m49]; r == 0 { - fromM49map[m49] = i - } else if r != i { - dep := b.registry[regionISO.s[r-isoOffset]].deprecated - if t := b.registry[tc.Type]; t != nil && dep != "" && (t.deprecated == "" || t.deprecated > dep) { - fromM49map[m49] = i - } - } - } - for _, ta := range b.supp.Metadata.Alias.TerritoryAlias { - if len(ta.Type) == 3 && ta.Type[0] <= '9' && len(ta.Replacement) == 2 { - from := parseM49(ta.Type) - if r := fromM49map[from]; r == 0 { - fromM49map[from] = regionISO.index(ta.Replacement) + isoOffset - } - } - } - for _, tc := range b.supp.CodeMappings.TerritoryCodes { - if len(tc.Alpha3) == 3 { - update(tc.Type, tc.Alpha3) - } - } - // This entries are not included in territoryCodes. Mostly 3-letter variants - // of deleted codes and an entry for QU. - for _, m := range []struct{ iso2, iso3 string }{ - {"CT", "CTE"}, - {"DY", "DHY"}, - {"HV", "HVO"}, - {"JT", "JTN"}, - {"MI", "MID"}, - {"NH", "NHB"}, - {"NQ", "ATN"}, - {"PC", "PCI"}, - {"PU", "PUS"}, - {"PZ", "PCZ"}, - {"RH", "RHO"}, - {"VD", "VDR"}, - {"WK", "WAK"}, - // These three-letter codes are used for others as well. - {"FQ", "ATF"}, - } { - update(m.iso2, m.iso3) - } - for i, s := range regionISO.s { - if len(s) != 4 { - regionISO.s[i] = s + " " - } - } - b.writeConst("regionISO", tag.Index(regionISO.join())) - b.writeConst("altRegionISO3", altRegionISO3) - b.writeSlice("altRegionIDs", altRegionIDs) - - // Create list of deprecated regions. - // TODO: consider inserting SF -> FI. Not included by CLDR, but is the only - // Transitionally-reserved mapping not included. - regionOldMap := stringSet{} - // Include regions in territoryAlias (not all are in the IANA registry!) - for _, reg := range b.supp.Metadata.Alias.TerritoryAlias { - if len(reg.Type) == 2 && reg.Reason == "deprecated" && len(reg.Replacement) == 2 { - regionOldMap.add(reg.Type) - regionOldMap.updateLater(reg.Type, reg.Replacement) - i, _ := regionISO.find(reg.Type) - j, _ := regionISO.find(reg.Replacement) - if k := m49map[i+isoOffset]; k == 0 { - m49map[i+isoOffset] = m49map[j+isoOffset] - } - } - } - b.writeSortedMap("regionOldMap", ®ionOldMap, func(s string) uint16 { - return uint16(b.region.index(s)) - }) - // 3-digit region lookup, groupings. - for i := 1; i < isoOffset; i++ { - m := parseM49(b.region.s[i]) - m49map[i] = m - fromM49map[m] = i - } - b.writeSlice("m49", m49map) - - const ( - searchBits = 7 - regionBits = 9 - ) - if len(m49map) >= 1< %d", len(m49map), 1<>searchBits] = int16(len(fromM49)) - } - b.writeSlice("m49Index", m49Index) - b.writeSlice("fromM49", fromM49) -} - -const ( - // TODO: put these lists in regionTypes as user data? Could be used for - // various optimizations and refinements and could be exposed in the API. - iso3166Except = "AC CP DG EA EU FX IC SU TA UK" - iso3166Trans = "AN BU CS NT TP YU ZR" // SF is not in our set of Regions. - // DY and RH are actually not deleted, but indeterminately reserved. - iso3166DelCLDR = "CT DD DY FQ HV JT MI NH NQ PC PU PZ RH VD WK YD" -) - -const ( - iso3166UserAssgined = 1 << iota - ccTLD - bcp47Region -) - -func find(list []string, s string) int { - for i, t := range list { - if t == s { - return i - } - } - return -1 -} - -// writeVariants generates per-variant information and creates a map from variant -// name to index value. We assign index values such that sorting multiple -// variants by index value will result in the correct order. -// There are two types of variants: specialized and general. Specialized variants -// are only applicable to certain language or language-script pairs. Generalized -// variants apply to any language. Generalized variants always sort after -// specialized variants. We will therefore always assign a higher index value -// to a generalized variant than any other variant. Generalized variants are -// sorted alphabetically among themselves. -// Specialized variants may also sort after other specialized variants. Such -// variants will be ordered after any of the variants they may follow. -// We assume that if a variant x is followed by a variant y, then for any prefix -// p of x, p-x is a prefix of y. This allows us to order tags based on the -// maximum of the length of any of its prefixes. -// TODO: it is possible to define a set of Prefix values on variants such that -// a total order cannot be defined to the point that this algorithm breaks. -// In other words, we cannot guarantee the same order of variants for the -// future using the same algorithm or for non-compliant combinations of -// variants. For this reason, consider using simple alphabetic sorting -// of variants and ignore Prefix restrictions altogether. -func (b *builder) writeVariant() { - generalized := stringSet{} - specialized := stringSet{} - specializedExtend := stringSet{} - // Collate the variants by type and check assumptions. - for _, v := range b.variant.slice() { - e := b.registry[v] - if len(e.prefix) == 0 { - generalized.add(v) - continue - } - c := strings.Split(e.prefix[0], "-") - hasScriptOrRegion := false - if len(c) > 1 { - _, hasScriptOrRegion = b.script.find(c[1]) - if !hasScriptOrRegion { - _, hasScriptOrRegion = b.region.find(c[1]) - - } - } - if len(c) == 1 || len(c) == 2 && hasScriptOrRegion { - // Variant is preceded by a language. - specialized.add(v) - continue - } - // Variant is preceded by another variant. - specializedExtend.add(v) - prefix := c[0] + "-" - if hasScriptOrRegion { - prefix += c[1] - } - for _, p := range e.prefix { - // Verify that the prefix minus the last element is a prefix of the - // predecessor element. - i := strings.LastIndex(p, "-") - pred := b.registry[p[i+1:]] - if find(pred.prefix, p[:i]) < 0 { - log.Fatalf("prefix %q for variant %q not consistent with predecessor spec", p, v) - } - // The sorting used below does not work in the general case. It works - // if we assume that variants that may be followed by others only have - // prefixes of the same length. Verify this. - count := strings.Count(p[:i], "-") - for _, q := range pred.prefix { - if c := strings.Count(q, "-"); c != count { - log.Fatalf("variant %q preceding %q has a prefix %q of size %d; want %d", p[i+1:], v, q, c, count) - } - } - if !strings.HasPrefix(p, prefix) { - log.Fatalf("prefix %q of variant %q should start with %q", p, v, prefix) - } - } - } - - // Sort extended variants. - a := specializedExtend.s - less := func(v, w string) bool { - // Sort by the maximum number of elements. - maxCount := func(s string) (max int) { - for _, p := range b.registry[s].prefix { - if c := strings.Count(p, "-"); c > max { - max = c - } - } - return - } - if cv, cw := maxCount(v), maxCount(w); cv != cw { - return cv < cw - } - // Sort by name as tie breaker. - return v < w - } - sort.Sort(funcSorter{less, sort.StringSlice(a)}) - specializedExtend.frozen = true - - // Create index from variant name to index. - variantIndex := make(map[string]uint8) - add := func(s []string) { - for _, v := range s { - variantIndex[v] = uint8(len(variantIndex)) - } - } - add(specialized.slice()) - add(specializedExtend.s) - numSpecialized := len(variantIndex) - add(generalized.slice()) - if n := len(variantIndex); n > 255 { - log.Fatalf("maximum number of variants exceeded: was %d; want <= 255", n) - } - b.writeMap("variantIndex", variantIndex) - b.writeConst("variantNumSpecialized", numSpecialized) -} - -func (b *builder) writeLanguageInfo() { -} - -// writeLikelyData writes tables that are used both for finding parent relations and for -// language matching. Each entry contains additional bits to indicate the status of the -// data to know when it cannot be used for parent relations. -func (b *builder) writeLikelyData() { - const ( - isList = 1 << iota - scriptInFrom - regionInFrom - ) - type ( // generated types - likelyScriptRegion struct { - region uint16 - script uint8 - flags uint8 - } - likelyLangScript struct { - lang uint16 - script uint8 - flags uint8 - } - likelyLangRegion struct { - lang uint16 - region uint16 - } - // likelyTag is used for getting likely tags for group regions, where - // the likely region might be a region contained in the group. - likelyTag struct { - lang uint16 - region uint16 - script uint8 - } - ) - var ( // generated variables - likelyRegionGroup = make([]likelyTag, len(b.groups)) - likelyLang = make([]likelyScriptRegion, len(b.lang.s)) - likelyRegion = make([]likelyLangScript, len(b.region.s)) - likelyScript = make([]likelyLangRegion, len(b.script.s)) - likelyLangList = []likelyScriptRegion{} - likelyRegionList = []likelyLangScript{} - ) - type fromTo struct { - from, to []string - } - langToOther := map[int][]fromTo{} - regionToOther := map[int][]fromTo{} - for _, m := range b.supp.LikelySubtags.LikelySubtag { - from := strings.Split(m.From, "_") - to := strings.Split(m.To, "_") - if len(to) != 3 { - log.Fatalf("invalid number of subtags in %q: found %d, want 3", m.To, len(to)) - } - if len(from) > 3 { - log.Fatalf("invalid number of subtags: found %d, want 1-3", len(from)) - } - if from[0] != to[0] && from[0] != "und" { - log.Fatalf("unexpected language change in expansion: %s -> %s", from, to) - } - if len(from) == 3 { - if from[2] != to[2] { - log.Fatalf("unexpected region change in expansion: %s -> %s", from, to) - } - if from[0] != "und" { - log.Fatalf("unexpected fully specified from tag: %s -> %s", from, to) - } - } - if len(from) == 1 || from[0] != "und" { - id := 0 - if from[0] != "und" { - id = b.lang.index(from[0]) - } - langToOther[id] = append(langToOther[id], fromTo{from, to}) - } else if len(from) == 2 && len(from[1]) == 4 { - sid := b.script.index(from[1]) - likelyScript[sid].lang = uint16(b.langIndex(to[0])) - likelyScript[sid].region = uint16(b.region.index(to[2])) - } else { - r := b.region.index(from[len(from)-1]) - if id, ok := b.groups[r]; ok { - if from[0] != "und" { - log.Fatalf("region changed unexpectedly: %s -> %s", from, to) - } - likelyRegionGroup[id].lang = uint16(b.langIndex(to[0])) - likelyRegionGroup[id].script = uint8(b.script.index(to[1])) - likelyRegionGroup[id].region = uint16(b.region.index(to[2])) - } else { - regionToOther[r] = append(regionToOther[r], fromTo{from, to}) - } - } - } - b.writeType(likelyLangRegion{}) - b.writeSlice("likelyScript", likelyScript) - - for id := range b.lang.s { - list := langToOther[id] - if len(list) == 1 { - likelyLang[id].region = uint16(b.region.index(list[0].to[2])) - likelyLang[id].script = uint8(b.script.index(list[0].to[1])) - } else if len(list) > 1 { - likelyLang[id].flags = isList - likelyLang[id].region = uint16(len(likelyLangList)) - likelyLang[id].script = uint8(len(list)) - for _, x := range list { - flags := uint8(0) - if len(x.from) > 1 { - if x.from[1] == x.to[2] { - flags = regionInFrom - } else { - flags = scriptInFrom - } - } - likelyLangList = append(likelyLangList, likelyScriptRegion{ - region: uint16(b.region.index(x.to[2])), - script: uint8(b.script.index(x.to[1])), - flags: flags, - }) - } - } - } - // TODO: merge suppressScript data with this table. - b.writeType(likelyScriptRegion{}) - b.writeSlice("likelyLang", likelyLang) - b.writeSlice("likelyLangList", likelyLangList) - - for id := range b.region.s { - list := regionToOther[id] - if len(list) == 1 { - likelyRegion[id].lang = uint16(b.langIndex(list[0].to[0])) - likelyRegion[id].script = uint8(b.script.index(list[0].to[1])) - if len(list[0].from) > 2 { - likelyRegion[id].flags = scriptInFrom - } - } else if len(list) > 1 { - likelyRegion[id].flags = isList - likelyRegion[id].lang = uint16(len(likelyRegionList)) - likelyRegion[id].script = uint8(len(list)) - for i, x := range list { - if len(x.from) == 2 && i != 0 || i > 0 && len(x.from) != 3 { - log.Fatalf("unspecified script must be first in list: %v at %d", x.from, i) - } - x := likelyLangScript{ - lang: uint16(b.langIndex(x.to[0])), - script: uint8(b.script.index(x.to[1])), - } - if len(list[0].from) > 2 { - x.flags = scriptInFrom - } - likelyRegionList = append(likelyRegionList, x) - } - } - } - b.writeType(likelyLangScript{}) - b.writeSlice("likelyRegion", likelyRegion) - b.writeSlice("likelyRegionList", likelyRegionList) - - b.writeType(likelyTag{}) - b.writeSlice("likelyRegionGroup", likelyRegionGroup) -} - -type mutualIntelligibility struct { - want, have uint16 - conf uint8 - oneway bool -} - -type scriptIntelligibility struct { - lang uint16 // langID or 0 if * - want, have uint8 - conf uint8 -} - -type sortByConf []mutualIntelligibility - -func (l sortByConf) Less(a, b int) bool { - return l[a].conf > l[b].conf -} - -func (l sortByConf) Swap(a, b int) { - l[a], l[b] = l[b], l[a] -} - -func (l sortByConf) Len() int { - return len(l) -} - -// toConf converts a percentage value [0, 100] to a confidence class. -func toConf(pct uint8) uint8 { - switch { - case pct == 100: - return 3 // Exact - case pct >= 90: - return 2 // High - case pct > 50: - return 1 // Low - default: - return 0 // No - } -} - -// writeMatchData writes tables with languages and scripts for which there is -// mutual intelligibility. The data is based on CLDR's languageMatching data. -// Note that we use a different algorithm than the one defined by CLDR and that -// we slightly modify the data. For example, we convert scores to confidence levels. -// We also drop all region-related data as we use a different algorithm to -// determine region equivalence. -func (b *builder) writeMatchData() { - b.writeType(mutualIntelligibility{}) - b.writeType(scriptIntelligibility{}) - lm := b.supp.LanguageMatching.LanguageMatches - cldr.MakeSlice(&lm).SelectAnyOf("type", "written") - - matchLang := []mutualIntelligibility{} - matchScript := []scriptIntelligibility{} - // Convert the languageMatch entries in lists keyed by desired language. - for _, m := range lm[0].LanguageMatch { - // Different versions of CLDR use different separators. - desired := strings.Replace(m.Desired, "-", "_", -1) - supported := strings.Replace(m.Supported, "-", "_", -1) - d := strings.Split(desired, "_") - s := strings.Split(supported, "_") - if len(d) != len(s) || len(d) > 2 { - // Skip all entries with regions and work around CLDR bug. - continue - } - pct, _ := strconv.ParseInt(m.Percent, 10, 8) - if len(d) == 2 && d[0] == s[0] && len(d[1]) == 4 { - // language-script pair. - lang := uint16(0) - if d[0] != "*" { - lang = uint16(b.langIndex(d[0])) - } - matchScript = append(matchScript, scriptIntelligibility{ - lang: lang, - want: uint8(b.script.index(d[1])), - have: uint8(b.script.index(s[1])), - conf: toConf(uint8(pct)), - }) - if m.Oneway != "true" { - matchScript = append(matchScript, scriptIntelligibility{ - lang: lang, - want: uint8(b.script.index(s[1])), - have: uint8(b.script.index(d[1])), - conf: toConf(uint8(pct)), - }) - } - } else if len(d) == 1 && d[0] != "*" { - if pct == 100 { - // nb == no is already handled by macro mapping. Check there - // really is only this case. - if d[0] != "no" || s[0] != "nb" { - log.Fatalf("unhandled equivalence %s == %s", s[0], d[0]) - } - continue - } - matchLang = append(matchLang, mutualIntelligibility{ - want: uint16(b.langIndex(d[0])), - have: uint16(b.langIndex(s[0])), - conf: uint8(pct), - oneway: m.Oneway == "true", - }) - } else { - // TODO: Handle other mappings. - a := []string{"*;*", "*_*;*_*", "es_MX;es_419"} - s := strings.Join([]string{desired, supported}, ";") - if i := sort.SearchStrings(a, s); i == len(a) || a[i] != s { - log.Printf("%q not handled", s) - } - } - } - sort.Stable(sortByConf(matchLang)) - // collapse percentage into confidence classes - for i, m := range matchLang { - matchLang[i].conf = toConf(m.conf) - } - b.writeSlice("matchLang", matchLang) - b.writeSlice("matchScript", matchScript) -} - -func (b *builder) writeRegionInclusionData() { - var ( - // mm holds for each group the set of groups with a distance of 1. - mm = make(map[int][]index) - - // containment holds for each group the transitive closure of - // containment of other groups. - containment = make(map[index][]index) - ) - for _, g := range b.supp.TerritoryContainment.Group { - // Skip UN and EURO zone as they are flattening the containment - // relationship. - if g.Type == "EZ" || g.Type == "UN" { - continue - } - group := b.region.index(g.Type) - groupIdx := b.groups[group] - for _, mem := range strings.Split(g.Contains, " ") { - r := b.region.index(mem) - mm[r] = append(mm[r], groupIdx) - if g, ok := b.groups[r]; ok { - mm[group] = append(mm[group], g) - containment[groupIdx] = append(containment[groupIdx], g) - } - } - } - - regionContainment := make([]uint32, len(b.groups)) - for _, g := range b.groups { - l := containment[g] - - // Compute the transitive closure of containment. - for i := 0; i < len(l); i++ { - l = append(l, containment[l[i]]...) - } - - // Compute the bitmask. - regionContainment[g] = 1 << g - for _, v := range l { - regionContainment[g] |= 1 << v - } - // log.Printf("%d: %X", g, regionContainment[g]) - } - b.writeSlice("regionContainment", regionContainment) - - regionInclusion := make([]uint8, len(b.region.s)) - bvs := make(map[uint32]index) - // Make the first bitvector positions correspond with the groups. - for r, i := range b.groups { - bv := uint32(1 << i) - for _, g := range mm[r] { - bv |= 1 << g - } - bvs[bv] = i - regionInclusion[r] = uint8(bvs[bv]) - } - for r := 1; r < len(b.region.s); r++ { - if _, ok := b.groups[r]; !ok { - bv := uint32(0) - for _, g := range mm[r] { - bv |= 1 << g - } - if bv == 0 { - // Pick the world for unspecified regions. - bv = 1 << b.groups[b.region.index("001")] - } - if _, ok := bvs[bv]; !ok { - bvs[bv] = index(len(bvs)) - } - regionInclusion[r] = uint8(bvs[bv]) - } - } - b.writeSlice("regionInclusion", regionInclusion) - regionInclusionBits := make([]uint32, len(bvs)) - for k, v := range bvs { - regionInclusionBits[v] = uint32(k) - } - // Add bit vectors for increasingly large distances until a fixed point is reached. - regionInclusionNext := []uint8{} - for i := 0; i < len(regionInclusionBits); i++ { - bits := regionInclusionBits[i] - next := bits - for i := uint(0); i < uint(len(b.groups)); i++ { - if bits&(1<;Lo;0;L;;;;;N;;;;; -// 4DB5;;Lo;0;L;;;;;N;;;;; -// parseCharacter keeps a state variable indicating the weirdness. -type State int - -const ( - SNormal State = iota // known to be zero for the type - SFirst - SLast - SMissing -) - -var lastChar = rune('\u0000') - -func (c Char) isValid() bool { - return c.codePoint != 0 && c.state != SMissing -} - -type FormInfo struct { - quickCheck [MNumberOfModes]QCResult // index: MComposed or MDecomposed - verified [MNumberOfModes]bool // index: MComposed or MDecomposed - - combinesForward bool // May combine with rune on the right - combinesBackward bool // May combine with rune on the left - isOneWay bool // Never appears in result - inDecomp bool // Some decompositions result in this char. - decomp Decomposition - expandedDecomp Decomposition -} - -func (f FormInfo) String() string { - buf := bytes.NewBuffer(make([]byte, 0)) - - fmt.Fprintf(buf, " quickCheck[C]: %v\n", f.quickCheck[MComposed]) - fmt.Fprintf(buf, " quickCheck[D]: %v\n", f.quickCheck[MDecomposed]) - fmt.Fprintf(buf, " cmbForward: %v\n", f.combinesForward) - fmt.Fprintf(buf, " cmbBackward: %v\n", f.combinesBackward) - fmt.Fprintf(buf, " isOneWay: %v\n", f.isOneWay) - fmt.Fprintf(buf, " inDecomp: %v\n", f.inDecomp) - fmt.Fprintf(buf, " decomposition: %X\n", f.decomp) - fmt.Fprintf(buf, " expandedDecomp: %X\n", f.expandedDecomp) - - return buf.String() -} - -type Decomposition []rune - -func parseDecomposition(s string, skipfirst bool) (a []rune, err error) { - decomp := strings.Split(s, " ") - if len(decomp) > 0 && skipfirst { - decomp = decomp[1:] - } - for _, d := range decomp { - point, err := strconv.ParseUint(d, 16, 64) - if err != nil { - return a, err - } - a = append(a, rune(point)) - } - return a, nil -} - -func loadUnicodeData() { - f := gen.OpenUCDFile("UnicodeData.txt") - defer f.Close() - p := ucd.New(f) - for p.Next() { - r := p.Rune(ucd.CodePoint) - char := &chars[r] - - char.ccc = uint8(p.Uint(ucd.CanonicalCombiningClass)) - decmap := p.String(ucd.DecompMapping) - - exp, err := parseDecomposition(decmap, false) - isCompat := false - if err != nil { - if len(decmap) > 0 { - exp, err = parseDecomposition(decmap, true) - if err != nil { - log.Fatalf(`%U: bad decomp |%v|: "%s"`, r, decmap, err) - } - isCompat = true - } - } - - char.name = p.String(ucd.Name) - char.codePoint = r - char.forms[FCompatibility].decomp = exp - if !isCompat { - char.forms[FCanonical].decomp = exp - } else { - char.compatDecomp = true - } - if len(decmap) > 0 { - char.forms[FCompatibility].decomp = exp - } - } - if err := p.Err(); err != nil { - log.Fatal(err) - } -} - -// compactCCC converts the sparse set of CCC values to a continguous one, -// reducing the number of bits needed from 8 to 6. -func compactCCC() { - m := make(map[uint8]uint8) - for i := range chars { - c := &chars[i] - m[c.ccc] = 0 - } - cccs := []int{} - for v, _ := range m { - cccs = append(cccs, int(v)) - } - sort.Ints(cccs) - for i, c := range cccs { - cccMap[uint8(i)] = uint8(c) - m[uint8(c)] = uint8(i) - } - for i := range chars { - c := &chars[i] - c.origCCC = c.ccc - c.ccc = m[c.ccc] - } - if len(m) >= 1<<6 { - log.Fatalf("too many difference CCC values: %d >= 64", len(m)) - } -} - -// CompositionExclusions.txt has form: -// 0958 # ... -// See http://unicode.org/reports/tr44/ for full explanation -func loadCompositionExclusions() { - f := gen.OpenUCDFile("CompositionExclusions.txt") - defer f.Close() - p := ucd.New(f) - for p.Next() { - c := &chars[p.Rune(0)] - if c.excludeInComp { - log.Fatalf("%U: Duplicate entry in exclusions.", c.codePoint) - } - c.excludeInComp = true - } - if e := p.Err(); e != nil { - log.Fatal(e) - } -} - -// hasCompatDecomp returns true if any of the recursive -// decompositions contains a compatibility expansion. -// In this case, the character may not occur in NFK*. -func hasCompatDecomp(r rune) bool { - c := &chars[r] - if c.compatDecomp { - return true - } - for _, d := range c.forms[FCompatibility].decomp { - if hasCompatDecomp(d) { - return true - } - } - return false -} - -// Hangul related constants. -const ( - HangulBase = 0xAC00 - HangulEnd = 0xD7A4 // hangulBase + Jamo combinations (19 * 21 * 28) - - JamoLBase = 0x1100 - JamoLEnd = 0x1113 - JamoVBase = 0x1161 - JamoVEnd = 0x1176 - JamoTBase = 0x11A8 - JamoTEnd = 0x11C3 - - JamoLVTCount = 19 * 21 * 28 - JamoTCount = 28 -) - -func isHangul(r rune) bool { - return HangulBase <= r && r < HangulEnd -} - -func isHangulWithoutJamoT(r rune) bool { - if !isHangul(r) { - return false - } - r -= HangulBase - return r < JamoLVTCount && r%JamoTCount == 0 -} - -func ccc(r rune) uint8 { - return chars[r].ccc -} - -// Insert a rune in a buffer, ordered by Canonical Combining Class. -func insertOrdered(b Decomposition, r rune) Decomposition { - n := len(b) - b = append(b, 0) - cc := ccc(r) - if cc > 0 { - // Use bubble sort. - for ; n > 0; n-- { - if ccc(b[n-1]) <= cc { - break - } - b[n] = b[n-1] - } - } - b[n] = r - return b -} - -// Recursively decompose. -func decomposeRecursive(form int, r rune, d Decomposition) Decomposition { - dcomp := chars[r].forms[form].decomp - if len(dcomp) == 0 { - return insertOrdered(d, r) - } - for _, c := range dcomp { - d = decomposeRecursive(form, c, d) - } - return d -} - -func completeCharFields(form int) { - // Phase 0: pre-expand decomposition. - for i := range chars { - f := &chars[i].forms[form] - if len(f.decomp) == 0 { - continue - } - exp := make(Decomposition, 0) - for _, c := range f.decomp { - exp = decomposeRecursive(form, c, exp) - } - f.expandedDecomp = exp - } - - // Phase 1: composition exclusion, mark decomposition. - for i := range chars { - c := &chars[i] - f := &c.forms[form] - - // Marks script-specific exclusions and version restricted. - f.isOneWay = c.excludeInComp - - // Singletons - f.isOneWay = f.isOneWay || len(f.decomp) == 1 - - // Non-starter decompositions - if len(f.decomp) > 1 { - chk := c.ccc != 0 || chars[f.decomp[0]].ccc != 0 - f.isOneWay = f.isOneWay || chk - } - - // Runes that decompose into more than two runes. - f.isOneWay = f.isOneWay || len(f.decomp) > 2 - - if form == FCompatibility { - f.isOneWay = f.isOneWay || hasCompatDecomp(c.codePoint) - } - - for _, r := range f.decomp { - chars[r].forms[form].inDecomp = true - } - } - - // Phase 2: forward and backward combining. - for i := range chars { - c := &chars[i] - f := &c.forms[form] - - if !f.isOneWay && len(f.decomp) == 2 { - f0 := &chars[f.decomp[0]].forms[form] - f1 := &chars[f.decomp[1]].forms[form] - if !f0.isOneWay { - f0.combinesForward = true - } - if !f1.isOneWay { - f1.combinesBackward = true - } - } - if isHangulWithoutJamoT(rune(i)) { - f.combinesForward = true - } - } - - // Phase 3: quick check values. - for i := range chars { - c := &chars[i] - f := &c.forms[form] - - switch { - case len(f.decomp) > 0: - f.quickCheck[MDecomposed] = QCNo - case isHangul(rune(i)): - f.quickCheck[MDecomposed] = QCNo - default: - f.quickCheck[MDecomposed] = QCYes - } - switch { - case f.isOneWay: - f.quickCheck[MComposed] = QCNo - case (i & 0xffff00) == JamoLBase: - f.quickCheck[MComposed] = QCYes - if JamoLBase <= i && i < JamoLEnd { - f.combinesForward = true - } - if JamoVBase <= i && i < JamoVEnd { - f.quickCheck[MComposed] = QCMaybe - f.combinesBackward = true - f.combinesForward = true - } - if JamoTBase <= i && i < JamoTEnd { - f.quickCheck[MComposed] = QCMaybe - f.combinesBackward = true - } - case !f.combinesBackward: - f.quickCheck[MComposed] = QCYes - default: - f.quickCheck[MComposed] = QCMaybe - } - } -} - -func computeNonStarterCounts() { - // Phase 4: leading and trailing non-starter count - for i := range chars { - c := &chars[i] - - runes := []rune{rune(i)} - // We always use FCompatibility so that the CGJ insertion points do not - // change for repeated normalizations with different forms. - if exp := c.forms[FCompatibility].expandedDecomp; len(exp) > 0 { - runes = exp - } - // We consider runes that combine backwards to be non-starters for the - // purpose of Stream-Safe Text Processing. - for _, r := range runes { - if cr := &chars[r]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward { - break - } - c.nLeadingNonStarters++ - } - for i := len(runes) - 1; i >= 0; i-- { - if cr := &chars[runes[i]]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward { - break - } - c.nTrailingNonStarters++ - } - if c.nTrailingNonStarters > 3 { - log.Fatalf("%U: Decomposition with more than 3 (%d) trailing modifiers (%U)", i, c.nTrailingNonStarters, runes) - } - - if isHangul(rune(i)) { - c.nTrailingNonStarters = 2 - if isHangulWithoutJamoT(rune(i)) { - c.nTrailingNonStarters = 1 - } - } - - if l, t := c.nLeadingNonStarters, c.nTrailingNonStarters; l > 0 && l != t { - log.Fatalf("%U: number of leading and trailing non-starters should be equal (%d vs %d)", i, l, t) - } - if t := c.nTrailingNonStarters; t > 3 { - log.Fatalf("%U: number of trailing non-starters is %d > 3", t) - } - } -} - -func printBytes(w io.Writer, b []byte, name string) { - fmt.Fprintf(w, "// %s: %d bytes\n", name, len(b)) - fmt.Fprintf(w, "var %s = [...]byte {", name) - for i, c := range b { - switch { - case i%64 == 0: - fmt.Fprintf(w, "\n// Bytes %x - %x\n", i, i+63) - case i%8 == 0: - fmt.Fprintf(w, "\n") - } - fmt.Fprintf(w, "0x%.2X, ", c) - } - fmt.Fprint(w, "\n}\n\n") -} - -// See forminfo.go for format. -func makeEntry(f *FormInfo, c *Char) uint16 { - e := uint16(0) - if r := c.codePoint; HangulBase <= r && r < HangulEnd { - e |= 0x40 - } - if f.combinesForward { - e |= 0x20 - } - if f.quickCheck[MDecomposed] == QCNo { - e |= 0x4 - } - switch f.quickCheck[MComposed] { - case QCYes: - case QCNo: - e |= 0x10 - case QCMaybe: - e |= 0x18 - default: - log.Fatalf("Illegal quickcheck value %v.", f.quickCheck[MComposed]) - } - e |= uint16(c.nTrailingNonStarters) - return e -} - -// decompSet keeps track of unique decompositions, grouped by whether -// the decomposition is followed by a trailing and/or leading CCC. -type decompSet [7]map[string]bool - -const ( - normalDecomp = iota - firstMulti - firstCCC - endMulti - firstLeadingCCC - firstCCCZeroExcept - firstStarterWithNLead - lastDecomp -) - -var cname = []string{"firstMulti", "firstCCC", "endMulti", "firstLeadingCCC", "firstCCCZeroExcept", "firstStarterWithNLead", "lastDecomp"} - -func makeDecompSet() decompSet { - m := decompSet{} - for i := range m { - m[i] = make(map[string]bool) - } - return m -} -func (m *decompSet) insert(key int, s string) { - m[key][s] = true -} - -func printCharInfoTables(w io.Writer) int { - mkstr := func(r rune, f *FormInfo) (int, string) { - d := f.expandedDecomp - s := string([]rune(d)) - if max := 1 << 6; len(s) >= max { - const msg = "%U: too many bytes in decomposition: %d >= %d" - log.Fatalf(msg, r, len(s), max) - } - head := uint8(len(s)) - if f.quickCheck[MComposed] != QCYes { - head |= 0x40 - } - if f.combinesForward { - head |= 0x80 - } - s = string([]byte{head}) + s - - lccc := ccc(d[0]) - tccc := ccc(d[len(d)-1]) - cc := ccc(r) - if cc != 0 && lccc == 0 && tccc == 0 { - log.Fatalf("%U: trailing and leading ccc are 0 for non-zero ccc %d", r, cc) - } - if tccc < lccc && lccc != 0 { - const msg = "%U: lccc (%d) must be <= tcc (%d)" - log.Fatalf(msg, r, lccc, tccc) - } - index := normalDecomp - nTrail := chars[r].nTrailingNonStarters - nLead := chars[r].nLeadingNonStarters - if tccc > 0 || lccc > 0 || nTrail > 0 { - tccc <<= 2 - tccc |= nTrail - s += string([]byte{tccc}) - index = endMulti - for _, r := range d[1:] { - if ccc(r) == 0 { - index = firstCCC - } - } - if lccc > 0 || nLead > 0 { - s += string([]byte{lccc}) - if index == firstCCC { - log.Fatalf("%U: multi-segment decomposition not supported for decompositions with leading CCC != 0", r) - } - index = firstLeadingCCC - } - if cc != lccc { - if cc != 0 { - log.Fatalf("%U: for lccc != ccc, expected ccc to be 0; was %d", r, cc) - } - index = firstCCCZeroExcept - } - } else if len(d) > 1 { - index = firstMulti - } - return index, s - } - - decompSet := makeDecompSet() - const nLeadStr = "\x00\x01" // 0-byte length and tccc with nTrail. - decompSet.insert(firstStarterWithNLead, nLeadStr) - - // Store the uniqued decompositions in a byte buffer, - // preceded by their byte length. - for _, c := range chars { - for _, f := range c.forms { - if len(f.expandedDecomp) == 0 { - continue - } - if f.combinesBackward { - log.Fatalf("%U: combinesBackward and decompose", c.codePoint) - } - index, s := mkstr(c.codePoint, &f) - decompSet.insert(index, s) - } - } - - decompositions := bytes.NewBuffer(make([]byte, 0, 10000)) - size := 0 - positionMap := make(map[string]uint16) - decompositions.WriteString("\000") - fmt.Fprintln(w, "const (") - for i, m := range decompSet { - sa := []string{} - for s := range m { - sa = append(sa, s) - } - sort.Strings(sa) - for _, s := range sa { - p := decompositions.Len() - decompositions.WriteString(s) - positionMap[s] = uint16(p) - } - if cname[i] != "" { - fmt.Fprintf(w, "%s = 0x%X\n", cname[i], decompositions.Len()) - } - } - fmt.Fprintln(w, "maxDecomp = 0x8000") - fmt.Fprintln(w, ")") - b := decompositions.Bytes() - printBytes(w, b, "decomps") - size += len(b) - - varnames := []string{"nfc", "nfkc"} - for i := 0; i < FNumberOfFormTypes; i++ { - trie := triegen.NewTrie(varnames[i]) - - for r, c := range chars { - f := c.forms[i] - d := f.expandedDecomp - if len(d) != 0 { - _, key := mkstr(c.codePoint, &f) - trie.Insert(rune(r), uint64(positionMap[key])) - if c.ccc != ccc(d[0]) { - // We assume the lead ccc of a decomposition !=0 in this case. - if ccc(d[0]) == 0 { - log.Fatalf("Expected leading CCC to be non-zero; ccc is %d", c.ccc) - } - } - } else if c.nLeadingNonStarters > 0 && len(f.expandedDecomp) == 0 && c.ccc == 0 && !f.combinesBackward { - // Handle cases where it can't be detected that the nLead should be equal - // to nTrail. - trie.Insert(c.codePoint, uint64(positionMap[nLeadStr])) - } else if v := makeEntry(&f, &c)<<8 | uint16(c.ccc); v != 0 { - trie.Insert(c.codePoint, uint64(0x8000|v)) - } - } - sz, err := trie.Gen(w, triegen.Compact(&normCompacter{name: varnames[i]})) - if err != nil { - log.Fatal(err) - } - size += sz - } - return size -} - -func contains(sa []string, s string) bool { - for _, a := range sa { - if a == s { - return true - } - } - return false -} - -func makeTables() { - w := &bytes.Buffer{} - - size := 0 - if *tablelist == "" { - return - } - list := strings.Split(*tablelist, ",") - if *tablelist == "all" { - list = []string{"recomp", "info"} - } - - // Compute maximum decomposition size. - max := 0 - for _, c := range chars { - if n := len(string(c.forms[FCompatibility].expandedDecomp)); n > max { - max = n - } - } - - fmt.Fprintln(w, "const (") - fmt.Fprintln(w, "\t// Version is the Unicode edition from which the tables are derived.") - fmt.Fprintf(w, "\tVersion = %q\n", gen.UnicodeVersion()) - fmt.Fprintln(w) - fmt.Fprintln(w, "\t// MaxTransformChunkSize indicates the maximum number of bytes that Transform") - fmt.Fprintln(w, "\t// may need to write atomically for any Form. Making a destination buffer at") - fmt.Fprintln(w, "\t// least this size ensures that Transform can always make progress and that") - fmt.Fprintln(w, "\t// the user does not need to grow the buffer on an ErrShortDst.") - fmt.Fprintf(w, "\tMaxTransformChunkSize = %d+maxNonStarters*4\n", len(string(0x034F))+max) - fmt.Fprintln(w, ")\n") - - // Print the CCC remap table. - size += len(cccMap) - fmt.Fprintf(w, "var ccc = [%d]uint8{", len(cccMap)) - for i := 0; i < len(cccMap); i++ { - if i%8 == 0 { - fmt.Fprintln(w) - } - fmt.Fprintf(w, "%3d, ", cccMap[uint8(i)]) - } - fmt.Fprintln(w, "\n}\n") - - if contains(list, "info") { - size += printCharInfoTables(w) - } - - if contains(list, "recomp") { - // Note that we use 32 bit keys, instead of 64 bit. - // This clips the bits of three entries, but we know - // this won't cause a collision. The compiler will catch - // any changes made to UnicodeData.txt that introduces - // a collision. - // Note that the recomposition map for NFC and NFKC - // are identical. - - // Recomposition map - nrentries := 0 - for _, c := range chars { - f := c.forms[FCanonical] - if !f.isOneWay && len(f.decomp) > 0 { - nrentries++ - } - } - sz := nrentries * 8 - size += sz - fmt.Fprintf(w, "// recompMap: %d bytes (entries only)\n", sz) - fmt.Fprintln(w, "var recompMap = map[uint32]rune{") - for i, c := range chars { - f := c.forms[FCanonical] - d := f.decomp - if !f.isOneWay && len(d) > 0 { - key := uint32(uint16(d[0]))<<16 + uint32(uint16(d[1])) - fmt.Fprintf(w, "0x%.8X: 0x%.4X,\n", key, i) - } - } - fmt.Fprintf(w, "}\n\n") - } - - fmt.Fprintf(w, "// Total size of tables: %dKB (%d bytes)\n", (size+512)/1024, size) - gen.WriteGoFile("tables.go", "norm", w.Bytes()) -} - -func printChars() { - if *verbose { - for _, c := range chars { - if !c.isValid() || c.state == SMissing { - continue - } - fmt.Println(c) - } - } -} - -// verifyComputed does various consistency tests. -func verifyComputed() { - for i, c := range chars { - for _, f := range c.forms { - isNo := (f.quickCheck[MDecomposed] == QCNo) - if (len(f.decomp) > 0) != isNo && !isHangul(rune(i)) { - log.Fatalf("%U: NF*D QC must be No if rune decomposes", i) - } - - isMaybe := f.quickCheck[MComposed] == QCMaybe - if f.combinesBackward != isMaybe { - log.Fatalf("%U: NF*C QC must be Maybe if combinesBackward", i) - } - if len(f.decomp) > 0 && f.combinesForward && isMaybe { - log.Fatalf("%U: NF*C QC must be Yes or No if combinesForward and decomposes", i) - } - - if len(f.expandedDecomp) != 0 { - continue - } - if a, b := c.nLeadingNonStarters > 0, (c.ccc > 0 || f.combinesBackward); a != b { - // We accept these runes to be treated differently (it only affects - // segment breaking in iteration, most likely on improper use), but - // reconsider if more characters are added. - // U+FF9E HALFWIDTH KATAKANA VOICED SOUND MARK;Lm;0;L; 3099;;;;N;;;;; - // U+FF9F HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK;Lm;0;L; 309A;;;;N;;;;; - // U+3133 HANGUL LETTER KIYEOK-SIOS;Lo;0;L; 11AA;;;;N;HANGUL LETTER GIYEOG SIOS;;;; - // U+318E HANGUL LETTER ARAEAE;Lo;0;L; 11A1;;;;N;HANGUL LETTER ALAE AE;;;; - // U+FFA3 HALFWIDTH HANGUL LETTER KIYEOK-SIOS;Lo;0;L; 3133;;;;N;HALFWIDTH HANGUL LETTER GIYEOG SIOS;;;; - // U+FFDC HALFWIDTH HANGUL LETTER I;Lo;0;L; 3163;;;;N;;;;; - if i != 0xFF9E && i != 0xFF9F && !(0x3133 <= i && i <= 0x318E) && !(0xFFA3 <= i && i <= 0xFFDC) { - log.Fatalf("%U: nLead was %v; want %v", i, a, b) - } - } - } - nfc := c.forms[FCanonical] - nfkc := c.forms[FCompatibility] - if nfc.combinesBackward != nfkc.combinesBackward { - log.Fatalf("%U: Cannot combine combinesBackward\n", c.codePoint) - } - } -} - -// Use values in DerivedNormalizationProps.txt to compare against the -// values we computed. -// DerivedNormalizationProps.txt has form: -// 00C0..00C5 ; NFD_QC; N # ... -// 0374 ; NFD_QC; N # ... -// See http://unicode.org/reports/tr44/ for full explanation -func testDerived() { - f := gen.OpenUCDFile("DerivedNormalizationProps.txt") - defer f.Close() - p := ucd.New(f) - for p.Next() { - r := p.Rune(0) - c := &chars[r] - - var ftype, mode int - qt := p.String(1) - switch qt { - case "NFC_QC": - ftype, mode = FCanonical, MComposed - case "NFD_QC": - ftype, mode = FCanonical, MDecomposed - case "NFKC_QC": - ftype, mode = FCompatibility, MComposed - case "NFKD_QC": - ftype, mode = FCompatibility, MDecomposed - default: - continue - } - var qr QCResult - switch p.String(2) { - case "Y": - qr = QCYes - case "N": - qr = QCNo - case "M": - qr = QCMaybe - default: - log.Fatalf(`Unexpected quick check value "%s"`, p.String(2)) - } - if got := c.forms[ftype].quickCheck[mode]; got != qr { - log.Printf("%U: FAILED %s (was %v need %v)\n", r, qt, got, qr) - } - c.forms[ftype].verified[mode] = true - } - if err := p.Err(); err != nil { - log.Fatal(err) - } - // Any unspecified value must be QCYes. Verify this. - for i, c := range chars { - for j, fd := range c.forms { - for k, qr := range fd.quickCheck { - if !fd.verified[k] && qr != QCYes { - m := "%U: FAIL F:%d M:%d (was %v need Yes) %s\n" - log.Printf(m, i, j, k, qr, c.name) - } - } - } - } -} - -var testHeader = `const ( - Yes = iota - No - Maybe -) - -type formData struct { - qc uint8 - combinesForward bool - decomposition string -} - -type runeData struct { - r rune - ccc uint8 - nLead uint8 - nTrail uint8 - f [2]formData // 0: canonical; 1: compatibility -} - -func f(qc uint8, cf bool, dec string) [2]formData { - return [2]formData{{qc, cf, dec}, {qc, cf, dec}} -} - -func g(qc, qck uint8, cf, cfk bool, d, dk string) [2]formData { - return [2]formData{{qc, cf, d}, {qck, cfk, dk}} -} - -var testData = []runeData{ -` - -func printTestdata() { - type lastInfo struct { - ccc uint8 - nLead uint8 - nTrail uint8 - f string - } - - last := lastInfo{} - w := &bytes.Buffer{} - fmt.Fprintf(w, testHeader) - for r, c := range chars { - f := c.forms[FCanonical] - qc, cf, d := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp) - f = c.forms[FCompatibility] - qck, cfk, dk := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp) - s := "" - if d == dk && qc == qck && cf == cfk { - s = fmt.Sprintf("f(%s, %v, %q)", qc, cf, d) - } else { - s = fmt.Sprintf("g(%s, %s, %v, %v, %q, %q)", qc, qck, cf, cfk, d, dk) - } - current := lastInfo{c.ccc, c.nLeadingNonStarters, c.nTrailingNonStarters, s} - if last != current { - fmt.Fprintf(w, "\t{0x%x, %d, %d, %d, %s},\n", r, c.origCCC, c.nLeadingNonStarters, c.nTrailingNonStarters, s) - last = current - } - } - fmt.Fprintln(w, "}") - gen.WriteGoFile("data_test.go", "norm", w.Bytes()) -} diff --git a/vendor/golang.org/x/text/unicode/norm/triegen.go b/vendor/golang.org/x/text/unicode/norm/triegen.go deleted file mode 100644 index 45d711900..000000000 --- a/vendor/golang.org/x/text/unicode/norm/triegen.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -// Trie table generator. -// Used by make*tables tools to generate a go file with trie data structures -// for mapping UTF-8 to a 16-bit value. All but the last byte in a UTF-8 byte -// sequence are used to lookup offsets in the index table to be used for the -// next byte. The last byte is used to index into a table with 16-bit values. - -package main - -import ( - "fmt" - "io" -) - -const maxSparseEntries = 16 - -type normCompacter struct { - sparseBlocks [][]uint64 - sparseOffset []uint16 - sparseCount int - name string -} - -func mostFrequentStride(a []uint64) int { - counts := make(map[int]int) - var v int - for _, x := range a { - if stride := int(x) - v; v != 0 && stride >= 0 { - counts[stride]++ - } - v = int(x) - } - var maxs, maxc int - for stride, cnt := range counts { - if cnt > maxc || (cnt == maxc && stride < maxs) { - maxs, maxc = stride, cnt - } - } - return maxs -} - -func countSparseEntries(a []uint64) int { - stride := mostFrequentStride(a) - var v, count int - for _, tv := range a { - if int(tv)-v != stride { - if tv != 0 { - count++ - } - } - v = int(tv) - } - return count -} - -func (c *normCompacter) Size(v []uint64) (sz int, ok bool) { - if n := countSparseEntries(v); n <= maxSparseEntries { - return (n+1)*4 + 2, true - } - return 0, false -} - -func (c *normCompacter) Store(v []uint64) uint32 { - h := uint32(len(c.sparseOffset)) - c.sparseBlocks = append(c.sparseBlocks, v) - c.sparseOffset = append(c.sparseOffset, uint16(c.sparseCount)) - c.sparseCount += countSparseEntries(v) + 1 - return h -} - -func (c *normCompacter) Handler() string { - return c.name + "Sparse.lookup" -} - -func (c *normCompacter) Print(w io.Writer) (retErr error) { - p := func(f string, x ...interface{}) { - if _, err := fmt.Fprintf(w, f, x...); retErr == nil && err != nil { - retErr = err - } - } - - ls := len(c.sparseBlocks) - p("// %sSparseOffset: %d entries, %d bytes\n", c.name, ls, ls*2) - p("var %sSparseOffset = %#v\n\n", c.name, c.sparseOffset) - - ns := c.sparseCount - p("// %sSparseValues: %d entries, %d bytes\n", c.name, ns, ns*4) - p("var %sSparseValues = [%d]valueRange {", c.name, ns) - for i, b := range c.sparseBlocks { - p("\n// Block %#x, offset %#x", i, c.sparseOffset[i]) - var v int - stride := mostFrequentStride(b) - n := countSparseEntries(b) - p("\n{value:%#04x,lo:%#02x},", stride, uint8(n)) - for i, nv := range b { - if int(nv)-v != stride { - if v != 0 { - p(",hi:%#02x},", 0x80+i-1) - } - if nv != 0 { - p("\n{value:%#04x,lo:%#02x", nv, 0x80+i) - } - } - v = int(nv) - } - if v != 0 { - p(",hi:%#02x},", 0x80+len(b)-1) - } - } - p("\n}\n\n") - return -} diff --git a/vendor/vendor.json b/vendor/vendor.json index 162963b77..c0d4a5023 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -1498,95 +1498,95 @@ { "checksumSHA1": "Mr4ur60bgQJnQFfJY0dGtwWwMPE=", "path": "golang.org/x/text/encoding", - "revision": "a8b38433e35b65ba247bb267317037dee1b70cea", - "revisionTime": "2016-10-19T13:35:53Z" + "revision": "2bf8f2a19ec09c670e931282edfe6567f6be21c9", + "revisionTime": "2017-06-27T21:03:49Z" }, { - "checksumSHA1": "gJG/5S8KrCA1QGkIkpa5a/wnmy4=", + "checksumSHA1": "DSdlK4MKI/a3U8Zaee2XKBe01Fo=", "path": "golang.org/x/text/encoding/charmap", - "revision": "a8b38433e35b65ba247bb267317037dee1b70cea", - "revisionTime": "2016-10-19T13:35:53Z" + "revision": "2bf8f2a19ec09c670e931282edfe6567f6be21c9", + "revisionTime": "2017-06-27T21:03:49Z" }, { - "checksumSHA1": "mI8YM2LehMxYDcauq5loMZr1pP8=", + "checksumSHA1": "z7tgTCQT62mHxtNMi/AXui/FAfQ=", "path": "golang.org/x/text/encoding/htmlindex", - "revision": "a8b38433e35b65ba247bb267317037dee1b70cea", - "revisionTime": "2016-10-19T13:35:53Z" + "revision": "2bf8f2a19ec09c670e931282edfe6567f6be21c9", + "revisionTime": "2017-06-27T21:03:49Z" }, { "checksumSHA1": "zeHyHebIZl1tGuwGllIhjfci+wI=", "path": "golang.org/x/text/encoding/internal", - "revision": "a8b38433e35b65ba247bb267317037dee1b70cea", - "revisionTime": "2016-10-19T13:35:53Z" + "revision": "2bf8f2a19ec09c670e931282edfe6567f6be21c9", + "revisionTime": "2017-06-27T21:03:49Z" }, { - "checksumSHA1": "TF4hoIqHVEAvOq67rfnSLSkcZ1Y=", + "checksumSHA1": "7kYqxy64WhMjFIFZgN7tJ3lbKxM=", "path": "golang.org/x/text/encoding/internal/identifier", - "revision": "a8b38433e35b65ba247bb267317037dee1b70cea", - "revisionTime": "2016-10-19T13:35:53Z" + "revision": "2bf8f2a19ec09c670e931282edfe6567f6be21c9", + "revisionTime": "2017-06-27T21:03:49Z" }, { - "checksumSHA1": "HeZV82ktrmgyAaYLtNFS0qYgspI=", + "checksumSHA1": "2YqVpmvjWGEBATyUphTP1MS34JE=", "path": "golang.org/x/text/encoding/japanese", - "revision": "a8b38433e35b65ba247bb267317037dee1b70cea", - "revisionTime": "2016-10-19T13:35:53Z" + "revision": "2bf8f2a19ec09c670e931282edfe6567f6be21c9", + "revisionTime": "2017-06-27T21:03:49Z" }, { - "checksumSHA1": "8y87WJz3OkDWtPCIXxJcYpo+OY8=", + "checksumSHA1": "+ErWCAdaMwO4PLtrk9D/Hh+7oQM=", "path": "golang.org/x/text/encoding/korean", - "revision": "a8b38433e35b65ba247bb267317037dee1b70cea", - "revisionTime": "2016-10-19T13:35:53Z" + "revision": "2bf8f2a19ec09c670e931282edfe6567f6be21c9", + "revisionTime": "2017-06-27T21:03:49Z" }, { - "checksumSHA1": "WYfmebIyX5Zae8NUfu9PsQjQff0=", + "checksumSHA1": "mTuZi5urYwgDIO8+Gfql2pv8Vwg=", "path": "golang.org/x/text/encoding/simplifiedchinese", - "revision": "a8b38433e35b65ba247bb267317037dee1b70cea", - "revisionTime": "2016-10-19T13:35:53Z" + "revision": "2bf8f2a19ec09c670e931282edfe6567f6be21c9", + "revisionTime": "2017-06-27T21:03:49Z" }, { - "checksumSHA1": "KKqYmi6fxt3r3uo4lExss2yTMbs=", + "checksumSHA1": "D+VI4j0Wjzr8SeupWdOB5KBdFOw=", "path": "golang.org/x/text/encoding/traditionalchinese", - "revision": "a8b38433e35b65ba247bb267317037dee1b70cea", - "revisionTime": "2016-10-19T13:35:53Z" + "revision": "2bf8f2a19ec09c670e931282edfe6567f6be21c9", + "revisionTime": "2017-06-27T21:03:49Z" }, { "checksumSHA1": "G9LfJI9gySazd+MyyC6QbTHx4to=", "path": "golang.org/x/text/encoding/unicode", - "revision": "a8b38433e35b65ba247bb267317037dee1b70cea", - "revisionTime": "2016-10-19T13:35:53Z" + "revision": "2bf8f2a19ec09c670e931282edfe6567f6be21c9", + "revisionTime": "2017-06-27T21:03:49Z" }, { "checksumSHA1": "hyNCcTwMQnV6/MK8uUW9E5H0J0M=", "path": "golang.org/x/text/internal/tag", - "revision": "a8b38433e35b65ba247bb267317037dee1b70cea", - "revisionTime": "2016-10-19T13:35:53Z" + "revision": "2bf8f2a19ec09c670e931282edfe6567f6be21c9", + "revisionTime": "2017-06-27T21:03:49Z" }, { "checksumSHA1": "Qk7dljcrEK1BJkAEZguxAbG9dSo=", "path": "golang.org/x/text/internal/utf8internal", - "revision": "a8b38433e35b65ba247bb267317037dee1b70cea", - "revisionTime": "2016-10-19T13:35:53Z" + "revision": "2bf8f2a19ec09c670e931282edfe6567f6be21c9", + "revisionTime": "2017-06-27T21:03:49Z" }, { - "checksumSHA1": "bsNFI/kfmF0p43jLKiMYRqw9Dfs=", + "checksumSHA1": "SnP28TAvq7k08OmkHClZDYFEWww=", "path": "golang.org/x/text/language", - "revision": "a8b38433e35b65ba247bb267317037dee1b70cea", - "revisionTime": "2016-10-19T13:35:53Z" + "revision": "2bf8f2a19ec09c670e931282edfe6567f6be21c9", + "revisionTime": "2017-06-27T21:03:49Z" }, { "checksumSHA1": "IV4MN7KGBSocu/5NR3le3sxup4Y=", "path": "golang.org/x/text/runes", - "revision": "a8b38433e35b65ba247bb267317037dee1b70cea", - "revisionTime": "2016-10-19T13:35:53Z" + "revision": "2bf8f2a19ec09c670e931282edfe6567f6be21c9", + "revisionTime": "2017-06-27T21:03:49Z" }, { "checksumSHA1": "ziMb9+ANGRJSSIuxYdRbA+cDRBQ=", "path": "golang.org/x/text/transform", - "revision": "a8b38433e35b65ba247bb267317037dee1b70cea", - "revisionTime": "2016-10-19T13:35:53Z" + "revision": "2bf8f2a19ec09c670e931282edfe6567f6be21c9", + "revisionTime": "2017-06-27T21:03:49Z" }, { - "checksumSHA1": "Anof4bt0AU+Sa3R8Rq0KBnlpbaQ=", + "checksumSHA1": "kKylzIrLEnH8NKyeVAL0dq5gjVQ=", "path": "golang.org/x/text/unicode/norm", "revision": "2bf8f2a19ec09c670e931282edfe6567f6be21c9", "revisionTime": "2017-06-27T21:03:49Z"