1 /*
2 * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
3 * Copyright 2012 Garrett D'Amore <garrett@damore.org> All rights reserved.
4 * Copyright 2015 John Marino <draco@marino.st>
5 *
6 * This source code is derived from the illumos localedef command, and
7 * provided under BSD-style license terms by Nexenta Systems, Inc.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 *
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*
33 * The functions in this file convert from the standard multibyte forms
34 * to the wide character forms used internally by libc. Unfortunately,
35 * this approach means that we need a method for each and every encoding.
36 */
37 #include <sys/cdefs.h>
38 __FBSDID("$FreeBSD$");
39
40 #include <ctype.h>
41 #include <stdlib.h>
42 #include <wchar.h>
43 #include <string.h>
44 #include <sys/types.h>
45 #include "localedef.h"
46
47 static int towide_none(wchar_t *, const char *, unsigned);
48 static int towide_utf8(wchar_t *, const char *, unsigned);
49 static int towide_big5(wchar_t *, const char *, unsigned);
50 static int towide_gbk(wchar_t *, const char *, unsigned);
51 static int towide_gb2312(wchar_t *, const char *, unsigned);
52 static int towide_gb18030(wchar_t *, const char *, unsigned);
53 static int towide_mskanji(wchar_t *, const char *, unsigned);
54 static int towide_euccn(wchar_t *, const char *, unsigned);
55 static int towide_eucjp(wchar_t *, const char *, unsigned);
56 static int towide_euckr(wchar_t *, const char *, unsigned);
57 static int towide_euctw(wchar_t *, const char *, unsigned);
58
59 static int tomb_none(char *, wchar_t);
60 static int tomb_utf8(char *, wchar_t);
61 static int tomb_mbs(char *, wchar_t);
62
63 static int (*_towide)(wchar_t *, const char *, unsigned) = towide_none;
64 static int (*_tomb)(char *, wchar_t) = tomb_none;
65 static char _encoding_buffer[20] = {'N','O','N','E'};
66 static const char *_encoding = _encoding_buffer;
67 static int _nbits = 7;
68
69 /*
70 * Table of supported encodings. We only bother to list the multibyte
71 * encodings here, because single byte locales are handed by "NONE".
72 */
73 static struct {
74 const char *name;
75 /* the name that the underlying libc implemenation uses */
76 const char *cname;
77 /* the maximum number of bits required for priorities */
78 int nbits;
79 int (*towide)(wchar_t *, const char *, unsigned);
80 int (*tomb)(char *, wchar_t);
81 } mb_encodings[] = {
82 /*
83 * UTF8 values max out at 0x1fffff (although in theory there could
84 * be later extensions, but it won't happen.) This means we only need
85 * 21 bits to be able to encode the entire range of priorities.
86 */
87 { "UTF-8", "UTF-8", 21, towide_utf8, tomb_utf8 },
88 { "UTF8", "UTF-8", 21, towide_utf8, tomb_utf8 },
89 { "utf8", "UTF-8", 21, towide_utf8, tomb_utf8 },
90 { "utf-8", "UTF-8", 21, towide_utf8, tomb_utf8 },
91
92 { "EUC-CN", "EUC-CN", 16, towide_euccn, tomb_mbs },
93 { "eucCN", "EUC-CN", 16, towide_euccn, tomb_mbs },
94 /*
95 * Becuase the 3-byte form of EUC-JP use the same leading byte,
96 * only 17 bits required to provide unique priorities. (The low
97 * bit of that first byte is set.) By setting this value low,
98 * we can get by with only 3 bytes in the strxfrm expansion.
99 */
100 { "EUC-JP", "EUC-JP", 17, towide_eucjp, tomb_mbs },
101 { "eucJP", "EUC-JP", 17, towide_eucjp, tomb_mbs },
102
103 { "EUC-KR", "EUC-KR", 16, towide_euckr, tomb_mbs },
104 { "eucKR", "EUC-KR", 16, towide_euckr, tomb_mbs },
105 /*
106 * EUC-TW uses 2 bytes most of the time, but 4 bytes if the
107 * high order byte is 0x8E. However, with 4 byte encodings,
108 * the third byte will be A0-B0. So we only need to consider
109 * the lower order 24 bits for collation.
110 */
111 { "EUC-TW", "EUC-TW", 24, towide_euctw, tomb_mbs },
112 { "eucTW", "EUC-TW", 24, towide_euctw, tomb_mbs },
113
114 { "MS_Kanji", "MSKanji", 16, towide_mskanji, tomb_mbs },
115 { "MSKanji", "MSKanji", 16, towide_mskanji, tomb_mbs },
116 { "PCK", "MSKanji", 16, towide_mskanji, tomb_mbs },
117 { "SJIS", "MSKanji", 16, towide_mskanji, tomb_mbs },
118 { "Shift_JIS", "MSKanji", 16, towide_mskanji, tomb_mbs },
119
120 { "BIG5", "BIG5", 16, towide_big5, tomb_mbs },
121 { "big5", "BIG5", 16, towide_big5, tomb_mbs },
122 { "Big5", "BIG5", 16, towide_big5, tomb_mbs },
123
124 { "GBK", "GBK", 16, towide_gbk, tomb_mbs },
125
126 /*
127 * GB18030 can get away with just 31 bits. This is because the
128 * high order bit is always set for 4 byte values, and the
129 * at least one of the other bits in that 4 byte value will
130 * be non-zero.
131 */
132 { "GB18030", "GB18030", 31, towide_gb18030, tomb_mbs },
133
134 /*
135 * This should probably be an aliase for euc-cn, or vice versa.
136 */
137 { "GB2312", "GB2312", 16, towide_gb2312, tomb_mbs },
138
139 { NULL, NULL, 0, 0, 0 },
140 };
141
142 static char *
show_mb(const char * mb)143 show_mb(const char *mb)
144 {
145 static char buf[64];
146
147 /* ASCII stuff we just print */
148 if (isascii(*mb) && isgraph(*mb)) {
149 buf[0] = *mb;
150 buf[1] = 0;
151 return (buf);
152 }
153 buf[0] = 0;
154 while (*mb != 0) {
155 char scr[8];
156 (void) snprintf(scr, sizeof (scr), "\\x%02x", *mb);
157 (void) strlcat(buf, scr, sizeof (buf));
158 mb++;
159 }
160 return (buf);
161 }
162
163 static char *widemsg;
164
165 void
werr(const char * fmt,...)166 werr(const char *fmt, ...)
167 {
168 char *msg;
169
170 va_list va;
171 va_start(va, fmt);
172 (void) vasprintf(&msg, fmt, va);
173 va_end(va);
174
175 free(widemsg);
176 widemsg = msg;
177 }
178
179 /*
180 * This is used for 8-bit encodings.
181 */
182 int
towide_none(wchar_t * c,const char * mb,unsigned n __unused)183 towide_none(wchar_t *c, const char *mb, unsigned n __unused)
184 {
185 if (mb_cur_max != 1) {
186 werr("invalid or unsupported multibyte locale");
187 return (-1);
188 }
189 *c = (uint8_t)*mb;
190 return (1);
191 }
192
193 int
tomb_none(char * mb,wchar_t wc)194 tomb_none(char *mb, wchar_t wc)
195 {
196 if (mb_cur_max != 1) {
197 werr("invalid or unsupported multibyte locale");
198 return (-1);
199 }
200 *(uint8_t *)mb = (wc & 0xff);
201 mb[1] = 0;
202 return (1);
203 }
204
205 /*
206 * UTF-8 stores wide characters in UTF-32 form.
207 */
208 int
towide_utf8(wchar_t * wc,const char * mb,unsigned n)209 towide_utf8(wchar_t *wc, const char *mb, unsigned n)
210 {
211 wchar_t c;
212 int nb;
213 wchar_t lv; /* lowest legal value */
214 int i;
215 const uint8_t *s = (const uint8_t *)mb;
216
217 c = *s;
218
219 if ((c & 0x80) == 0) {
220 /* 7-bit ASCII */
221 *wc = c;
222 return (1);
223 } else if ((c & 0xe0) == 0xc0) {
224 /* u80-u7ff - two bytes encoded */
225 nb = 2;
226 lv = 0x80;
227 c &= ~0xe0;
228 } else if ((c & 0xf0) == 0xe0) {
229 /* u800-uffff - three bytes encoded */
230 nb = 3;
231 lv = 0x800;
232 c &= ~0xf0;
233 } else if ((c & 0xf8) == 0xf0) {
234 /* u1000-u1fffff - four bytes encoded */
235 nb = 4;
236 lv = 0x1000;
237 c &= ~0xf8;
238 } else {
239 /* 5 and 6 byte encodings are not legal unicode */
240 werr("utf8 encoding too large (%s)", show_mb(mb));
241 return (-1);
242 }
243 if (nb > (int)n) {
244 werr("incomplete utf8 sequence (%s)", show_mb(mb));
245 return (-1);
246 }
247
248 for (i = 1; i < nb; i++) {
249 if (((s[i]) & 0xc0) != 0x80) {
250 werr("illegal utf8 byte (%x)", s[i]);
251 return (-1);
252 }
253 c <<= 6;
254 c |= (s[i] & 0x3f);
255 }
256
257 if (c < lv) {
258 werr("illegal redundant utf8 encoding (%s)", show_mb(mb));
259 return (-1);
260 }
261 *wc = c;
262 return (nb);
263 }
264
265 int
tomb_utf8(char * mb,wchar_t wc)266 tomb_utf8(char *mb, wchar_t wc)
267 {
268 uint8_t *s = (uint8_t *)mb;
269 uint8_t msk;
270 int cnt;
271 int i;
272
273 if (wc <= 0x7f) {
274 s[0] = wc & 0x7f;
275 s[1] = 0;
276 return (1);
277 }
278 if (wc <= 0x7ff) {
279 cnt = 2;
280 msk = 0xc0;
281 } else if (wc <= 0xffff) {
282 cnt = 3;
283 msk = 0xe0;
284 } else if (wc <= 0x1fffff) {
285 cnt = 4;
286 msk = 0xf0;
287 } else {
288 werr("illegal uf8 char (%x)", wc);
289 return (-1);
290 }
291 for (i = cnt - 1; i; i--) {
292 s[i] = (wc & 0x3f) | 0x80;
293 wc >>= 6;
294 }
295 s[0] = (msk) | wc;
296 s[cnt] = 0;
297 return (cnt);
298 }
299
300 /*
301 * Several encodings share a simplistic dual byte encoding. In these
302 * forms, they all indicate that a two byte sequence is to be used if
303 * the first byte has its high bit set. They all store this simple
304 * encoding as a 16-bit value, although a great many of the possible
305 * code points are not used in most character sets. This gives a possible
306 * set of just over 32,000 valid code points.
307 *
308 * 0x00 - 0x7f - 1 byte encoding
309 * 0x80 - 0x7fff - illegal
310 * 0x8000 - 0xffff - 2 byte encoding
311 */
312
313 static int
towide_dbcs(wchar_t * wc,const char * mb,unsigned n)314 towide_dbcs(wchar_t *wc, const char *mb, unsigned n)
315 {
316 wchar_t c;
317
318 c = *(const uint8_t *)mb;
319
320 if ((c & 0x80) == 0) {
321 /* 7-bit */
322 *wc = c;
323 return (1);
324 }
325 if (n < 2) {
326 werr("incomplete character sequence (%s)", show_mb(mb));
327 return (-1);
328 }
329
330 /* Store both bytes as a single 16-bit wide. */
331 c <<= 8;
332 c |= (uint8_t)(mb[1]);
333 *wc = c;
334 return (2);
335 }
336
337 /*
338 * Most multibyte locales just convert the wide character to the multibyte
339 * form by stripping leading null bytes, and writing the 32-bit quantity
340 * in big-endian order.
341 */
342 int
tomb_mbs(char * mb,wchar_t wc)343 tomb_mbs(char *mb, wchar_t wc)
344 {
345 uint8_t *s = (uint8_t *)mb;
346 int n = 0, c;
347
348 if ((wc & 0xff000000U) != 0) {
349 n = 4;
350 } else if ((wc & 0x00ff0000U) != 0) {
351 n = 3;
352 } else if ((wc & 0x0000ff00U) != 0) {
353 n = 2;
354 } else {
355 n = 1;
356 }
357 c = n;
358 while (n) {
359 n--;
360 s[n] = wc & 0xff;
361 wc >>= 8;
362 }
363 /* ensure null termination */
364 s[c] = 0;
365 return (c);
366 }
367
368
369 /*
370 * big5 is a simple dual byte character set.
371 */
372 int
towide_big5(wchar_t * wc,const char * mb,unsigned n)373 towide_big5(wchar_t *wc, const char *mb, unsigned n)
374 {
375 return (towide_dbcs(wc, mb, n));
376 }
377
378 /*
379 * GBK encodes wides in the same way that big5 does, the high order
380 * bit of the first byte indicates a double byte character.
381 */
382 int
towide_gbk(wchar_t * wc,const char * mb,unsigned n)383 towide_gbk(wchar_t *wc, const char *mb, unsigned n)
384 {
385 return (towide_dbcs(wc, mb, n));
386 }
387
388 /*
389 * GB2312 is another DBCS. Its cleaner than others in that the second
390 * byte does not encode ASCII, but it supports characters.
391 */
392 int
towide_gb2312(wchar_t * wc,const char * mb,unsigned n)393 towide_gb2312(wchar_t *wc, const char *mb, unsigned n)
394 {
395 return (towide_dbcs(wc, mb, n));
396 }
397
398 /*
399 * GB18030. This encodes as 8, 16, or 32-bits.
400 * 7-bit values are in 1 byte, 4 byte sequences are used when
401 * the second byte encodes 0x30-39 and all other sequences are 2 bytes.
402 */
403 int
towide_gb18030(wchar_t * wc,const char * mb,unsigned n)404 towide_gb18030(wchar_t *wc, const char *mb, unsigned n)
405 {
406 wchar_t c;
407
408 c = *(const uint8_t *)mb;
409
410 if ((c & 0x80) == 0) {
411 /* 7-bit */
412 *wc = c;
413 return (1);
414 }
415 if (n < 2) {
416 werr("incomplete character sequence (%s)", show_mb(mb));
417 return (-1);
418 }
419
420 /* pull in the second byte */
421 c <<= 8;
422 c |= (uint8_t)(mb[1]);
423
424 if (((c & 0xff) >= 0x30) && ((c & 0xff) <= 0x39)) {
425 if (n < 4) {
426 werr("incomplete 4-byte character sequence (%s)",
427 show_mb(mb));
428 return (-1);
429 }
430 c <<= 8;
431 c |= (uint8_t)(mb[2]);
432 c <<= 8;
433 c |= (uint8_t)(mb[3]);
434 *wc = c;
435 return (4);
436 }
437
438 *wc = c;
439 return (2);
440 }
441
442 /*
443 * MS-Kanji (aka SJIS) is almost a clean DBCS like the others, but it
444 * also has a range of single byte characters above 0x80. (0xa1-0xdf).
445 */
446 int
towide_mskanji(wchar_t * wc,const char * mb,unsigned n)447 towide_mskanji(wchar_t *wc, const char *mb, unsigned n)
448 {
449 wchar_t c;
450
451 c = *(const uint8_t *)mb;
452
453 if ((c < 0x80) || ((c > 0xa0) && (c < 0xe0))) {
454 /* 7-bit */
455 *wc = c;
456 return (1);
457 }
458
459 if (n < 2) {
460 werr("incomplete character sequence (%s)", show_mb(mb));
461 return (-1);
462 }
463
464 /* Store both bytes as a single 16-bit wide. */
465 c <<= 8;
466 c |= (uint8_t)(mb[1]);
467 *wc = c;
468 return (2);
469 }
470
471 /*
472 * EUC forms. EUC encodings are "variable". FreeBSD carries some additional
473 * variable data to encode these, but we're going to treat each as independent
474 * instead. Its the only way we can sensibly move forward.
475 *
476 * Note that the way in which the different EUC forms vary is how wide
477 * CS2 and CS3 are and what the first byte of them is.
478 */
479 static int
towide_euc_impl(wchar_t * wc,const char * mb,unsigned n,uint8_t cs2,uint8_t cs2width,uint8_t cs3,uint8_t cs3width)480 towide_euc_impl(wchar_t *wc, const char *mb, unsigned n,
481 uint8_t cs2, uint8_t cs2width, uint8_t cs3, uint8_t cs3width)
482 {
483 int i;
484 int width = 2;
485 wchar_t c;
486
487 c = *(const uint8_t *)mb;
488
489 /*
490 * All variations of EUC encode 7-bit ASCII as one byte, and use
491 * additional bytes for more than that.
492 */
493 if ((c & 0x80) == 0) {
494 /* 7-bit */
495 *wc = c;
496 return (1);
497 }
498
499 /*
500 * All EUC variants reserve 0xa1-0xff to identify CS1, which
501 * is always two bytes wide. Note that unused CS will be zero,
502 * and that cannot be true because we know that the high order
503 * bit must be set.
504 */
505 if (c >= 0xa1) {
506 width = 2;
507 } else if (c == cs2) {
508 width = cs2width;
509 } else if (c == cs3) {
510 width = cs3width;
511 }
512
513 if ((int)n < width) {
514 werr("incomplete character sequence (%s)", show_mb(mb));
515 return (-1);
516 }
517
518 for (i = 1; i < width; i++) {
519 /* pull in the next byte */
520 c <<= 8;
521 c |= (uint8_t)(mb[i]);
522 }
523
524 *wc = c;
525 return (width);
526 }
527
528 /*
529 * EUC-CN encodes as follows:
530 *
531 * Code set 0 (ASCII): 0x21-0x7E
532 * Code set 1 (CNS 11643-1992 Plane 1): 0xA1A1-0xFEFE
533 * Code set 2: unused
534 * Code set 3: unused
535 */
536 int
towide_euccn(wchar_t * wc,const char * mb,unsigned n)537 towide_euccn(wchar_t *wc, const char *mb, unsigned n)
538 {
539 return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0));
540 }
541
542 /*
543 * EUC-JP encodes as follows:
544 *
545 * Code set 0 (ASCII or JIS X 0201-1976 Roman): 0x21-0x7E
546 * Code set 1 (JIS X 0208): 0xA1A1-0xFEFE
547 * Code set 2 (half-width katakana): 0x8EA1-0x8EDF
548 * Code set 3 (JIS X 0212-1990): 0x8FA1A1-0x8FFEFE
549 */
550 int
towide_eucjp(wchar_t * wc,const char * mb,unsigned n)551 towide_eucjp(wchar_t *wc, const char *mb, unsigned n)
552 {
553 return (towide_euc_impl(wc, mb, n, 0x8e, 2, 0x8f, 3));
554 }
555
556 /*
557 * EUC-KR encodes as follows:
558 *
559 * Code set 0 (ASCII or KS C 5636-1993): 0x21-0x7E
560 * Code set 1 (KS C 5601-1992): 0xA1A1-0xFEFE
561 * Code set 2: unused
562 * Code set 3: unused
563 */
564 int
towide_euckr(wchar_t * wc,const char * mb,unsigned n)565 towide_euckr(wchar_t *wc, const char *mb, unsigned n)
566 {
567 return (towide_euc_impl(wc, mb, n, 0, 0, 0, 0));
568 }
569
570 /*
571 * EUC-TW encodes as follows:
572 *
573 * Code set 0 (ASCII): 0x21-0x7E
574 * Code set 1 (CNS 11643-1992 Plane 1): 0xA1A1-0xFEFE
575 * Code set 2 (CNS 11643-1992 Planes 1-16): 0x8EA1A1A1-0x8EB0FEFE
576 * Code set 3: unused
577 */
578 int
towide_euctw(wchar_t * wc,const char * mb,unsigned n)579 towide_euctw(wchar_t *wc, const char *mb, unsigned n)
580 {
581 return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0));
582 }
583
584 /*
585 * Public entry points.
586 */
587
588 int
to_wide(wchar_t * wc,const char * mb)589 to_wide(wchar_t *wc, const char *mb)
590 {
591 /* this won't fail hard */
592 return (_towide(wc, mb, strlen(mb)));
593 }
594
595 int
to_mb(char * mb,wchar_t wc)596 to_mb(char *mb, wchar_t wc)
597 {
598 int rv;
599
600 if ((rv = _tomb(mb, wc)) < 0) {
601 errf(widemsg);
602 free(widemsg);
603 widemsg = NULL;
604 }
605 return (rv);
606 }
607
608 char *
to_mb_string(const wchar_t * wcs)609 to_mb_string(const wchar_t *wcs)
610 {
611 char *mbs;
612 char *ptr;
613 int len;
614
615 mbs = malloc((wcslen(wcs) * mb_cur_max) + 1);
616 if (mbs == NULL) {
617 errf("out of memory");
618 return (NULL);
619 }
620 ptr = mbs;
621 while (*wcs) {
622 if ((len = to_mb(ptr, *wcs)) < 0) {
623 INTERR;
624 free(mbs);
625 return (NULL);
626 }
627 wcs++;
628 ptr += len;
629 }
630 *ptr = 0;
631 return (mbs);
632 }
633
634 void
set_wide_encoding(const char * encoding)635 set_wide_encoding(const char *encoding)
636 {
637 int i;
638
639 _towide = towide_none;
640 _tomb = tomb_none;
641 _nbits = 8;
642
643 snprintf(_encoding_buffer, sizeof(_encoding_buffer), "NONE:%s",
644 encoding);
645 for (i = 0; mb_encodings[i].name; i++) {
646 if (strcasecmp(encoding, mb_encodings[i].name) == 0) {
647 _towide = mb_encodings[i].towide;
648 _tomb = mb_encodings[i].tomb;
649 _encoding = mb_encodings[i].cname;
650 _nbits = mb_encodings[i].nbits;
651 break;
652 }
653 }
654 }
655
656 const char *
get_wide_encoding(void)657 get_wide_encoding(void)
658 {
659 return (_encoding);
660 }
661
662 int
max_wide(void)663 max_wide(void)
664 {
665 return ((int)((1U << _nbits) - 1));
666 }
667