Fix long standing Asian multibyte charsets bug.

tatsuo-ishii · tatsuo-ishii · commit 804f6b8fc93f · 2005-12-24T09:35:36.000Z
See:

Subject: [HACKERS] bugs with certain Asian multibyte charsets
From: Tatsuo Ishii &lt;ishii@sraoss.co.jp&gt;
To: pgsql-hackers@postgresql.org
Date: Sat, 24 Dec 2005 18:25:33 +0900 (JST)

for more details/
diff --git a/src/backend/utils/mb/wchar.c b/src/backend/utils/mb/wchar.c
@@ -1,7 +1,7 @@
 /*
  * conversion functions between pg_wchar and multibyte streams.
  * Tatsuo Ishii
- * $PostgreSQL: pgsql/src/backend/utils/mb/wchar.c,v 1.47 2005/10/29 00:31:52 petere Exp $
+ * $PostgreSQL: pgsql/src/backend/utils/mb/wchar.c,v 1.48 2005/12/24 09:35:36 ishii Exp $
  *
  * WIN1250 client encoding updated by Pavel Behal
  *
@@ -58,34 +58,33 @@ pg_ascii_dsplen(const unsigned char *s)
 /*
  * EUC
  */
-
 static int	pg_euc2wchar_with_len
 			(const unsigned char *from, pg_wchar *to, int len)
 {
 	int			cnt = 0;
 
 	while (len > 0 && *from)
 	{
-		if (*from == SS2 && len >= 2)
+		if (*from == SS2 && len >= 2)	/* JIS X 0201 (so called "1 byte KANA") */
 		{
 			from++;
-			*to = 0xff & *from++;
+			*to = (SS2 << 8) | *from++;
 			len -= 2;
 		}
-		else if (*from == SS3 && len >= 3)
+		else if (*from == SS3 && len >= 3)		/* JIS X 0212 KANJI */
 		{
 			from++;
-			*to = *from++ << 8;
-			*to |= 0x3f & *from++;
+			*to = (SS3 << 16) | (*from++ << 8);
+			*to |= *from++;
 			len -= 3;
 		}
-		else if ((*from & 0x80) && len >= 2)
+		else if ((*from & 0x80) && len >= 2)	/* JIS X 0208 KANJI */
 		{
 			*to = *from++ << 8;
 			*to |= *from++;
 			len -= 2;
 		}
-		else
+		else	/* must be ASCII */
 		{
 			*to = *from++;
 			len--;
@@ -183,6 +182,7 @@ pg_euckr_dsplen(const unsigned char *s)
 
 /*
  * EUC_CN
+ *
  */
 static int	pg_euccn2wchar_with_len
 			(const unsigned char *from, pg_wchar *to, int len)
@@ -191,21 +191,21 @@ static int	pg_euccn2wchar_with_len
 
 	while (len > 0 && *from)
 	{
-		if (*from == SS2 && len >= 3)
+		if (*from == SS2 && len >= 3)	/* code set 2 (unused?) */
 		{
 			from++;
-			*to = 0x3f00 & (*from++ << 8);
-			*to = *from++;
+			*to = (SS2 << 16) | (*from++ << 8);
+			*to |= *from++;
 			len -= 3;
 		}
-		else if (*from == SS3 && len >= 3)
+		else if (*from == SS3 && len >= 3)		/* code set 3 (unsed ?) */
 		{
 			from++;
-			*to = *from++ << 8;
-			*to |= 0x3f & *from++;
+			*to = (SS3 << 16) | (*from++ << 8);
+			*to |= *from++;
 			len -= 3;
 		}
-		else if ((*from & 0x80) && len >= 2)
+		else if ((*from & 0x80) && len >= 2)	/* code set 1 */
 		{
 			*to = *from++ << 8;
 			*to |= *from++;
@@ -249,6 +249,7 @@ pg_euccn_dsplen(const unsigned char *s)
 
 /*
  * EUC_TW
+ *
  */
 static int	pg_euctw2wchar_with_len
 			(const unsigned char *from, pg_wchar *to, int len)
@@ -257,22 +258,22 @@ static int	pg_euctw2wchar_with_len
 
 	while (len > 0 && *from)
 	{
-		if (*from == SS2 && len >= 4)
+		if (*from == SS2 && len >= 4)	/* code set 2 */
 		{
 			from++;
-			*to = *from++ << 16;
+			*to = (SS2 << 24) | (*from++ << 16) ;
 			*to |= *from++ << 8;
 			*to |= *from++;
 			len -= 4;
 		}
-		else if (*from == SS3 && len >= 3)
+		else if (*from == SS3 && len >= 3)		/* code set 3 (unused?) */
 		{
 			from++;
-			*to = *from++ << 8;
-			*to |= 0x3f & *from++;
+			*to = (SS3 << 16) | (*from++ << 8);
+			*to |= *from++;
 			len -= 3;
 		}
-		else if ((*from & 0x80) && len >= 2)
+		else if ((*from & 0x80) && len >= 2)	/* code set 2 */
 		{
 			*to = *from++ << 8;
 			*to |= *from++;

Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,7 @@`
`1`	`1`	`/*`
`2`	`2`	`* conversion functions between pg_wchar and multibyte streams.`
`3`	`3`	`* Tatsuo Ishii`
`4`		`- * $PostgreSQL: pgsql/src/backend/utils/mb/wchar.c,v 1.47 2005/10/29 00:31:52 petere Exp $`
	`4`	`+ * $PostgreSQL: pgsql/src/backend/utils/mb/wchar.c,v 1.48 2005/12/24 09:35:36 ishii Exp $`
`5`	`5`	`*`
`6`	`6`	`* WIN1250 client encoding updated by Pavel Behal`
`7`	`7`	`*`
`@@ -58,34 +58,33 @@ pg_ascii_dsplen(const unsigned char *s)`
`58`	`58`	`/*`
`59`	`59`	`* EUC`
`60`	`60`	`*/`
`61`		`-`
`62`	`61`	`static int pg_euc2wchar_with_len`
`63`	`62`	`(const unsigned char from, pg_wchar to, int len)`
`64`	`63`	`{`
`65`	`64`	`int cnt = 0;`
`66`	`65`
`67`	`66`	`while (len > 0 && *from)`
`68`	`67`	`{`
`69`		`- if (*from == SS2 && len >= 2)`
	`68`	`+ if (from == SS2 && len >= 2) / JIS X 0201 (so called "1 byte KANA") */`
`70`	`69`	`{`
`71`	`70`	`from++;`
`72`		`- to = 0xff & from++;`
	`71`	`+ to = (SS2 << 8) \| from++;`
`73`	`72`	`len -= 2;`
`74`	`73`	`}`
`75`		`- else if (*from == SS3 && len >= 3)`
	`74`	`+ else if (from == SS3 && len >= 3) / JIS X 0212 KANJI */`
`76`	`75`	`{`
`77`	`76`	`from++;`
`78`		`- to = from++ << 8;`
`79`		`- to \|= 0x3f & from++;`
	`77`	`+ to = (SS3 << 16) \| (from++ << 8);`
	`78`	`+ to \|= from++;`
`80`	`79`	`len -= 3;`
`81`	`80`	`}`
`82`		`- else if ((*from & 0x80) && len >= 2)`
	`81`	`+ else if ((from & 0x80) && len >= 2) / JIS X 0208 KANJI */`
`83`	`82`	`{`
`84`	`83`	`to = from++ << 8;`
`85`	`84`	`to \|= from++;`
`86`	`85`	`len -= 2;`
`87`	`86`	`}`
`88`		`- else`
	`87`	`+ else /* must be ASCII */`
`89`	`88`	`{`
`90`	`89`	`to = from++;`
`91`	`90`	`len--;`
`@@ -183,6 +182,7 @@ pg_euckr_dsplen(const unsigned char *s)`
`183`	`182`
`184`	`183`	`/*`
`185`	`184`	`* EUC_CN`
	`185`	`+ *`
`186`	`186`	`*/`
`187`	`187`	`static int pg_euccn2wchar_with_len`
`188`	`188`	`(const unsigned char from, pg_wchar to, int len)`
`@@ -191,21 +191,21 @@ static int pg_euccn2wchar_with_len`
`191`	`191`
`192`	`192`	`while (len > 0 && *from)`
`193`	`193`	`{`
`194`		`- if (*from == SS2 && len >= 3)`
	`194`	`+ if (from == SS2 && len >= 3) / code set 2 (unused?) */`
`195`	`195`	`{`
`196`	`196`	`from++;`
`197`		`- to = 0x3f00 & (from++ << 8);`
`198`		`- to = from++;`
	`197`	`+ to = (SS2 << 16) \| (from++ << 8);`
	`198`	`+ to \|= from++;`
`199`	`199`	`len -= 3;`
`200`	`200`	`}`
`201`		`- else if (*from == SS3 && len >= 3)`
	`201`	`+ else if (from == SS3 && len >= 3) / code set 3 (unsed ?) */`
`202`	`202`	`{`
`203`	`203`	`from++;`
`204`		`- to = from++ << 8;`
`205`		`- to \|= 0x3f & from++;`
	`204`	`+ to = (SS3 << 16) \| (from++ << 8);`
	`205`	`+ to \|= from++;`
`206`	`206`	`len -= 3;`
`207`	`207`	`}`
`208`		`- else if ((*from & 0x80) && len >= 2)`
	`208`	`+ else if ((from & 0x80) && len >= 2) / code set 1 */`
`209`	`209`	`{`
`210`	`210`	`to = from++ << 8;`
`211`	`211`	`to \|= from++;`
`@@ -249,6 +249,7 @@ pg_euccn_dsplen(const unsigned char *s)`
`249`	`249`
`250`	`250`	`/*`
`251`	`251`	`* EUC_TW`
	`252`	`+ *`
`252`	`253`	`*/`
`253`	`254`	`static int pg_euctw2wchar_with_len`
`254`	`255`	`(const unsigned char from, pg_wchar to, int len)`
`@@ -257,22 +258,22 @@ static int pg_euctw2wchar_with_len`
`257`	`258`
`258`	`259`	`while (len > 0 && *from)`
`259`	`260`	`{`
`260`		`- if (*from == SS2 && len >= 4)`
	`261`	`+ if (from == SS2 && len >= 4) / code set 2 */`
`261`	`262`	`{`
`262`	`263`	`from++;`
`263`		`- to = from++ << 16;`
	`264`	`+ to = (SS2 << 24) \| (from++ << 16) ;`
`264`	`265`	`to \|= from++ << 8;`
`265`	`266`	`to \|= from++;`
`266`	`267`	`len -= 4;`
`267`	`268`	`}`
`268`		`- else if (*from == SS3 && len >= 3)`
	`269`	`+ else if (from == SS3 && len >= 3) / code set 3 (unused?) */`
`269`	`270`	`{`
`270`	`271`	`from++;`
`271`		`- to = from++ << 8;`
`272`		`- to \|= 0x3f & from++;`
	`272`	`+ to = (SS3 << 16) \| (from++ << 8);`
	`273`	`+ to \|= from++;`
`273`	`274`	`len -= 3;`
`274`	`275`	`}`
`275`		`- else if ((*from & 0x80) && len >= 2)`
	`276`	`+ else if ((from & 0x80) && len >= 2) / code set 2 */`
`276`	`277`	`{`
`277`	`278`	`to = from++ << 8;`
`278`	`279`	`to \|= from++;`