Skip to content

Commit 50c16fb

Browse files
jeff-davishs-liuxh
jeff-davis
authored and
hs-liuxh
committed
Use C11 char16_t and char32_t for Unicode code points.
Reviewed-by: Tatsuo Ishii <ishii@postgresql.org> Reviewed-by: Thomas Munro <thomas.munro@gmail.com> Reviewed-by: Peter Eisentraut <peter@eisentraut.org> Discussion: https://postgr.es/m/bedcc93d06203dfd89815b10f815ca2de8626e85.camel%40j-davis.com
1 parent 91f850b commit 50c16fb

29 files changed

Lines changed: 284 additions & 244 deletions

‎configure‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13775,7 +13775,7 @@ fi
1377513775
## Header files
1377613776
##
1377713777

13778-
for ac_header in atomic.h copyfile.h execinfo.h getopt.h ifaddrs.h mbarrier.h sys/epoll.h sys/event.h sys/personality.h sys/prctl.h sys/procctl.h sys/signalfd.h sys/ucred.h termios.h ucred.h xlocale.h
13778+
for ac_header in atomic.h copyfile.h execinfo.h getopt.h ifaddrs.h mbarrier.h sys/epoll.h sys/event.h sys/personality.h sys/prctl.h sys/procctl.h sys/signalfd.h sys/ucred.h termios.h uchar.h ucred.h xlocale.h
1377913779
do :
1378013780
as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh`
1378113781
ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default"

‎configure.ac‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1601,6 +1601,7 @@ AC_CHECK_HEADERS(m4_normalize([
16011601
sys/signalfd.h
16021602
sys/ucred.h
16031603
termios.h
1604+
uchar.h
16041605
ucred.h
16051606
xlocale.h
16061607
]))

‎meson.build‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2621,6 +2621,7 @@ header_checks = [
26212621
'sys/signalfd.h',
26222622
'sys/ucred.h',
26232623
'termios.h',
2624+
'uchar.h',
26242625
'ucred.h',
26252626
'xlocale.h',
26262627
]

‎src/backend/parser/parser.c‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -370,7 +370,7 @@ hexval(unsigned char c)
370370

371371
/* is Unicode code point acceptable? */
372372
static void
373-
check_unicode_value(pg_wchar c)
373+
check_unicode_value(char32_t c)
374374
{
375375
if (!is_valid_unicode_codepoint(c))
376376
ereport(ERROR,
@@ -407,7 +407,7 @@ str_udeescape(const char *str, char escape,
407407
char *new,
408408
*out;
409409
size_t new_len;
410-
pg_wchar pair_first = 0;
410+
char16_t pair_first = 0;
411411
ScannerCallbackState scbstate;
412412

413413
/*
@@ -451,7 +451,7 @@ str_udeescape(const char *str, char escape,
451451
isxdigit((unsigned char) in[3]) &&
452452
isxdigit((unsigned char) in[4]))
453453
{
454-
pg_wchar unicode;
454+
char32_t unicode;
455455

456456
unicode = (hexval(in[1]) << 12) +
457457
(hexval(in[2]) << 8) +
@@ -488,7 +488,7 @@ str_udeescape(const char *str, char escape,
488488
isxdigit((unsigned char) in[6]) &&
489489
isxdigit((unsigned char) in[7]))
490490
{
491-
pg_wchar unicode;
491+
char32_t unicode;
492492

493493
unicode = (hexval(in[2]) << 20) +
494494
(hexval(in[3]) << 16) +

‎src/backend/parser/scan.l‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ static void addlitchar(unsigned char ychar, core_yyscan_t yyscanner);
121121
static char *litbufdup(core_yyscan_t yyscanner);
122122
static unsigned char unescape_single_char(unsigned char c, core_yyscan_t yyscanner);
123123
static int process_integer_literal(const char *token, YYSTYPE *lval, int base);
124-
static void addunicode(pg_wchar c, yyscan_t yyscanner);
124+
static void addunicode(char32_t c, yyscan_t yyscanner);
125125

126126
#define yyerror(msg) scanner_yyerror(msg, yyscanner)
127127

@@ -640,7 +640,7 @@ other .
640640
addlit(yytext, yyleng, yyscanner);
641641
}
642642
<xe>{xeunicode} {
643-
pg_wchar c = strtoul(yytext + 2, NULL, 16);
643+
char32_t c = strtoul(yytext + 2, NULL, 16);
644644

645645
/*
646646
* For consistency with other productions, issue any
@@ -668,7 +668,7 @@ other .
668668
POP_YYLLOC();
669669
}
670670
<xeu>{xeunicode} {
671-
pg_wchar c = strtoul(yytext + 2, NULL, 16);
671+
char32_t c = strtoul(yytext + 2, NULL, 16);
672672

673673
/* Remember start of overall string token ... */
674674
PUSH_YYLLOC();
@@ -1376,7 +1376,7 @@ process_integer_literal(const char *token, YYSTYPE *lval, int base)
13761376
}
13771377

13781378
static void
1379-
addunicode(pg_wchar c, core_yyscan_t yyscanner)
1379+
addunicode(char32_t c, core_yyscan_t yyscanner)
13801380
{
13811381
ScannerCallbackState scbstate;
13821382
char buf[MAX_UNICODE_EQUIVALENT_STRING + 1];

‎src/backend/utils/adt/jsonpath_scan.l‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -574,7 +574,7 @@ hexval(char c, int *result, struct Node *escontext, yyscan_t yyscanner)
574574

575575
/* Add given unicode character to scanstring */
576576
static bool
577-
addUnicodeChar(int ch, struct Node *escontext, yyscan_t yyscanner)
577+
addUnicodeChar(char32_t ch, struct Node *escontext, yyscan_t yyscanner)
578578
{
579579
if (ch == 0)
580580
{
@@ -607,7 +607,7 @@ addUnicodeChar(int ch, struct Node *escontext, yyscan_t yyscanner)
607607

608608
/* Add unicode character, processing any surrogate pairs */
609609
static bool
610-
addUnicode(int ch, int *hi_surrogate, struct Node *escontext, yyscan_t yyscanner)
610+
addUnicode(char32_t ch, int *hi_surrogate, struct Node *escontext, yyscan_t yyscanner)
611611
{
612612
if (is_utf16_surrogate_first(ch))
613613
{
@@ -655,7 +655,7 @@ parseUnicode(char *s, int l, struct Node *escontext, yyscan_t yyscanner)
655655

656656
for (i = 2; i < l; i += 2) /* skip '\u' */
657657
{
658-
int ch = 0;
658+
char32_t ch = 0;
659659
int j,
660660
si;
661661

‎src/backend/utils/adt/pg_locale_builtin.c‎

Lines changed: 30 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
#include "catalog/pg_collation.h"
1616
#include "common/unicode_case.h"
1717
#include "common/unicode_category.h"
18-
#include "mb/pg_wchar.h"
1918
#include "miscadmin.h"
2019
#include "utils/builtins.h"
2120
#include "utils/pg_locale.h"
@@ -35,6 +34,23 @@ struct WordBoundaryState
3534
bool prev_alnum;
3635
};
3736

37+
/*
38+
* In UTF-8, pg_wchar is guaranteed to be the code point value.
39+
*/
40+
static inline char32_t
41+
to_char32(pg_wchar wc)
42+
{
43+
Assert(GetDatabaseEncoding() == PG_UTF8);
44+
return (char32_t) wc;
45+
}
46+
47+
static inline pg_wchar
48+
to_pg_wchar(char32_t c32)
49+
{
50+
Assert(GetDatabaseEncoding() == PG_UTF8);
51+
return (pg_wchar) c32;
52+
}
53+
3854
/*
3955
* Simple word boundary iterator that draws boundaries each time the result of
4056
* pg_u_isalnum() changes.
@@ -47,7 +63,7 @@ initcap_wbnext(void *state)
4763
while (wbstate->offset < wbstate->len &&
4864
wbstate->str[wbstate->offset] != '\0')
4965
{
50-
pg_wchar u = utf8_to_unicode((unsigned char *) wbstate->str +
66+
char32_t u = utf8_to_unicode((unsigned char *) wbstate->str +
5167
wbstate->offset);
5268
bool curr_alnum = pg_u_isalnum(u, wbstate->posix);
5369

@@ -112,61 +128,61 @@ strfold_builtin(char *dest, size_t destsize, const char *src, ssize_t srclen,
112128
static bool
113129
wc_isdigit_builtin(pg_wchar wc, pg_locale_t locale)
114130
{
115-
return pg_u_isdigit(wc, !locale->builtin.casemap_full);
131+
return pg_u_isdigit(to_char32(wc), !locale->builtin.casemap_full);
116132
}
117133

118134
static bool
119135
wc_isalpha_builtin(pg_wchar wc, pg_locale_t locale)
120136
{
121-
return pg_u_isalpha(wc);
137+
return pg_u_isalpha(to_char32(wc));
122138
}
123139

124140
static bool
125141
wc_isalnum_builtin(pg_wchar wc, pg_locale_t locale)
126142
{
127-
return pg_u_isalnum(wc, !locale->builtin.casemap_full);
143+
return pg_u_isalnum(to_char32(wc), !locale->builtin.casemap_full);
128144
}
129145

130146
static bool
131147
wc_isupper_builtin(pg_wchar wc, pg_locale_t locale)
132148
{
133-
return pg_u_isupper(wc);
149+
return pg_u_isupper(to_char32(wc));
134150
}
135151

136152
static bool
137153
wc_islower_builtin(pg_wchar wc, pg_locale_t locale)
138154
{
139-
return pg_u_islower(wc);
155+
return pg_u_islower(to_char32(wc));
140156
}
141157

142158
static bool
143159
wc_isgraph_builtin(pg_wchar wc, pg_locale_t locale)
144160
{
145-
return pg_u_isgraph(wc);
161+
return pg_u_isgraph(to_char32(wc));
146162
}
147163

148164
static bool
149165
wc_isprint_builtin(pg_wchar wc, pg_locale_t locale)
150166
{
151-
return pg_u_isprint(wc);
167+
return pg_u_isprint(to_char32(wc));
152168
}
153169

154170
static bool
155171
wc_ispunct_builtin(pg_wchar wc, pg_locale_t locale)
156172
{
157-
return pg_u_ispunct(wc, !locale->builtin.casemap_full);
173+
return pg_u_ispunct(to_char32(wc), !locale->builtin.casemap_full);
158174
}
159175

160176
static bool
161177
wc_isspace_builtin(pg_wchar wc, pg_locale_t locale)
162178
{
163-
return pg_u_isspace(wc);
179+
return pg_u_isspace(to_char32(wc));
164180
}
165181

166182
static bool
167183
wc_isxdigit_builtin(pg_wchar wc, pg_locale_t locale)
168184
{
169-
return pg_u_isxdigit(wc, !locale->builtin.casemap_full);
185+
return pg_u_isxdigit(to_char32(wc), !locale->builtin.casemap_full);
170186
}
171187

172188
static bool
@@ -179,13 +195,13 @@ char_is_cased_builtin(char ch, pg_locale_t locale)
179195
static pg_wchar
180196
wc_toupper_builtin(pg_wchar wc, pg_locale_t locale)
181197
{
182-
return unicode_uppercase_simple(wc);
198+
return to_pg_wchar(unicode_uppercase_simple(to_char32(wc)));
183199
}
184200

185201
static pg_wchar
186202
wc_tolower_builtin(pg_wchar wc, pg_locale_t locale)
187203
{
188-
return unicode_lowercase_simple(wc);
204+
return to_pg_wchar(unicode_lowercase_simple(to_char32(wc)));
189205
}
190206

191207
static const struct ctype_methods ctype_methods_builtin = {

‎src/backend/utils/adt/varlena.c‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -5798,12 +5798,12 @@ unicode_assigned(PG_FUNCTION_ARGS)
57985798
ereport(ERROR,
57995799
(errmsg("Unicode categorization can only be performed if server encoding is UTF8")));
58005800

5801-
/* convert to pg_wchar */
5801+
/* convert to char32_t */
58025802
size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input));
58035803
p = (unsigned char *) VARDATA_ANY(input);
58045804
for (int i = 0; i < size; i++)
58055805
{
5806-
pg_wchar uchar = utf8_to_unicode(p);
5806+
char32_t uchar = utf8_to_unicode(p);
58075807
int category = unicode_category(uchar);
58085808

58095809
if (category == PG_U_UNASSIGNED)
@@ -5822,32 +5822,32 @@ unicode_normalize_func(PG_FUNCTION_ARGS)
58225822
char *formstr = text_to_cstring(PG_GETARG_TEXT_PP(1));
58235823
UnicodeNormalizationForm form;
58245824
int size;
5825-
pg_wchar *input_chars;
5826-
pg_wchar *output_chars;
5825+
char32_t *input_chars;
5826+
char32_t *output_chars;
58275827
unsigned char *p;
58285828
text *result;
58295829
int i;
58305830

58315831
form = unicode_norm_form_from_string(formstr);
58325832

5833-
/* convert to pg_wchar */
5833+
/* convert to char32_t */
58345834
size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input));
5835-
input_chars = palloc((size + 1) * sizeof(pg_wchar));
5835+
input_chars = palloc((size + 1) * sizeof(char32_t));
58365836
p = (unsigned char *) VARDATA_ANY(input);
58375837
for (i = 0; i < size; i++)
58385838
{
58395839
input_chars[i] = utf8_to_unicode(p);
58405840
p += pg_utf_mblen(p);
58415841
}
5842-
input_chars[i] = (pg_wchar) '\0';
5842+
input_chars[i] = (char32_t) '\0';
58435843
Assert((char *) p == VARDATA_ANY(input) + VARSIZE_ANY_EXHDR(input));
58445844

58455845
/* action */
58465846
output_chars = unicode_normalize(form, input_chars);
58475847

58485848
/* convert back to UTF-8 string */
58495849
size = 0;
5850-
for (pg_wchar *wp = output_chars; *wp; wp++)
5850+
for (char32_t *wp = output_chars; *wp; wp++)
58515851
{
58525852
unsigned char buf[4];
58535853

@@ -5859,7 +5859,7 @@ unicode_normalize_func(PG_FUNCTION_ARGS)
58595859
SET_VARSIZE(result, size + VARHDRSZ);
58605860

58615861
p = (unsigned char *) VARDATA_ANY(result);
5862-
for (pg_wchar *wp = output_chars; *wp; wp++)
5862+
for (char32_t *wp = output_chars; *wp; wp++)
58635863
{
58645864
unicode_to_utf8(*wp, p);
58655865
p += pg_utf_mblen(p);
@@ -5888,8 +5888,8 @@ unicode_is_normalized(PG_FUNCTION_ARGS)
58885888
char *formstr = text_to_cstring(PG_GETARG_TEXT_PP(1));
58895889
UnicodeNormalizationForm form;
58905890
int size;
5891-
pg_wchar *input_chars;
5892-
pg_wchar *output_chars;
5891+
char32_t *input_chars;
5892+
char32_t *output_chars;
58935893
unsigned char *p;
58945894
int i;
58955895
UnicodeNormalizationQC quickcheck;
@@ -5898,16 +5898,16 @@ unicode_is_normalized(PG_FUNCTION_ARGS)
58985898

58995899
form = unicode_norm_form_from_string(formstr);
59005900

5901-
/* convert to pg_wchar */
5901+
/* convert to char32_t */
59025902
size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input));
5903-
input_chars = palloc((size + 1) * sizeof(pg_wchar));
5903+
input_chars = palloc((size + 1) * sizeof(char32_t));
59045904
p = (unsigned char *) VARDATA_ANY(input);
59055905
for (i = 0; i < size; i++)
59065906
{
59075907
input_chars[i] = utf8_to_unicode(p);
59085908
p += pg_utf_mblen(p);
59095909
}
5910-
input_chars[i] = (pg_wchar) '\0';
5910+
input_chars[i] = (char32_t) '\0';
59115911
Assert((char *) p == VARDATA_ANY(input) + VARSIZE_ANY_EXHDR(input));
59125912

59135913
/* quick check (see UAX #15) */
@@ -5921,11 +5921,11 @@ unicode_is_normalized(PG_FUNCTION_ARGS)
59215921
output_chars = unicode_normalize(form, input_chars);
59225922

59235923
output_size = 0;
5924-
for (pg_wchar *wp = output_chars; *wp; wp++)
5924+
for (char32_t *wp = output_chars; *wp; wp++)
59255925
output_size++;
59265926

59275927
result = (size == output_size) &&
5928-
(memcmp(input_chars, output_chars, size * sizeof(pg_wchar)) == 0);
5928+
(memcmp(input_chars, output_chars, size * sizeof(char32_t)) == 0);
59295929

59305930
PG_RETURN_BOOL(result);
59315931
}
@@ -5981,7 +5981,7 @@ unistr(PG_FUNCTION_ARGS)
59815981
int len;
59825982
StringInfoData str;
59835983
text *result;
5984-
pg_wchar pair_first = 0;
5984+
char16_t pair_first = 0;
59855985
char cbuf[MAX_UNICODE_EQUIVALENT_STRING + 1];
59865986

59875987
instr = VARDATA_ANY(input_text);
@@ -6005,7 +6005,7 @@ unistr(PG_FUNCTION_ARGS)
60056005
else if ((len >= 5 && isxdigits_n(instr + 1, 4)) ||
60066006
(len >= 6 && instr[1] == 'u' && isxdigits_n(instr + 2, 4)))
60076007
{
6008-
pg_wchar unicode;
6008+
char32_t unicode;
60096009
int offset = instr[1] == 'u' ? 2 : 1;
60106010

60116011
unicode = hexval_n(instr + offset, 4);
@@ -6041,7 +6041,7 @@ unistr(PG_FUNCTION_ARGS)
60416041
}
60426042
else if (len >= 8 && instr[1] == '+' && isxdigits_n(instr + 2, 6))
60436043
{
6044-
pg_wchar unicode;
6044+
char32_t unicode;
60456045

60466046
unicode = hexval_n(instr + 2, 6);
60476047

@@ -6076,7 +6076,7 @@ unistr(PG_FUNCTION_ARGS)
60766076
}
60776077
else if (len >= 10 && instr[1] == 'U' && isxdigits_n(instr + 2, 8))
60786078
{
6079-
pg_wchar unicode;
6079+
char32_t unicode;
60806080

60816081
unicode = hexval_n(instr + 2, 8);
60826082

0 commit comments

Comments
 (0)