Skip to content

gh-103583: Isolate CJK codec modules #103869

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
erlend-aasland merged 7 commits into from
Apr 27, 2023
Merged

gh-103583: Isolate CJK codec modules #103869

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
cb2c9e7
Pass codec iso. config to mbcodec API handlers
erlend-aasland Apr 26, 2023
409bf98
Add cjk module state pointer to MultibyteCodec struct
erlend-aasland Apr 26, 2023
fd6a2a0
Make it possible to specify cjk module specific state
erlend-aasland Apr 26, 2023
eb44ae6
PoC: put big5 codec in cjk module state
erlend-aasland Apr 26, 2023
f6ed5df
PoC: iso2022 codecs
erlend-aasland Apr 26, 2023
ef983c4
Add NEWS
erlend-aasland Apr 26, 2023
dec79a7
Merge branch 'main' into isolate-cjk-alt2
erlend-aasland Apr 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Pass codec iso. config to mbcodec API handlers
  • Loading branch information
@erlend-aasland
erlend-aasland committed Apr 26, 2023
commit cb2c9e7b43994dd0e8ad1969318fafd1fc0fc040
21 changes: 11 additions & 10 deletions Modules/cjkcodecs/_codecs_iso2022.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@
#define STATE_CLEARFLAG(f) do { ((state)->c[4]) &= ~(f); } while (0)
#define STATE_CLEARFLAGS() do { ((state)->c[4]) = 0; } while (0)

#define ISO2022_CONFIG ((const struct iso2022_config *)config)
#define ISO2022_CONFIG ((const struct iso2022_config *)(codec->config))
#define CONFIG_ISSET(flag) (ISO2022_CONFIG->flags & (flag))
#define CONFIG_DESIGNATIONS (ISO2022_CONFIG->designations)

Expand Down Expand Up @@ -288,7 +288,7 @@ DECODER_RESET(iso2022)
}

static Py_ssize_t
iso2022processesc(const void *config, MultibyteCodec_State *state,
iso2022processesc(const MultibyteCodec *codec, MultibyteCodec_State *state,
const unsigned char **inbuf, Py_ssize_t *inleft)
{
unsigned char charset, designation;
Expand Down Expand Up @@ -388,7 +388,7 @@ iso2022processesc(const void *config, MultibyteCodec_State *state,
}

static Py_ssize_t
iso2022processg2(const void *config, MultibyteCodec_State *state,
iso2022processg2(const MultibyteCodec *codec, MultibyteCodec_State *state,
const unsigned char **inbuf, Py_ssize_t *inleft,
_PyUnicodeWriter *writer)
{
Expand Down Expand Up @@ -442,14 +442,14 @@ DECODER(iso2022)
case ESC:
REQUIRE_INBUF(2);
if (IS_ISO2022ESC(INBYTE2)) {
err = iso2022processesc(config, state,
err = iso2022processesc(codec, state,
inbuf, &inleft);
if (err != 0)
return err;
}
else if (CONFIG_ISSET(USE_G2) && INBYTE2 == 'N') {/* SS2 */
REQUIRE_INBUF(3);
err = iso2022processg2(config, state,
err = iso2022processg2(codec, state,
inbuf, &inleft, writer);
if (err != 0)
return err;
Expand Down Expand Up @@ -712,7 +712,7 @@ static Py_UCS4
jisx0213_2000_1_decoder(const unsigned char *data)
{
Py_UCS4 u;
EMULATE_JISX0213_2000_DECODE_PLANE1(u, data[0], data[1])
EMULATE_JISX0213_2000_DECODE_PLANE1(config, u, data[0], data[1])
else if (data[0] == 0x21 && data[1] == 0x40) /* F/W REVERSE SOLIDUS */
return 0xff3c;
else if (TRYMAP_DEC(jisx0208, u, data[0], data[1]))
Expand All @@ -732,7 +732,7 @@ static Py_UCS4
jisx0213_2000_2_decoder(const unsigned char *data)
{
Py_UCS4 u;
EMULATE_JISX0213_2000_DECODE_PLANE2_CHAR(u, data[0], data[1])
EMULATE_JISX0213_2000_DECODE_PLANE2_CHAR(config, u, data[0], data[1])
if (TRYMAP_DEC(jisx0213_2_bmp, u, data[0], data[1]))
;
else if (TRYMAP_DEC(jisx0213_2_emp, u, data[0], data[1]))
Expand Down Expand Up @@ -776,22 +776,23 @@ jisx0213_2004_2_decoder(const unsigned char *data)
}

static DBCHAR
jisx0213_encoder(const Py_UCS4 *data, Py_ssize_t *length, void *config)
jisx0213_encoder(const Py_UCS4 *data, Py_ssize_t *length,
const void *config)
{
DBCHAR coded;

switch (*length) {
case 1: /* first character */
if (*data >= 0x10000) {
if ((*data) >> 16 == 0x20000 >> 16) {
EMULATE_JISX0213_2000_ENCODE_EMP(coded, *data)
EMULATE_JISX0213_2000_ENCODE_EMP(config, coded, *data)
else if (TRYMAP_ENC(jisx0213_emp, coded, (*data) & 0xffff))
return coded;
}
return MAP_UNMAPPABLE;
}

EMULATE_JISX0213_2000_ENCODE_BMP(coded, *data)
EMULATE_JISX0213_2000_ENCODE_BMP(config, coded, *data)
else if (TRYMAP_ENC(jisx0213_bmp, coded, *data)) {
if (coded == MULTIC)
return MAP_MULTIPLE_AVAIL;
Expand Down
16 changes: 8 additions & 8 deletions Modules/cjkcodecs/_codecs_jp.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ ENCODER(euc_jis_2004)
insize = 1;

if (c <= 0xFFFF) {
EMULATE_JISX0213_2000_ENCODE_BMP(code, c)
EMULATE_JISX0213_2000_ENCODE_BMP(codec->config, code, c)
else if (TRYMAP_ENC(jisx0213_bmp, code, c)) {
if (code == MULTIC) {
if (inlen - *inpos < 2) {
Expand Down Expand Up @@ -215,7 +215,7 @@ ENCODER(euc_jis_2004)
return 1;
}
else if (c >> 16 == EMPBASE >> 16) {
EMULATE_JISX0213_2000_ENCODE_EMP(code, c)
EMULATE_JISX0213_2000_ENCODE_EMP(codec->config, code, c)
else if (TRYMAP_ENC(jisx0213_emp, code, c & 0xffff))
;
else
Expand Down Expand Up @@ -271,7 +271,7 @@ DECODER(euc_jis_2004)
c3 = INBYTE3 ^ 0x80;

/* JIS X 0213 Plane 2 or JIS X 0212 (see NOTES) */
EMULATE_JISX0213_2000_DECODE_PLANE2(writer, c2, c3)
EMULATE_JISX0213_2000_DECODE_PLANE2(codec->config, writer, c2, c3)
else if (TRYMAP_DEC(jisx0213_2_bmp, decoded, c2, c3))
OUTCHAR(decoded);
else if (TRYMAP_DEC(jisx0213_2_emp, code, c2, c3)) {
Expand All @@ -293,7 +293,7 @@ DECODER(euc_jis_2004)
c2 = INBYTE2 ^ 0x80;

/* JIS X 0213 Plane 1 */
EMULATE_JISX0213_2000_DECODE_PLANE1(writer, c, c2)
EMULATE_JISX0213_2000_DECODE_PLANE1(codec->config, writer, c, c2)
else if (c == 0x21 && c2 == 0x40)
OUTCHAR(0xff3c);
else if (c == 0x22 && c2 == 0x32)
Expand Down Expand Up @@ -582,7 +582,7 @@ ENCODER(shift_jis_2004)

if (code == NOCHAR) {
if (c <= 0xffff) {
EMULATE_JISX0213_2000_ENCODE_BMP(code, c)
EMULATE_JISX0213_2000_ENCODE_BMP(codec->config, code, c)
else if (TRYMAP_ENC(jisx0213_bmp, code, c)) {
if (code == MULTIC) {
if (inlen - *inpos < 2) {
Expand Down Expand Up @@ -625,7 +625,7 @@ ENCODER(shift_jis_2004)
return 1;
}
else if (c >> 16 == EMPBASE >> 16) {
EMULATE_JISX0213_2000_ENCODE_EMP(code, c)
EMULATE_JISX0213_2000_ENCODE_EMP(codec->config, code, c)
else if (TRYMAP_ENC(jisx0213_emp, code, c&0xffff))
;
else
Expand Down Expand Up @@ -686,7 +686,7 @@ DECODER(shift_jis_2004)

if (c1 < 0x5e) { /* Plane 1 */
c1 += 0x21;
EMULATE_JISX0213_2000_DECODE_PLANE1(writer,
EMULATE_JISX0213_2000_DECODE_PLANE1(codec->config, writer,
c1, c2)
else if (TRYMAP_DEC(jisx0208, decoded, c1, c2))
OUTCHAR(decoded);
Expand All @@ -708,7 +708,7 @@ DECODER(shift_jis_2004)
else
c1 -= 0x3d;

EMULATE_JISX0213_2000_DECODE_PLANE2(writer,
EMULATE_JISX0213_2000_DECODE_PLANE2(codec->config, writer,
c1, c2)
else if (TRYMAP_DEC(jisx0213_2_bmp, decoded, c1, c2))
OUTCHAR(decoded);
Expand Down
14 changes: 7 additions & 7 deletions Modules/cjkcodecs/cjkcodecs.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,33 +76,33 @@ get_module_state(PyObject *mod)
}

#define CODEC_INIT(encoding) \
static int encoding##_codec_init(const void *config)
static int encoding##_codec_init(const MultibyteCodec *codec)

#define ENCODER_INIT(encoding) \
static int encoding##_encode_init( \
MultibyteCodec_State *state, const void *config)
MultibyteCodec_State *state, const MultibyteCodec *codec)
#define ENCODER(encoding) \
static Py_ssize_t encoding##_encode( \
MultibyteCodec_State *state, const void *config, \
MultibyteCodec_State *state, const MultibyteCodec *codec, \
int kind, const void *data, \
Py_ssize_t *inpos, Py_ssize_t inlen, \
unsigned char **outbuf, Py_ssize_t outleft, int flags)
#define ENCODER_RESET(encoding) \
static Py_ssize_t encoding##_encode_reset( \
MultibyteCodec_State *state, const void *config, \
MultibyteCodec_State *state, const MultibyteCodec *codec, \
unsigned char **outbuf, Py_ssize_t outleft)

#define DECODER_INIT(encoding) \
static int encoding##_decode_init( \
MultibyteCodec_State *state, const void *config)
MultibyteCodec_State *state, const MultibyteCodec *codec)
#define DECODER(encoding) \
static Py_ssize_t encoding##_decode( \
MultibyteCodec_State *state, const void *config, \
MultibyteCodec_State *state, const MultibyteCodec *codec, \
const unsigned char **inbuf, Py_ssize_t inleft, \
_PyUnicodeWriter *writer)
#define DECODER_RESET(encoding) \
static Py_ssize_t encoding##_decode_reset( \
MultibyteCodec_State *state, const void *config)
MultibyteCodec_State *state, const MultibyteCodec *codec)

#define NEXT_IN(i) \
do { \
Expand Down
22 changes: 11 additions & 11 deletions Modules/cjkcodecs/emu_jisx0213_2000.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,30 +5,30 @@
# define EMULATE_JISX0213_2000_ENCODE_INVALID 1
#endif

#define EMULATE_JISX0213_2000_ENCODE_BMP(assi, c) \
if (config == (void *)2000 && ( \
#define EMULATE_JISX0213_2000_ENCODE_BMP(config, assi, c) \
if ((config) == (void *)2000 && ( \
(c) == 0x9B1C || (c) == 0x4FF1 || \
(c) == 0x525D || (c) == 0x541E || \
(c) == 0x5653 || (c) == 0x59F8 || \
(c) == 0x5C5B || (c) == 0x5E77 || \
(c) == 0x7626 || (c) == 0x7E6B)) { \
return EMULATE_JISX0213_2000_ENCODE_INVALID; \
} \
else if (config == (void *)2000 && (c) == 0x9B1D) { \
else if ((config) == (void *)2000 && (c) == 0x9B1D) { \
(assi) = 0x8000 | 0x7d3b; \
}

#define EMULATE_JISX0213_2000_ENCODE_EMP(assi, c) \
if (config == (void *)2000 && (c) == 0x20B9F) { \
#define EMULATE_JISX0213_2000_ENCODE_EMP(config, assi, c) \
if ((config) == (void *)2000 && (c) == 0x20B9F) { \
return EMULATE_JISX0213_2000_ENCODE_INVALID; \
}

#ifndef EMULATE_JISX0213_2000_DECODE_INVALID
# define EMULATE_JISX0213_2000_DECODE_INVALID 2
#endif

#define EMULATE_JISX0213_2000_DECODE_PLANE1(assi, c1, c2) \
if (config == (void *)2000 && \
#define EMULATE_JISX0213_2000_DECODE_PLANE1(config, assi, c1, c2) \
if ((config) == (void *)2000 && \
(((c1) == 0x2E && (c2) == 0x21) || \
((c1) == 0x2F && (c2) == 0x7E) || \
((c1) == 0x4F && (c2) == 0x54) || \
Expand All @@ -42,13 +42,13 @@
return EMULATE_JISX0213_2000_DECODE_INVALID; \
}

#define EMULATE_JISX0213_2000_DECODE_PLANE2(writer, c1, c2) \
if (config == (void *)2000 && (c1) == 0x7D && (c2) == 0x3B) { \
#define EMULATE_JISX0213_2000_DECODE_PLANE2(config, writer, c1, c2) \
if ((config) == (void *)2000 && (c1) == 0x7D && (c2) == 0x3B) { \
OUTCHAR(0x9B1D); \
}

#define EMULATE_JISX0213_2000_DECODE_PLANE2_CHAR(assi, c1, c2) \
if (config == (void *)2000 && (c1) == 0x7D && (c2) == 0x3B) { \
#define EMULATE_JISX0213_2000_DECODE_PLANE2_CHAR(config, assi, c1, c2) \
if ((config) == (void *)2000 && (c1) == 0x7D && (c2) == 0x3B) { \
(assi) = 0x9B1D; \
}

30 changes: 15 additions & 15 deletions Modules/cjkcodecs/multibytecodec.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,7 @@ multibytecodec_encerror(const MultibyteCodec *codec,
for (;;) {
Py_ssize_t outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf);

r = codec->encode(state, codec->config,
r = codec->encode(state, codec,
kind, data, &inpos, 1,
&buf->outbuf, outleft, 0);
if (r == MBERR_TOOSMALL) {
Expand Down Expand Up @@ -521,7 +521,7 @@ multibytecodec_encode(const MultibyteCodec *codec,
* error callbacks can relocate the cursor anywhere on buffer*/
Py_ssize_t outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);

r = codec->encode(state, codec->config,
r = codec->encode(state, codec,
kind, data,
&buf.inpos, buf.inlen,
&buf.outbuf, outleft, flags);
Expand All @@ -538,7 +538,7 @@ multibytecodec_encode(const MultibyteCodec *codec,
Py_ssize_t outleft;

outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);
r = codec->encreset(state, codec->config, &buf.outbuf,
r = codec->encreset(state, codec, &buf.outbuf,
outleft);
if (r == 0)
break;
Expand Down Expand Up @@ -616,7 +616,7 @@ _multibytecodec_MultibyteCodec_encode_impl(MultibyteCodecObject *self,
}

if (self->codec->encinit != NULL &&
self->codec->encinit(&state, self->codec->config) != 0)
self->codec->encinit(&state, self->codec) != 0)
goto errorexit;
r = multibytecodec_encode(self->codec, &state,
input, NULL, errorcb,
Expand Down Expand Up @@ -680,15 +680,15 @@ _multibytecodec_MultibyteCodec_decode_impl(MultibyteCodecObject *self,
buf.inbuf_end = buf.inbuf_top + datalen;

if (self->codec->decinit != NULL &&
self->codec->decinit(&state, self->codec->config) != 0)
self->codec->decinit(&state, self->codec) != 0)
goto errorexit;

while (buf.inbuf < buf.inbuf_end) {
Py_ssize_t inleft, r;

inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf);

r = self->codec->decode(&state, self->codec->config,
r = self->codec->decode(&state, self->codec,
&buf.inbuf, inleft, &buf.writer);
if (r == 0)
break;
Expand Down Expand Up @@ -888,7 +888,7 @@ decoder_feed_buffer(MultibyteStatefulDecoderContext *ctx,

inleft = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);

r = ctx->codec->decode(&ctx->state, ctx->codec->config,
r = ctx->codec->decode(&ctx->state, ctx->codec,
&buf->inbuf, inleft, &buf->writer);
if (r == 0 || r == MBERR_TOOFEW)
break;
Expand Down Expand Up @@ -1015,7 +1015,7 @@ _multibytecodec_MultibyteIncrementalEncoder_reset_impl(MultibyteIncrementalEncod
Py_ssize_t r;
if (self->codec->encreset != NULL) {
outbuf = buffer;
r = self->codec->encreset(&self->state, self->codec->config,
r = self->codec->encreset(&self->state, self->codec,
&outbuf, sizeof(buffer));
if (r != 0)
return NULL;
Expand Down Expand Up @@ -1063,7 +1063,7 @@ mbiencoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
if (self->errors == NULL)
goto errorexit;
if (self->codec->encinit != NULL &&
self->codec->encinit(&self->state, self->codec->config) != 0)
self->codec->encinit(&self->state, self->codec) != 0)
goto errorexit;

Py_DECREF(codec);
Expand Down Expand Up @@ -1292,7 +1292,7 @@ _multibytecodec_MultibyteIncrementalDecoder_reset_impl(MultibyteIncrementalDecod
/*[clinic end generated code: output=da423b1782c23ed1 input=3b63b3be85b2fb45]*/
{
if (self->codec->decreset != NULL &&
self->codec->decreset(&self->state, self->codec->config) != 0)
self->codec->decreset(&self->state, self->codec) != 0)
return NULL;
self->pendingsize = 0;

Expand Down Expand Up @@ -1338,7 +1338,7 @@ mbidecoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
if (self->errors == NULL)
goto errorexit;
if (self->codec->decinit != NULL &&
self->codec->decinit(&self->state, self->codec->config) != 0)
self->codec->decinit(&self->state, self->codec) != 0)
goto errorexit;

Py_DECREF(codec);
Expand Down Expand Up @@ -1600,7 +1600,7 @@ _multibytecodec_MultibyteStreamReader_reset_impl(MultibyteStreamReaderObject *se
/*[clinic end generated code: output=138490370a680abc input=5d4140db84b5e1e2]*/
{
if (self->codec->decreset != NULL &&
self->codec->decreset(&self->state, self->codec->config) != 0)
self->codec->decreset(&self->state, self->codec) != 0)
return NULL;
self->pendingsize = 0;

Expand Down Expand Up @@ -1654,7 +1654,7 @@ mbstreamreader_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
if (self->errors == NULL)
goto errorexit;
if (self->codec->decinit != NULL &&
self->codec->decinit(&self->state, self->codec->config) != 0)
self->codec->decinit(&self->state, self->codec) != 0)
goto errorexit;

Py_DECREF(codec);
Expand Down Expand Up @@ -1877,7 +1877,7 @@ mbstreamwriter_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
if (self->errors == NULL)
goto errorexit;
if (self->codec->encinit != NULL &&
self->codec->encinit(&self->state, self->codec->config) != 0)
self->codec->encinit(&self->state, self->codec) != 0)
goto errorexit;

Py_DECREF(codec);
Expand Down Expand Up @@ -1971,7 +1971,7 @@ _multibytecodec___create_codec(PyObject *module, PyObject *arg)

codec_capsule *data = PyCapsule_GetPointer(arg, CODEC_CAPSULE);
const MultibyteCodec *codec = data->codec;
if (codec->codecinit != NULL && codec->codecinit(codec->config) != 0)
if (codec->codecinit != NULL && codec->codecinit(codec) != 0)
return NULL;

module_state *state = get_module_state(module);
Expand Down
Loading