Skip to content

Commit 0016507

Browse files
vstinnervstinner
committed
Issue python#25318: Move _PyBytesWriter to bytesobject.c
Declare also the private API in bytesobject.h.
1 parent e7bf86c commit 0016507

3 files changed

Lines changed: 245 additions & 210 deletions

File tree

‎Include/bytesobject.h‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,58 @@ PyAPI_FUNC(Py_ssize_t) _PyBytes_InsertThousandsGrouping(char *buffer,
123123
#define F_ALT (1<<3)
124124
#define F_ZERO (1<<4)
125125

126+
#ifndef Py_LIMITED_API
127+
/* The _PyBytesWriter structure is big: it contains an embeded "stack buffer".
128+
A _PyBytesWriter variable must be declared at the end of variables in a
129+
function to optimize the memory allocation on the stack. */
130+
typedef struct {
131+
/* bytes object */
132+
PyObject *buffer;
133+
134+
/* Number of allocated size */
135+
Py_ssize_t allocated;
136+
137+
/* Current size of the buffer (can be smaller than the allocated size) */
138+
Py_ssize_t size;
139+
140+
/* If non-zero, overallocate the buffer (default: 0). */
141+
int overallocate;
142+
143+
/* Stack buffer */
144+
int use_stack_buffer;
145+
char stack_buffer[512];
146+
} _PyBytesWriter;
147+
148+
/* Initialize a bytes writer
149+
150+
By default, the overallocation is disabled. Set the overallocate attribute
151+
to control the allocation of the buffer. */
152+
PyAPI_FUNC(void) _PyBytesWriter_Init(_PyBytesWriter *writer);
153+
154+
/* Get the buffer content and reset the writer.
155+
Return a bytes object.
156+
Raise an exception and return NULL on error. */
157+
PyAPI_FUNC(PyObject *) _PyBytesWriter_Finish(_PyBytesWriter *writer,
158+
char *str);
159+
160+
/* Deallocate memory of a writer (clear its internal buffer). */
161+
PyAPI_FUNC(void) _PyBytesWriter_Dealloc(_PyBytesWriter *writer);
162+
163+
/* Allocate the buffer to write size bytes.
164+
Return the pointer to the beginning of buffer data.
165+
Raise an exception and return NULL on error. */
166+
PyAPI_FUNC(char*) _PyBytesWriter_Alloc(_PyBytesWriter *writer,
167+
Py_ssize_t size);
168+
169+
/* Add *size* bytes to the buffer.
170+
str is the current pointer inside the buffer.
171+
Return the updated current pointer inside the buffer.
172+
Raise an exception and return NULL on error. */
173+
PyAPI_FUNC(char*) _PyBytesWriter_Prepare(_PyBytesWriter *writer,
174+
char *str,
175+
Py_ssize_t size);
176+
#endif /* Py_LIMITED_API */
177+
126178
#ifdef __cplusplus
127179
}
128180
#endif

‎Objects/bytesobject.c‎

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3735,3 +3735,196 @@ bytes_iter(PyObject *seq)
37353735
_PyObject_GC_TRACK(it);
37363736
return (PyObject *)it;
37373737
}
3738+
3739+
3740+
/* _PyBytesWriter API */
3741+
3742+
#ifdef MS_WINDOWS
3743+
/* On Windows, overallocate by 50% is the best factor */
3744+
# define OVERALLOCATE_FACTOR 2
3745+
#else
3746+
/* On Linux, overallocate by 25% is the best factor */
3747+
# define OVERALLOCATE_FACTOR 4
3748+
#endif
3749+
3750+
void
3751+
_PyBytesWriter_Init(_PyBytesWriter *writer)
3752+
{
3753+
writer->buffer = NULL;
3754+
writer->allocated = 0;
3755+
writer->size = 0;
3756+
writer->overallocate = 0;
3757+
writer->use_stack_buffer = 0;
3758+
#ifdef Py_DEBUG
3759+
memset(writer->stack_buffer, 0xCB, sizeof(writer->stack_buffer));
3760+
#endif
3761+
}
3762+
3763+
void
3764+
_PyBytesWriter_Dealloc(_PyBytesWriter *writer)
3765+
{
3766+
Py_CLEAR(writer->buffer);
3767+
}
3768+
3769+
Py_LOCAL_INLINE(char*)
3770+
_PyBytesWriter_AsString(_PyBytesWriter *writer)
3771+
{
3772+
if (!writer->use_stack_buffer) {
3773+
assert(writer->buffer != NULL);
3774+
return PyBytes_AS_STRING(writer->buffer);
3775+
}
3776+
else {
3777+
assert(writer->buffer == NULL);
3778+
return writer->stack_buffer;
3779+
}
3780+
}
3781+
3782+
Py_LOCAL_INLINE(Py_ssize_t)
3783+
_PyBytesWriter_GetPos(_PyBytesWriter *writer, char *str)
3784+
{
3785+
char *start = _PyBytesWriter_AsString(writer);
3786+
assert(str != NULL);
3787+
assert(str >= start);
3788+
return str - start;
3789+
}
3790+
3791+
Py_LOCAL_INLINE(void)
3792+
_PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str)
3793+
{
3794+
#ifdef Py_DEBUG
3795+
char *start, *end;
3796+
3797+
if (!writer->use_stack_buffer) {
3798+
assert(writer->buffer != NULL);
3799+
assert(PyBytes_CheckExact(writer->buffer));
3800+
assert(Py_REFCNT(writer->buffer) == 1);
3801+
}
3802+
else {
3803+
assert(writer->buffer == NULL);
3804+
}
3805+
3806+
start = _PyBytesWriter_AsString(writer);
3807+
assert(0 <= writer->size && writer->size <= writer->allocated);
3808+
/* the last byte must always be null */
3809+
assert(start[writer->allocated] == 0);
3810+
3811+
end = start + writer->allocated;
3812+
assert(str != NULL);
3813+
assert(start <= str && str <= end);
3814+
#endif
3815+
}
3816+
3817+
char*
3818+
_PyBytesWriter_Prepare(_PyBytesWriter *writer, char *str, Py_ssize_t size)
3819+
{
3820+
Py_ssize_t allocated, pos;
3821+
3822+
_PyBytesWriter_CheckConsistency(writer, str);
3823+
assert(size >= 0);
3824+
3825+
if (size == 0) {
3826+
/* nothing to do */
3827+
return str;
3828+
}
3829+
3830+
if (writer->size > PY_SSIZE_T_MAX - size) {
3831+
PyErr_NoMemory();
3832+
_PyBytesWriter_Dealloc(writer);
3833+
return NULL;
3834+
}
3835+
writer->size += size;
3836+
3837+
allocated = writer->allocated;
3838+
if (writer->size <= allocated)
3839+
return str;
3840+
3841+
allocated = writer->size;
3842+
if (writer->overallocate
3843+
&& allocated <= (PY_SSIZE_T_MAX - allocated / OVERALLOCATE_FACTOR)) {
3844+
/* overallocate to limit the number of realloc() */
3845+
allocated += allocated / OVERALLOCATE_FACTOR;
3846+
}
3847+
3848+
pos = _PyBytesWriter_GetPos(writer, str);
3849+
if (!writer->use_stack_buffer) {
3850+
/* Note: Don't use a bytearray object because the conversion from
3851+
byterray to bytes requires to copy all bytes. */
3852+
if (_PyBytes_Resize(&writer->buffer, allocated)) {
3853+
assert(writer->buffer == NULL);
3854+
return NULL;
3855+
}
3856+
}
3857+
else {
3858+
/* convert from stack buffer to bytes object buffer */
3859+
assert(writer->buffer == NULL);
3860+
3861+
writer->buffer = PyBytes_FromStringAndSize(NULL, allocated);
3862+
if (writer->buffer == NULL)
3863+
return NULL;
3864+
3865+
if (pos != 0) {
3866+
Py_MEMCPY(PyBytes_AS_STRING(writer->buffer),
3867+
writer->stack_buffer,
3868+
pos);
3869+
}
3870+
3871+
#ifdef Py_DEBUG
3872+
memset(writer->stack_buffer, 0xDB, sizeof(writer->stack_buffer));
3873+
#endif
3874+
3875+
writer->use_stack_buffer = 0;
3876+
}
3877+
writer->allocated = allocated;
3878+
3879+
str = _PyBytesWriter_AsString(writer) + pos;
3880+
_PyBytesWriter_CheckConsistency(writer, str);
3881+
return str;
3882+
}
3883+
3884+
/* Allocate the buffer to write size bytes.
3885+
Return the pointer to the beginning of buffer data.
3886+
Raise an exception and return NULL on error. */
3887+
char*
3888+
_PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size)
3889+
{
3890+
/* ensure that _PyBytesWriter_Alloc() is only called once */
3891+
assert(writer->size == 0 && writer->buffer == NULL);
3892+
assert(size >= 0);
3893+
3894+
writer->use_stack_buffer = 1;
3895+
#if Py_DEBUG
3896+
/* the last byte is reserved, it must be '\0' */
3897+
writer->stack_buffer[sizeof(writer->stack_buffer) - 1] = 0;
3898+
writer->allocated = sizeof(writer->stack_buffer) - 1;
3899+
#else
3900+
writer->allocated = sizeof(writer->stack_buffer);
3901+
#endif
3902+
return _PyBytesWriter_Prepare(writer, writer->stack_buffer, size);
3903+
}
3904+
3905+
PyObject *
3906+
_PyBytesWriter_Finish(_PyBytesWriter *writer, char *str)
3907+
{
3908+
Py_ssize_t pos;
3909+
PyObject *result;
3910+
3911+
_PyBytesWriter_CheckConsistency(writer, str);
3912+
3913+
pos = _PyBytesWriter_GetPos(writer, str);
3914+
if (!writer->use_stack_buffer) {
3915+
if (pos != writer->allocated) {
3916+
if (_PyBytes_Resize(&writer->buffer, pos)) {
3917+
assert(writer->buffer == NULL);
3918+
return NULL;
3919+
}
3920+
}
3921+
3922+
result = writer->buffer;
3923+
writer->buffer = NULL;
3924+
}
3925+
else {
3926+
result = PyBytes_FromStringAndSize(writer->stack_buffer, pos);
3927+
}
3928+
3929+
return result;
3930+
}

0 commit comments

Comments
 (0)