You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mod_python-dev@quetz.apache.org by Ron Reisor <ro...@UDel.Edu> on 2003/12/09 17:38:02 UTC
memory allocation
I have been doing some development using:
mod_python-3.1.2b
Python-2.3.2
apache2.0.48
MacOSX 10.3.1
With the straight distribution of mod_python-3.1.2b, the MacOSX library
free routine complains about trying to free memory it had never allocated.
Looking through the mod_python source we have calls on PyMem_NEW to
allocate memory for a new object instead of calling PyObject_NEW. With
Python-2.3 this is going to cause problems. Looking further, there are a
bunch of places where free is called instead of PyObject_DEL.
Also, the MpHList_Type structure has a size field that doesn't seem
correct. This was changed to sizeof(hlistobject).
I have been using these changes since mod_python-3.1.0a and things seem
pretty smooth on MacOSX. I just rebuilt everything using Python-2.3.3c1
and so far so good!
cheers,
Ron
context diffs:
diff -rc mod_python-3.1.2b/src/connobject.c mod_python-3.1.2b-udel/src/connobject.c
*** mod_python-3.1.2b/src/connobject.c Tue Sep 9 22:11:22 2003
--- mod_python-3.1.2b-udel/src/connobject.c Mon Dec 8 10:05:56 2003
***************
*** 80,86 ****
{
connobject *result;
! result = PyMem_NEW(connobject, 1);
if (! result)
return PyErr_NoMemory();
--- 80,86 ----
{
connobject *result;
! result = PyObject_New(connobject, &MpConn_Type);
if (! result)
return PyErr_NoMemory();
***************
*** 91,97 ****
result->notes = MpTable_FromTable(c->notes);
result->hlo = NULL;
- _Py_NewReference(result);
return (PyObject *)result;
}
--- 91,96 ----
***************
*** 322,328 ****
Py_XDECREF(self->base_server);
Py_XDECREF(self->notes);
Py_XDECREF(self->hlo);
! free(self);
}
/**
--- 321,327 ----
Py_XDECREF(self->base_server);
Py_XDECREF(self->notes);
Py_XDECREF(self->hlo);
! PyObject_Del(self);
}
/**
diff -rc mod_python-3.1.2b/src/filterobject.c mod_python-3.1.2b-udel/src/filterobject.c
*** mod_python-3.1.2b/src/filterobject.c Wed Oct 8 23:18:45 2003
--- mod_python-3.1.2b-udel/src/filterobject.c Mon Dec 8 10:07:50 2003
***************
*** 118,124 ****
{
filterobject *result;
! result = PyMem_NEW(filterobject, 1);
if (! result)
return PyErr_NoMemory();
--- 118,124 ----
{
filterobject *result;
! result = PyObject_New(filterobject, &MpFilter_Type);
if (! result)
return PyErr_NoMemory();
***************
*** 149,155 ****
result->request_obj = NULL;
- _Py_NewReference(result);
apr_pool_cleanup_register(f->r->pool, (PyObject *)result, python_decref,
apr_pool_cleanup_null);
--- 149,154 ----
***************
*** 547,553 ****
static void filter_dealloc(filterobject *self)
{
Py_XDECREF(self->request_obj);
! free(self);
}
--- 546,552 ----
static void filter_dealloc(filterobject *self)
{
Py_XDECREF(self->request_obj);
! PyObject_Del(self);
}
diff -rc mod_python-3.1.2b/src/hlistobject.c mod_python-3.1.2b-udel/src/hlistobject.c
*** mod_python-3.1.2b/src/hlistobject.c Thu Nov 7 19:15:11 2002
--- mod_python-3.1.2b-udel/src/hlistobject.c Mon Dec 8 10:10:05 2003
***************
*** 77,84 ****
hlistobject *result;
apr_pool_t *p;
! result = PyMem_NEW(hlistobject, 1);
! result->ob_type = &MpHList_Type;
if (! result)
PyErr_NoMemory();
--- 77,83 ----
hlistobject *result;
apr_pool_t *p;
! result = PyObject_New(hlistobject, &MpHList_Type);
if (! result)
PyErr_NoMemory();
***************
*** 88,94 ****
result->pool = p;
result->head = hlist_copy(p, hle);
- _Py_NewReference(result);
return (PyObject *) result;
}
--- 87,92 ----
***************
*** 144,150 ****
{
if (self->pool)
apr_pool_destroy(self->pool);
! free(self);
}
/**
--- 142,148 ----
{
if (self->pool)
apr_pool_destroy(self->pool);
! PyObject_Del(self);
}
/**
***************
*** 205,211 ****
PyObject_HEAD_INIT(NULL)
0,
"mp_hlist",
! sizeof(hl_entry),
0,
(destructor) hlist_dealloc, /*tp_dealloc*/
0, /*tp_print*/
--- 203,209 ----
PyObject_HEAD_INIT(NULL)
0,
"mp_hlist",
! sizeof(hlistobject),
0,
(destructor) hlist_dealloc, /*tp_dealloc*/
0, /*tp_print*/
diff -rc mod_python-3.1.2b/src/requestobject.c mod_python-3.1.2b-udel/src/requestobject.c
*** mod_python-3.1.2b/src/requestobject.c Tue Oct 7 23:48:17 2003
--- mod_python-3.1.2b-udel/src/requestobject.c Mon Dec 8 10:15:38 2003
***************
*** 75,81 ****
{
requestobject *result;
! result = PyMem_NEW(requestobject, 1);
if (! result)
return PyErr_NoMemory();
--- 75,81 ----
{
requestobject *result;
! result = PyObject_New(requestobject, &MpRequest_Type);
if (! result)
return PyErr_NoMemory();
***************
*** 103,110 ****
result->rbuff_pos = 0;
result->rbuff_len = 0;
- _Py_NewReference(result);
-
return (PyObject *) result;
}
--- 103,108 ----
***************
*** 1344,1350 ****
Py_XDECREF(self->phase);
Py_XDECREF(self->hlo);
! free(self);
}
static char request_doc[] =
--- 1342,1348 ----
Py_XDECREF(self->phase);
Py_XDECREF(self->hlo);
! PyObject_Del(self);
}
static char request_doc[] =
diff -rc mod_python-3.1.2b/src/serverobject.c mod_python-3.1.2b-udel/src/serverobject.c
*** mod_python-3.1.2b/src/serverobject.c Thu Oct 23 09:43:36 2003
--- mod_python-3.1.2b-udel/src/serverobject.c Mon Dec 8 10:18:25 2003
***************
*** 76,82 ****
{
serverobject *result;
! result = PyMem_NEW(serverobject, 1);
if (! result)
return PyErr_NoMemory();
--- 76,82 ----
{
serverobject *result;
! result = PyObject_New(serverobject, &MpServer_Type);
if (! result)
return PyErr_NoMemory();
***************
*** 88,94 ****
result->ob_type = &MpServer_Type;
result->next = NULL;
- _Py_NewReference(result);
return (PyObject *)result;
}
--- 88,93 ----
***************
*** 326,332 ****
{
Py_XDECREF(self->dict);
Py_XDECREF(self->next);
! free(self);
}
static char server_doc[] =
--- 325,331 ----
{
Py_XDECREF(self->dict);
Py_XDECREF(self->next);
! PyObject_Del(self);
}
static char server_doc[] =
diff -rc mod_python-3.1.2b/src/tableobject.c mod_python-3.1.2b-udel/src/tableobject.c
*** mod_python-3.1.2b/src/tableobject.c Mon Jul 14 16:51:32 2003
--- mod_python-3.1.2b-udel/src/tableobject.c Mon Dec 8 10:19:25 2003
***************
*** 88,94 ****
{
tableobject *result;
! result = PyMem_NEW(tableobject, 1);
if (! result)
return PyErr_NoMemory();
--- 88,94 ----
{
tableobject *result;
! result = PyObject_New(tableobject, &MpTable_Type);
if (! result)
return PyErr_NoMemory();
***************
*** 96,102 ****
result->ob_type = &MpTable_Type;
result->pool = NULL;
- _Py_NewReference(result);
return (PyObject *)result;
}
--- 96,101 ----
***************
*** 144,150 ****
if (MpTable_Check(self)) {
if (self->pool)
apr_pool_destroy(self->pool);
! free(self);
}
else
self->ob_type->tp_free((PyObject *)self);
--- 143,149 ----
if (MpTable_Check(self)) {
if (self->pool)
apr_pool_destroy(self->pool);
! PyObject_Del(self);
}
else
self->ob_type->tp_free((PyObject *)self);
Ron Reisor <ro...@udel.edu> (RWR3)
University of Delaware Information Technologies/Network and Systems Services
Computing Center/192 South Chapel Street/Newark DE, 19716
pgp finger print: 0D 73 06 6F D3 6A 99 D3 F5 D5 6E FF 3B B9 7C 2C
Re: memory allocation
Posted by Ron Reisor <ro...@UDel.Edu>.
On Wed, 10 Dec 2003, Gregory (Grisha) Trubetskoy wrote:
> BTW - does this break Python 2.2? I don't particularly care if it does,
> but it should be noted in the documentation then.
Grisha,
These changes should not break Python 2.2 but this is untested.
I tried to build using Python-2.2.3, ran into a little trouble, and
gave up. I've never used mod_python with any Python earlier than 2.3. We
depend on 2.3 features so there's no going back for us. Our development
is all on MacOSX.
In versions < 2.3, the different memory allocators ended up calling the
same code, so you could intermix object/general memory allocations and
frees and things would still work. The patch follows the C API for Python
2.3.2, Python 2.2.3 and Python 2.1.3. So, mod_python SHOULD still work
with 2.2, but it's untested.
cheers,
Ron
Ron Reisor <ro...@udel.edu> (RWR3)
University of Delaware Information Technologies/Network and Systems Services
Computing Center/192 South Chapel Street/Newark DE, 19716
pgp finger print: 0D 73 06 6F D3 6A 99 D3 F5 D5 6E FF 3B B9 7C 2C
Re: memory allocation
Posted by "Gregory (Grisha) Trubetskoy" <gr...@apache.org>.
Thanks Ron,
this is applied. I also removed all the assignments to ob_type since
PyObject_New seems to take care of that.
BTW - does this break Python 2.2? I don't particularly care if it does,
but it should be noted in the documentation then.
Grisha
On Tue, 9 Dec 2003, Ron Reisor wrote:
> I have been doing some development using:
>
> mod_python-3.1.2b
> Python-2.3.2
> apache2.0.48
> MacOSX 10.3.1
>
> With the straight distribution of mod_python-3.1.2b, the MacOSX library
> free routine complains about trying to free memory it had never allocated.
> Looking through the mod_python source we have calls on PyMem_NEW to
> allocate memory for a new object instead of calling PyObject_NEW. With
> Python-2.3 this is going to cause problems. Looking further, there are a
> bunch of places where free is called instead of PyObject_DEL.
>
> Also, the MpHList_Type structure has a size field that doesn't seem
> correct. This was changed to sizeof(hlistobject).
>
> I have been using these changes since mod_python-3.1.0a and things seem
> pretty smooth on MacOSX. I just rebuilt everything using Python-2.3.3c1
> and so far so good!
>
> cheers,
>
> Ron
>
> context diffs:
> diff -rc mod_python-3.1.2b/src/connobject.c mod_python-3.1.2b-udel/src/connobject.c
> *** mod_python-3.1.2b/src/connobject.c Tue Sep 9 22:11:22 2003
> --- mod_python-3.1.2b-udel/src/connobject.c Mon Dec 8 10:05:56 2003
> ***************
> *** 80,86 ****
> {
> connobject *result;
>
> ! result = PyMem_NEW(connobject, 1);
> if (! result)
> return PyErr_NoMemory();
>
> --- 80,86 ----
> {
> connobject *result;
>
> ! result = PyObject_New(connobject, &MpConn_Type);
> if (! result)
> return PyErr_NoMemory();
>
> ***************
> *** 91,97 ****
> result->notes = MpTable_FromTable(c->notes);
> result->hlo = NULL;
>
> - _Py_NewReference(result);
> return (PyObject *)result;
> }
>
> --- 91,96 ----
> ***************
> *** 322,328 ****
> Py_XDECREF(self->base_server);
> Py_XDECREF(self->notes);
> Py_XDECREF(self->hlo);
> ! free(self);
> }
>
> /**
> --- 321,327 ----
> Py_XDECREF(self->base_server);
> Py_XDECREF(self->notes);
> Py_XDECREF(self->hlo);
> ! PyObject_Del(self);
> }
>
> /**
> diff -rc mod_python-3.1.2b/src/filterobject.c mod_python-3.1.2b-udel/src/filterobject.c
> *** mod_python-3.1.2b/src/filterobject.c Wed Oct 8 23:18:45 2003
> --- mod_python-3.1.2b-udel/src/filterobject.c Mon Dec 8 10:07:50 2003
> ***************
> *** 118,124 ****
> {
> filterobject *result;
>
> ! result = PyMem_NEW(filterobject, 1);
> if (! result)
> return PyErr_NoMemory();
>
> --- 118,124 ----
> {
> filterobject *result;
>
> ! result = PyObject_New(filterobject, &MpFilter_Type);
> if (! result)
> return PyErr_NoMemory();
>
> ***************
> *** 149,155 ****
>
> result->request_obj = NULL;
>
> - _Py_NewReference(result);
> apr_pool_cleanup_register(f->r->pool, (PyObject *)result, python_decref,
> apr_pool_cleanup_null);
>
> --- 149,154 ----
> ***************
> *** 547,553 ****
> static void filter_dealloc(filterobject *self)
> {
> Py_XDECREF(self->request_obj);
> ! free(self);
> }
>
>
> --- 546,552 ----
> static void filter_dealloc(filterobject *self)
> {
> Py_XDECREF(self->request_obj);
> ! PyObject_Del(self);
> }
>
>
> diff -rc mod_python-3.1.2b/src/hlistobject.c mod_python-3.1.2b-udel/src/hlistobject.c
> *** mod_python-3.1.2b/src/hlistobject.c Thu Nov 7 19:15:11 2002
> --- mod_python-3.1.2b-udel/src/hlistobject.c Mon Dec 8 10:10:05 2003
> ***************
> *** 77,84 ****
> hlistobject *result;
> apr_pool_t *p;
>
> ! result = PyMem_NEW(hlistobject, 1);
> ! result->ob_type = &MpHList_Type;
> if (! result)
> PyErr_NoMemory();
>
> --- 77,83 ----
> hlistobject *result;
> apr_pool_t *p;
>
> ! result = PyObject_New(hlistobject, &MpHList_Type);
> if (! result)
> PyErr_NoMemory();
>
> ***************
> *** 88,94 ****
> result->pool = p;
> result->head = hlist_copy(p, hle);
>
> - _Py_NewReference(result);
> return (PyObject *) result;
> }
>
> --- 87,92 ----
> ***************
> *** 144,150 ****
> {
> if (self->pool)
> apr_pool_destroy(self->pool);
> ! free(self);
> }
>
> /**
> --- 142,148 ----
> {
> if (self->pool)
> apr_pool_destroy(self->pool);
> ! PyObject_Del(self);
> }
>
> /**
> ***************
> *** 205,211 ****
> PyObject_HEAD_INIT(NULL)
> 0,
> "mp_hlist",
> ! sizeof(hl_entry),
> 0,
> (destructor) hlist_dealloc, /*tp_dealloc*/
> 0, /*tp_print*/
> --- 203,209 ----
> PyObject_HEAD_INIT(NULL)
> 0,
> "mp_hlist",
> ! sizeof(hlistobject),
> 0,
> (destructor) hlist_dealloc, /*tp_dealloc*/
> 0, /*tp_print*/
> diff -rc mod_python-3.1.2b/src/requestobject.c mod_python-3.1.2b-udel/src/requestobject.c
> *** mod_python-3.1.2b/src/requestobject.c Tue Oct 7 23:48:17 2003
> --- mod_python-3.1.2b-udel/src/requestobject.c Mon Dec 8 10:15:38 2003
> ***************
> *** 75,81 ****
> {
> requestobject *result;
>
> ! result = PyMem_NEW(requestobject, 1);
> if (! result)
> return PyErr_NoMemory();
>
> --- 75,81 ----
> {
> requestobject *result;
>
> ! result = PyObject_New(requestobject, &MpRequest_Type);
> if (! result)
> return PyErr_NoMemory();
>
> ***************
> *** 103,110 ****
> result->rbuff_pos = 0;
> result->rbuff_len = 0;
>
> - _Py_NewReference(result);
> -
> return (PyObject *) result;
> }
>
> --- 103,108 ----
> ***************
> *** 1344,1350 ****
> Py_XDECREF(self->phase);
> Py_XDECREF(self->hlo);
>
> ! free(self);
> }
>
> static char request_doc[] =
> --- 1342,1348 ----
> Py_XDECREF(self->phase);
> Py_XDECREF(self->hlo);
>
> ! PyObject_Del(self);
> }
>
> static char request_doc[] =
> diff -rc mod_python-3.1.2b/src/serverobject.c mod_python-3.1.2b-udel/src/serverobject.c
> *** mod_python-3.1.2b/src/serverobject.c Thu Oct 23 09:43:36 2003
> --- mod_python-3.1.2b-udel/src/serverobject.c Mon Dec 8 10:18:25 2003
> ***************
> *** 76,82 ****
> {
> serverobject *result;
>
> ! result = PyMem_NEW(serverobject, 1);
> if (! result)
> return PyErr_NoMemory();
>
> --- 76,82 ----
> {
> serverobject *result;
>
> ! result = PyObject_New(serverobject, &MpServer_Type);
> if (! result)
> return PyErr_NoMemory();
>
> ***************
> *** 88,94 ****
> result->ob_type = &MpServer_Type;
> result->next = NULL;
>
> - _Py_NewReference(result);
> return (PyObject *)result;
> }
>
> --- 88,93 ----
> ***************
> *** 326,332 ****
> {
> Py_XDECREF(self->dict);
> Py_XDECREF(self->next);
> ! free(self);
> }
>
> static char server_doc[] =
> --- 325,331 ----
> {
> Py_XDECREF(self->dict);
> Py_XDECREF(self->next);
> ! PyObject_Del(self);
> }
>
> static char server_doc[] =
> diff -rc mod_python-3.1.2b/src/tableobject.c mod_python-3.1.2b-udel/src/tableobject.c
> *** mod_python-3.1.2b/src/tableobject.c Mon Jul 14 16:51:32 2003
> --- mod_python-3.1.2b-udel/src/tableobject.c Mon Dec 8 10:19:25 2003
> ***************
> *** 88,94 ****
> {
> tableobject *result;
>
> ! result = PyMem_NEW(tableobject, 1);
> if (! result)
> return PyErr_NoMemory();
>
> --- 88,94 ----
> {
> tableobject *result;
>
> ! result = PyObject_New(tableobject, &MpTable_Type);
> if (! result)
> return PyErr_NoMemory();
>
> ***************
> *** 96,102 ****
> result->ob_type = &MpTable_Type;
> result->pool = NULL;
>
> - _Py_NewReference(result);
> return (PyObject *)result;
> }
>
> --- 96,101 ----
> ***************
> *** 144,150 ****
> if (MpTable_Check(self)) {
> if (self->pool)
> apr_pool_destroy(self->pool);
> ! free(self);
> }
> else
> self->ob_type->tp_free((PyObject *)self);
> --- 143,149 ----
> if (MpTable_Check(self)) {
> if (self->pool)
> apr_pool_destroy(self->pool);
> ! PyObject_Del(self);
> }
> else
> self->ob_type->tp_free((PyObject *)self);
>
> Ron Reisor <ro...@udel.edu> (RWR3)
> University of Delaware Information Technologies/Network and Systems Services
> Computing Center/192 South Chapel Street/Newark DE, 19716
> pgp finger print: 0D 73 06 6F D3 6A 99 D3 F5 D5 6E FF 3B B9 7C 2C
>