Prev: How to test/troubshoot an extension (pylibconfig)?
Next: The real problem with Python 3 - no business case for conversion (was "I strongly dislike Python 3")
From: Alf P. Steinbach /Usenet on 7 Jul 2010 22:57
The code below, very much work in progress, just trying things, is C++.

Sorry about the formatting, I had to reformat manually for this posting:


<code>
class Module
{
private:
Ptr p_;

public:
Module( PyModuleDef const& def )
: p_( ::PyModule_Create( const_cast< PyModuleDef* >( &def ) ) )
{
(p_.get() != 0) || cppx::throwX( "Module::<init>: failed" );
}

PyObject* rawPtr() const { return p_.get(); }
PyObject* release() { return p_.release(); }

void setDocString( wchar_t const s[] )
{
Ptr const v = ::PyUnicode_FromWideChar( s, -1 );
(v.get() != 0)
|| cppx::throwX( "PyUnicode_FromWideChar failed" );
int const _ = ::PyObject_SetAttrString(
p_.get(), "__doc__", v.get()
);
(_ != -1 )
|| cppx::throwX( "PyObject_SetAttrString failed" );
}

void setRoutine(
char const name[], PyCFunction f, char const doc[] = ""
)
{
PyMethodDef def = { name, f, METH_VARARGS, doc };

Ptr const pyName = ::PyUnicode_FromString( name );
Ptr r = ::PyCFunction_NewEx(
&def, p_.get(), pyName.get()
);

int const _ = ::PyModule_AddObject( p_.get(), name, r.release() );
(_ != -1 )
|| cppx::throwX( "PyModule_AddObject failed" );
}
};
</code>


Problem: when a routine installed by 'setRoutine' above is called from Python,
then it fails with e.g. "SystemError: Bad call flags in PyCFunction_Call.
METH_OLDARGS is no longer supported!"

And since things work for a single method when I declare 'def' as 'static', I
suspect that means that the function object created by PyCFunction_NewEx holds
on to a pointer to the PyMethodDef structure?

I'm unable to find documentation of PyCFunction_NewEx, and more criticially, I'm
unable to find any documented way to turn a C or C++ function into a Python
function object?


Cheers,

- Alf

--
blog at <url: http://alfps.wordpress.com>
From: Martin v. Loewis on 8 Jul 2010 01:23
> And since things work for a single method when I declare 'def' as
> 'static', I suspect that means that the function object created by
> PyCFunction_NewEx holds on to a pointer to the PyMethodDef structure?

Correct; it doesn't make a copy of the struct. So when you want the
function object to outlive the setRoutine call, you need to allocate
the PyMethodDef on the heap.

Regards,
Martin
From: Alf P. Steinbach /Usenet on 8 Jul 2010 02:47
* Martin v. Loewis, on 08.07.2010 07:23:
>> And since things work for a single method when I declare 'def' as
>> 'static', I suspect that means that the function object created by
>> PyCFunction_NewEx holds on to a pointer to the PyMethodDef structure?
>
> Correct; it doesn't make a copy of the struct. So when you want the
> function object to outlive the setRoutine call, you need to allocate
> the PyMethodDef on the heap.

Thanks!

That's the direction I tentatively had started to investigate.

But second problem now is cleanup: I'd like to deallocate when the module is
freed. I tried (1) adding a __del__, but no dice, I guess because it wasn't
really an object method but just a free function in a module; and (2) the m_free
callback in the module definition structure, but it was not called.

Perhaps I don't need to to clean up?

Anyway, current looks like this:


<code>
// progrock.cppy -- "C++ plus Python"
// A simple C++ framework for writing Python 3.x extensions.
//
// Copyright (C) Alf P. Steinbach, 2010.

#ifndef CPPY_MODULE_H
#define CPPY_MODULE_H
#include <progrock/cppx/devsupport/better_experience.h>


//----------------------------------------- Dependencies:

#include "Ptr.h"
#include <progrock/cppx/exception/throwing.h>
#include <list>



//----------------------------------------- Interface:

namespace progrock{ namespace cppy {

namespace detail {
inline PyModuleDef* moduleDefPtr()
{
static PyMethodDef methodDefs[] = {
//...
//{ "system", &pyni_system, METH_VARARGS, "Execute a shell
command." },
//{ "__del__", &onModuleDestroy, METH_NOARGS, "Destructor" },
//...
{ NULL, NULL, 0, NULL } // Sentinel
};

static PyModuleDef moduleDef = {
PyModuleDef_HEAD_INIT,
"cppy", // name of module
NULL, // m_doc, // module documentation in UTF-8
sizeof(void*), // size of per-interpreter state of the
module,
// or -1 if the module keeps state in global
variables.
methodDefs,
NULL, // m_reload
NULL, // m_traverse
NULL, // m_clear
NULL // m_free
};

return &moduleDef;
}
}

class Module
{
private:
struct InstanceData
{
std::list< PyMethodDef > methodDefs;
};

void* instanceMemory() const
{
void* const result = PyModule_GetState( p_.get() );
assert( result != 0 );
return result;
}

InstanceData*& instanceDataPtr() const
{
return *reinterpret_cast< InstanceData** >( instanceMemory() );
}

Ptr p_;
InstanceData* data_;

public:
Module()
: p_( ::PyModule_Create( detail::moduleDefPtr() ) )
{
assert( def.m_size == sizeof( void* ) );
(p_.get() != 0) || cppx::throwX( "Module::<init>: failed" );
instanceDataPtr() = data_ = new InstanceData();
}

PyObject* rawPtr() const { return p_.get(); }
PyObject* release() { return p_.release(); }

void setDocString( wchar_t const s[] )
{
Ptr const v = ::PyUnicode_FromWideChar( s, -1 );
(v.get() != 0)
|| cppx::throwX( "Module::setDocString: PyUnicode_FromWideChar
failed" );
::PyObject_SetAttrString( p_.get(), "__doc__", v.get() )
>> cppx::is( cppx::notMinusOne )
|| cppx::throwX( "Module::setDocString: PyObject_SetAttrString
failed" );
}

void addRoutine( char const name[], PyCFunction f, char const doc[] = "" )
{
PyMethodDef const defData = { name, f, METH_VARARGS, doc };

data_->methodDefs.push_back( defData );
try
{
PyMethodDef* pDef = &data_->methodDefs.back();

Ptr const pyName = ::PyUnicode_FromString( name );
Ptr r = ::PyCFunction_NewEx( pDef, p_.get(),
pyName.get());

::PyModule_AddObject( p_.get(), name, r.release() )
>> cppx::is( cppx::notMinusOne )
|| cppx::throwX( "Module::addRoutine: PyModule_AddObject
failed" );
}
catch( ... )
{
data_->methodDefs.pop_back();
throw;
}
}
};

} } // namespace progrock::cppy


#endif
</code>


I changed the module name from "pyni*" to "cppy"... ;-)


Cheers & thanks!, but how to clean up, or must I?

- Alf

--
blog at <url: http://alfps.wordpress.com>
From: Martin v. Loewis on 8 Jul 2010 03:13
> I tried (1) adding a __del__, but no dice, I guess
> because it wasn't really an object method but just a free function in a
> module; and (2) the m_free callback in the module definition structure,
> but it was not called.

m_free will be called if the module object gets deallocated. So if
m_free really isn't called, the module got never deallocated.

Regards,
Martin
From: Alf P. Steinbach /Usenet on 8 Jul 2010 04:48
* Martin v. Loewis, on 08.07.2010 09:13:
>> I tried (1) adding a __del__, but no dice, I guess
>> because it wasn't really an object method but just a free function in a
>> module; and (2) the m_free callback in the module definition structure,
>> but it was not called.
>
> m_free will be called if the module object gets deallocated. So if
> m_free really isn't called, the module got never deallocated.

Thanks again. I don't know what I did wrong. Now it's called. :-)

But I wasted much time googling to try to find out the /responsibilities/ of the
m_free callback, and what its void* argument was. E.g., should it deallocate the
module object, and if so, via what deallocation routine? I found some info, but
even your PEP, otherwise clear, was silent about this fine point.

Finally I looked at the source code that invokes it and found that it has no
responsibilities whatsoever, just a use-as-you-wish finalization callback. Nice!

But I think that could be more clear in the docs...

Code, for those who might be interested:


<code>
// progrock.cppy -- "C++ plus Python"
// A simple C++ framework for writing Python 3.x extensions.
//
// Copyright (C) Alf P. Steinbach, 2010.

#ifndef CPPY_MODULE_H
#define CPPY_MODULE_H
#include <progrock/cppx/devsupport/better_experience.h>


//----------------------------------------- Dependencies:

#include "Ptr.h"
#include <progrock/cppx/exception/throwing.h>
#include <list>



//----------------------------------------- Interface:

namespace progrock{ namespace cppy {

namespace detail {
struct InstanceData
{
std::list< PyMethodDef > methodDefs;
};

inline void* instanceMemoryOf( PyObject* pObject )
{
void* const result = PyModule_GetState( pObject );
assert( result != 0 );
return result;
}

inline InstanceData*& instanceDataPtrOf( PyObject* pObject )
{
return *reinterpret_cast< InstanceData** >(
instanceMemoryOf( pObject )
);
}

inline void on_freeModule( void* p )
{
PyObject* const pModule = reinterpret_cast< PyObject* >( p );
InstanceData* const pData = instanceDataPtrOf( pModule );

delete pData;
printf( "Deallocated!\n" ); // TODO: Remove.
}

inline PyModuleDef* moduleDefPtr()
{
static PyMethodDef methodDefs[] = {
//...
//{ "system", &pyni_system, METH_VARARGS, "Execute a shell
command." },
//{ "__del__", &onModuleDestroy, METH_NOARGS, "Destructor" },
//...
{ NULL, NULL, 0, NULL } // Sentinel
};

static PyModuleDef moduleDef = {
PyModuleDef_HEAD_INIT,
"cppy", // name of module
NULL, // m_doc, // module documentation in UTF-8
sizeof(void*), // size of per-interpreter state of the
module,
// or -1 if the module keeps state in global
variables.
methodDefs,
NULL, // m_reload
NULL, // m_traverse
NULL, // m_clear
&on_freeModule // m_free
};

return &moduleDef;
}
}

class Module
{
private:
Ptr p_;
detail::InstanceData* data_;

detail::InstanceData*& instanceDataPtr() const
{
return detail::instanceDataPtrOf( p_.get() );
}

public:
Module()
: p_( ::PyModule_Create( detail::moduleDefPtr() ) )
, data_( 0 )
{
assert( detail::moduleDefPtr()->m_size == sizeof( void* ) );
(p_.get() != 0) || cppx::throwX( "Module::<init>: failed" );
instanceDataPtr() = data_ = new detail::InstanceData();
}

PyObject* rawPtr() const { return p_.get(); }
PyObject* release() { return p_.release(); }

void setDocString( wchar_t const s[] )
{
Ptr const v = ::PyUnicode_FromWideChar( s, -1 );
(v.get() != 0)
|| cppx::throwX( "Module::setDocString: PyUnicode_FromWideChar
failed" );

::PyObject_SetAttrString( p_.get(), "__doc__", v.get() )
>> cppx::is( cppx::notMinusOne )
|| cppx::throwX( "Module::setDocString: PyObject_SetAttrString
failed" );
}

void addRoutine( char const name[], PyCFunction f, char const doc[] = "" )
{
PyMethodDef const defData = { name, f, METH_VARARGS, doc };

data_->methodDefs.push_back( defData );
try
{
PyMethodDef* const pDef = &data_->methodDefs.back();

Ptr const pyName = ::PyUnicode_FromString( name );
(pyName.get() != 0)
|| cppx::throwX( "Module::addRoutine: PyUnicode_FromString
failed" );

Ptr r = ::PyCFunction_NewEx( pDef, p_.get(),
pyName.get());
(r.get() != 0)
|| cppx::throwX( "Module::addRoutine: PyCFunction_NewEx
failed" );

::PyModule_AddObject( p_.get(), name, r.release() )
>> cppx::is( cppx::notMinusOne )
|| cppx::throwX( "Module::addRoutine: PyModule_AddObject
failed" );
}
catch( ... )
{
data_->methodDefs.pop_back();
throw;
}
}
};

} } // namespace progrock::cppy


#endif
</code>


Cheers,

- Alf

--
blog at <url: http://alfps.wordpress.com>
 | 
Pages: 1
Prev: How to test/troubshoot an extension (pylibconfig)?
Next: The real problem with Python 3 - no business case for conversion (was "I strongly dislike Python 3")