cxPython.h
@import "Ceda/cxObject/Object.h"
@import "Ceda/cxObject/SubString.h"
@import "Ceda/cxObject/NameSpaceRegistry.h"
@import "Ceda/cxObject/ReflectionByteCode.h"
CEDA_DEFINE_PROJECT_API_MACRO
// Support for compiling a debug dll or exe against the Release build of Python. This was lifted
// from the boost.python library.
#ifdef _DEBUG
# undef _DEBUG // Don't let Python force the debug library just because we're debugging.
# define NEED_TO_RESTORE_DEBUG_FLAG
#endif
#define HAVE_ROUND 1
// todo: this is needed for Python 2.x, not for Python 3.x
#if defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic push
// C:\python27x64\include\unicodeobject.h(534,5): error : ISO C++17 does not allow 'register' storage class specifier [-Wregister]
#pragma GCC diagnostic ignored "-Wregister"
#endif
#include <Python.h>
// todo: this is needed for Python 2.x, not for Python 3.x
#if defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic pop
#endif
#ifdef NEED_TO_RESTORE_DEBUG_FLAG
# undef NEED_TO_RESTORE_DEBUG_FLAG
# define _DEBUG
#endif
/*
Values and variables in Python
------------------------------
Consider the following example
a = 1
b = a
b = 2
print `a` # prints 1
a = [1,2,3]
b = a
b.append(4)
print `a` # prints [1,2,3,4]
It appears that integers have "value semantics" whereas lists have "reference semantics" - because
assignment on lists provides an alias.
However the distinction doesn't come from the treatment of the assignment (b = a). This *always*
creates an alias - Python simply copies the PyObject pointer and bumps the ref count.
The distinction arises because PyObjects representing numbers are immutable. Therefore we can
have many variables that point at the same underlying PyObject representing the value 7 and pretend
that each variable has independently stored the value 7. The only way to change the value of an
integer variable is to assign it to a *different* PyObject.
Strings
-------
In Python strings are treated like numbers. They are immutable so they behave like a value
s = "hello"
There is a PyObject variable with the value "hello". The variable is immutable so a pointer to the
variable is logically equivalent to storing the string value directly.
Could this technique be useful in C++? Consider a String class with no mutative functions, and a
ref count. This string can be passed around by pointer, and yet logically we think of strings
being passed around by value. That seems rather useful! There may be significant overheads caused
by multithreading protection of the ref count. Would need to understand the performance overheads
of InterlockedIncrement() and InterlockedDecrement().
*/
// Extern so that the functions are easily callable from Python.
extern "C"
{
// Initialise cxPython for use as an extension module (i.e. without initialising the python interpreter).
// Also calls Register_Project_cxPython().
@api void InitialisePythonExt();
// cxUtils!SetTraceFile is not an extern "C" function, so can't be called
// directly from Python using ctypes (also, the xstring argument would not
// be handled correctly).
$function+ void SetTraceFile(const ceda::char8* path, bool appendToExistingFile, bool traceTime);
}
/*
PyObject is actually a typedef defined in object.h of the python library, so we can't forward declare it using
$struct PyObject;
because the C++ compiler will balk.
$extern allows for declaring its existence to Xcpp so we can reflect functions using PyObject pointers.
*/
$extern+ PyObject;
namespace ceda
{
// Calls Py_Initialize(), adds the module bootstrapceda, sets up namespaces 'rootnamespace' and 'ceda'
@api bool InitialisePython(xstring& errorString);
///////////////////////////////////////////////////////////////////////////////////////////////
// Binding to Python sys.stdout and sys.stderr
$interface+ IPythonSysOutCallBacks
{
void OnWriteToStdOut(SubString8 s);
void OnWriteToStdErr(SubString8 s);
};
$function+ void PythonCallback_WriteToStdOut(SubString8 s);
$function+ void PythonCallback_WriteToStdErr(SubString8 s);
$function+ void SetTraceFile2(const ceda::char8* path, bool appendToExistingFile, bool traceTime);
// Allows for intercepting all output to sys.stdout or sys.stderr.
// Must have already called InitialisePython().
// This calls PyRun_SimpleString to run some python script.
// Returns false if the call to PyRun_SimpleString failed.
@api bool BindPythonSysOut(ptr<IPythonSysOutCallBacks> cb);
// Calls BindPythonSysOut() using an implementation of IPythonSysOutCallBacks that simply
// forwards the calls to the ceda Tracer()
// Returns false if the call to PyRun_SimpleString failed.
$function+ bool BindPythonSysOutputToTracer();
///////////////////////////////////////////////////////////////////////////////////////////////
// Temporary
struct @api Temporary
{
Temporary(ReflectionByteCode rbc, octet_t* self);
~Temporary();
ReflectionByteCode m_rbc;
octet_t* m_self;
Temporary* m_next;
};
///////////////////////////////////////////////////////////////////////////////////////////////////
// TemporaryList
class @api TemporaryList
{
public:
TemporaryList();
~TemporaryList();
void AddTemp(ReflectionByteCode rbc, octet_t* self);
private:
Temporary* m_first;
};
///////////////////////////////////////////////////////////////////////////////////////////////
struct ReflectedClass;
struct ReflectedVariant;
// Assign from PyObject
@api bool AssignReflectedInterfacePtrVariableFromPyObject(const ReflectedInterface& ri, AnyInterface& lhs, PyObject* rhs);
@api bool AssignReflectedVectorFromPyObject(ReflectionByteCode rbc, void* lhs, PyObject* rhs);
@api bool AssignReflectedClassVariableFromPyObject(const ReflectedClass& rc, void* lhs, PyObject* rhs);
@api bool AssignReflectedVariantVariableFromPyObject(const ReflectedVariant& rv, void* lhs, PyObject* rhs);
@api bool AssignReflectedVariableFromPyObject(ReflectionByteCode rbc, void* lhs, PyObject* rhs);
@api bool AssignReflectedPointerVariableFromPyObject(ReflectionByteCode rbc, void** lhs, PyObject* rhs, TemporaryList* temp);
@api bool AssignModelFieldVariableFromPyObject(ptr<IObject> datasource, const FieldPath& path,
ReflectionByteCode rbc, void* lhs, PyObject* rhs);
// Copy construct from PyObject
@api bool CopyConstructReflectedClassVariableFromPyObject(const ReflectedClass& rc, void* lhs, PyObject* rhs);
@api bool CopyConstructReflectedVariantVariableFromPyObject(const ReflectedVariant& rv, void* lhs, PyObject* rhs);
@api bool CopyConstructReflectedVectorFromPyObject(ReflectionByteCode rbc, void* lhs, PyObject* rhs);
@api bool CopyConstructReflectedVariableFromPyObject(ReflectionByteCode rbc, void* lhs, PyObject* rhs, TemporaryList* temp);
const bool CEDA_ACCESS_READ_ONLY = true;
const bool CEDA_ACCESS_MUTABLE = false;
// Convert to PyObject
@api PyObject* GetPyObjectFromPointerValue(ReflectionByteCode rbc, void* p);
@api PyObject* GetPyObjectFromReflectedInterfacePtr(const ReflectedInterface& ri, AnyInterface ai, bool readOnly);
@api PyObject* GetPyObjectFromFunctor(const ReflectedFunctor& rf, FunctionPointer functionPtr);
@api PyObject* GetPyObjectFromReflectedClassVariable(const ReflectedClass& rc, void* self, bool owned, bool readOnly);
@api PyObject* GetPyObjectFromReflectedVariantVariable(const ReflectedVariant& rv, void* self, bool owned, bool readOnly);
@api PyObject* GetPyObjectFromReflectedVariable(ReflectionByteCode rbc, void* self, bool readOnly);
@api PyObject* GetPyObjectFromReflectedModelVariable(
ptr<IObject> datasource, const FieldPath& path,ReflectionByteCode rbc, void* data, bool readOnly);
// The list of all python wrappers in the cxPython library
@def PythonWrappersList =
{
[
ArrayVar,
AttributeOnInterfacePtr,
CedaBuiltInType,
Class,
ClassVariable,
Enum,
Functor,
GlobalFunction,
InterfacePtr,
MapIterator,
MapVar,
MethodOnClassVariable,
MethodOnInterfacePtr,
ModelArrayVar,
ModelMapVar,
ModelStructVar,
ModelVectorVar,
NameSpace,
PointerValue,
Variant,
VariantConstructor,
VariantVariable,
Vector
]
}
/*
todo:
We have a bunch of structs which are used for reflected functions/objects/variables
ClassVariable_Py
VariantVariable_Py
ArrayVar_Py
MapVar_Py
MapIterator_Py
PointerValue_Py
InterfacePtr_Py
MethodOnInterfacePtr_Py
MethodOnClassVariable_Py
AttributeOnInterfacePtr_Py
Functor_Py
NameSpace_Py
Enum_Py
CedaBuiltInType_Py
Class_Py
Variant_Py
VariantConstructor_Py
Vector_Py
ModelStructVar_Py
ModelVectorVar_Py
ModelArrayVar_Py
ModelMapVar_Py
There is nothing in these structs that mentions Python.
These structs should be renamed and moved into the cxObject library.
The names become:
types:
R_Class
R_Variant
R_Interface
R_Map
R_Set
R_Enum
R_Vector
R_VariantCtor
variables:
RV_Class
RV_Variant
RV_Map
RV_MapIt
RV_Array
RV_Enum
RV_Vector
RV_Pointer
RV_Ptr
model variables:
RMV_Class
RMV_Vector
RMV_Array
RMV_Map
All the handy functions for working with them should be implemented in cxObject and tested in txObject.
Also, they should instead work with ReflectedType instead of ReflectionByteCode.
They should no longer use the 'm_' prefix on the variabe names.
We can create all these now, before making any changes to the cxPython library.
It is expected this will end up simplifying the cxPython library.
*/
struct ClassVariable_Py
{
void* m_self; // Ptr to the class/struct variable. Never NULL
const ReflectedClass* m_rc; // Reflection for the class/struct
bool m_owned; // Does this ClassVariable_Py own the (heap allocated) variable?
bool m_readOnly; // Do we impose read-only access to the variable?
};
struct VariantVariable_Py
{
void* m_self; // Ptr to the variant variable. Never NULL
const ReflectedVariant* m_rv; // Reflection for the variant
bool m_owned; // Does this VariantVariable_Py own the (heap allocated) variable?
bool m_readOnly; // Do we impose read-only access to the variable?
};
struct ArrayVar_Py
{
// The address of the array. Never NULL
void* m_array;
// The number of elements in the array.
ssize_t size;
// Describes the type of the element
ReflectionByteCode m_rbc;
bool m_owned;
bool m_readOnly; // Do we impose read-only access to the array variable?
};
/*
todo: Is this redundant? It exactly corresponds to using a PointerValue_Py
*/
struct MapVar_Py
{
// The address of the map. Never NULL
void* m_map;
// Describes the type of the xmap<K,V>
ReflectionByteCode m_rbc;
bool m_owned; // Does this MapVar_Py own the (heap allocated) variable?
bool m_readOnly; // Do we impose read-only access to the array variable?
};
enum class MapIteratorType
{
Keys,
Values,
Items
};
struct MapIterator_Py
{
// The address of the map
void* m_map;
// Describes the xmap type
ReflectionByteCode m_rbc;
ReflectionByteCode m_rbcKey;
ReflectionByteCode m_rbcPayload;
ssize_t m_payLoadOffset;
ptr<IBiDirIterator> m_iterator;
bool m_readOnly;
MapIteratorType m_type;
};
/*
Wraps a nun-null pointer value.
(null pointer values are represented with None)
If we regard the pointer value as being of type T* then the byte code defines the type T.
A PointerValue_Py is able to represent all kinds of variables and values.
However it is used sparingly - a more specific implementation is used in many cases.
Note that PointerValue_Py resembles a ceda::ReflectedInstance
(see Ceda/cxObject/ConstructDestructCopyAssignReflectedVariable.h)
For example PointerValue_Py can represent an int64 value by using FT_INT64 in the byte code,
and a heap allocated int64 and using m_owned=true and m_readOnly=true
However, we instead always represent an int64 value using a native PyLong object.
*/
struct PointerValue_Py
{
void* m_pointer; // Never NULL
// Reflection information about the type of a variable which the pointer value may point at
// This can be FT_VOID which means we have no type information.
ReflectionByteCode m_rbc;
bool m_owned; // Does this PointerValue_Py own the (heap allocated) variable?
bool m_readOnly; // Do we impose read-only access to the pointed at variable?
};
// Represents a particular interface pointer together with reflection information.
struct InterfacePtr_Py
{
// Interface pointer
AnyInterface m_anyInterface; // Never null
// Reflection information about the interface
const ReflectedInterface* m_ri;
bool m_readOnly;
};
// Represents a particular method on a particular interface pointer, together with reflection
// information.
struct MethodOnInterfacePtr_Py
{
// Interface pointer
AnyInterface m_anyInterface;
// Index of method to be invoked
ssize_t m_methodIndex;
// Reflection information about the interface
const ReflectedInterface* m_ri;
};
// Represents a particular method on a particular class variable, together with reflection
// information.
struct MethodOnClassVariable_Py
{
// Ptr to class variable
void* m_variable;
// Index of method to be invoked
ssize_t m_methodIndex;
// Reflection information about the class
const ReflectedClass* m_rc;
};
// Represents a particular attribute on a particular interface pointer, together with reflection
// information.
struct AttributeOnInterfacePtr_Py
{
// Interface pointer
AnyInterface m_anyInterface;
// Index of attribute to be accessed
ssize_t m_attributeIndex;
// Reflection information about the interface
const ReflectedInterface* m_ri;
};
struct GlobalFunction_Py
{
const ReflectedGlobalFunction* m_rgf;
//PyMethodDef m_method;
};
struct Functor_Py
{
const ReflectedFunctor* m_f;
FunctionPointer m_fnPointer;
void* m_self;
};
struct NameSpace_Py
{
NameSpace* m_ns;
};
struct Enum_Py
{
const ReflectedEnum* m_re;
};
/*
Associated with a Python object like ceda.int32 or ceda.string8 that is callable
and creates instances of heap allocated variables, that are referenced using a
PyObject of PointerValue_Py with m_owned = true.
*/
struct CedaBuiltInType_Py
{
// Describes the type of the ceda builtin type. Eg int32.
ReflectionByteCode m_rbc;
};
struct Class_Py
{
const ReflectedClass* m_rc;
PyMethodDef m_newMethod;
PyMethodDef m_createMethod;
};
struct Variant_Py
{
const ReflectedVariant* m_rv;
};
// Given a variant type V with a named field f, V.f represents a callable python object used to create
// instances of V of kind f.
struct VariantConstructor_Py
{
const ReflectedVariant* m_rv;
ssize_t m_fieldIndex; // Index of ReflectedField in m_rv->fields array;
};
/*
This is almost equivalent to a PointerValue_Py, except m_rbc describes the element type, not the vector itself
*/
struct Vector_Py
{
void* m_vector;
// Describes the type of the element
ReflectionByteCode m_rbc;
bool m_owned;
bool m_readOnly; // Do we impose read-only access to the vector itself?
};
///////////////////////////////////////////////////////////////////////////////////////////////
// Wrapping of models
//ModelStructVar // A struct
//ModelVectorVar // a vector
//ModelArrayVar // an array
//ModelMapVar // a map
struct ModelStructVar_Py
{
// Pointer to the containing datasource
ptr<IObject> m_datasource;
// The path down through the datasource to this model
FieldPath m_path;
// The address of this data struct
void* m_self;
// The type of this model
const ReflectedClass* m_rc;
bool m_readOnly;
};
struct ModelVectorVar_Py
{
// Pointer to the containing datasource
ptr<IObject> m_datasource;
// The path down through the datasource to this vector
FieldPath m_path;
// The address of the vector
void* m_vector;
// Describes the type of the element
ReflectionByteCode m_rbc;
bool m_isString; // This vector represents a string data type
bool m_readOnly;
};
struct ModelArrayVar_Py
{
// Pointer to the containing datasource
ptr<IObject> m_datasource;
// The path down through the datasource to this array
FieldPath m_path;
// The address of the array
void* m_array;
// The number of elements in the array.
ssize_t size;
// Describes the type of the element
ReflectionByteCode m_rbc;
bool m_readOnly;
};
struct ModelMapVar_Py
{
// Pointer to the containing datasource
ptr<IObject> m_datasource;
// The path down through the datasource to this map
FieldPath m_path;
// The address of the map
void* m_map;
// Describes the xmap type
ReflectionByteCode m_rbc;
bool m_readOnly;
};
///////////////////////////////////////////////////////////////////////////////////////////////
// ModelVariable
/*
Notes:
* The path is never empty because that would correspond to a reference to the datasource
itself, but we already have ClassVariable_Py for that purpose.
* We can distinguish between a member of a model versus an element of an array or map.
The path takes all this into account.
How we got here is only relevant for generating an operation.
* The type of the variable can be an array, map, vector, mvector, int, assign<T> etc
There is a huge amount of variation.
This variation in turn affects its appearance.
As an example, to generate an operation we may need to call
void AssignValue4(ptr<IObject> obj, const FieldPath& p, void* field, int32 newValue);
This shows that we need to store...
1) The pointer to the containing datasource
2) The path to the field
3) The address of the field
The read barrier should only be called on access to *stable* fields on the datasource.
i.e. it must not recurse into vectors, sets, bags, maps. Only nested models and fixed
arrays are stable.
void DataSourceReadBarrier(const void* fieldAddress, ptr<const IObject> obj);
*/
/*
struct ModelVariable_Py
{
// The type of the containing datasource
const ReflectedClass* m_rcDataSource;
// The address of the containing datasource
void* m_datasource;
// The type of the containing model. May equal m_rcDataSource if this field is
// directly under the datasource.
//const ReflectedClass* m_rcModel;
// The index of this field within m_rcModel->modelFields
//ssize_t m_fieldIndex;
// The path down through the datasource to this variable
FieldPath m_path;
// Describes the type of this variable
ReflectionByteCode m_rbc;
// The address of this variable
void* field;
};
*/
///////////////////////////////////////////////////////////////////////////////////////////////
// Generic functions
template<typename C>
struct PyContentTraits
{
};
@for(name in PythonWrappersList)
{
@api PyTypeObject& GetThePyTypeObject_@@name();
template<>
struct PyContentTraits<name@@_Py>
{
static PyTypeObject& GetThePyTypeObject() { return GetThePyTypeObject_@@name(); }
};
@api PyObject* GetPyObjectFromContent(name@@_Py content, PyObject* strongRef = NULL);
// Deprecated. Instead use GetWrappedContentOfPyObject
@api bool PyObjectWrapsContent(PyObject* p, name@@_Py& content);
@api void GetWrappedContentOfPyObject(PyObject* p, name@@_Py*& w);
}
template<typename C>
PyObject* ContentTypeToPyObject()
{
/*
C c;
PyObject* p = GetPyObjectFromContent(c);
PyObject* t = PyObject_Type(p);
Py_DECREF(p);
return t;
*/
PyObject* p = (PyObject*) &PyContentTraits<C>::GetThePyTypeObject();
Py_INCREF(p);
return p;
}
template<typename C>
C* GetWrappedContentOfPyObject(PyObject* p)
{
C* w;
GetWrappedContentOfPyObject(p, w);
return w;
}
}