How to call a field of CFUNCTYPE?
Question:
I have a DLL (.dll for Windows, .so for Linux) and this autogenerated C header file for that DLL (excerpt):
Edit notes
tl;dr: I have updated my header excerpt and Python code because I had left out too much.
Details: I had left out a bunch of declarations in the header that I deemed unimportant but that turned out to be crucial (revealed by the accepted answer). Additionally, I had reduced struct nesting and renamed a struct to reduce clutter. (kotlin.root.com.marcoeckstein.klib was just myLib.) But since the accepted answer was created based on the actual DLL (as requested in the comments and shared outside of Stack Overflow), I have also revised my header excerpt and Python code to be consistent with that DLL and thus the accepted answer.
typedef struct {
// #region
// The details of most of these declarations are not important,
// but as it turned out, their memory requirements in the
// binary layout are:
void (*DisposeStablePointer)(libnative_KNativePtr ptr);
void (*DisposeString)(const char* string);
libnative_KBoolean (*IsInstance)(libnative_KNativePtr ref, const libnative_KType* type);
libnative_kref_kotlin_Byte (*createNullableByte)(libnative_KByte);
libnative_kref_kotlin_Short (*createNullableShort)(libnative_KShort);
libnative_kref_kotlin_Int (*createNullableInt)(libnative_KInt);
libnative_kref_kotlin_Long (*createNullableLong)(libnative_KLong);
libnative_kref_kotlin_Float (*createNullableFloat)(libnative_KFloat);
libnative_kref_kotlin_Double (*createNullableDouble)(libnative_KDouble);
libnative_kref_kotlin_Char (*createNullableChar)(libnative_KChar);
libnative_kref_kotlin_Boolean (*createNullableBoolean)(libnative_KBoolean);
libnative_kref_kotlin_Unit (*createNullableUnit)(void);
// #endregion
struct {
struct {
struct {
struct {
struct {
const char* (*createMessage)();
// More nested structs here
// (irrelevant for this question)
} klib;
} marcoeckstein;
} com;
} root;
} kotlin;
} libnative_ExportedSymbols;
extern libnative_ExportedSymbols* libnative_symbols(void);
I want to call the function createMessage from Python. I have tried this:
import ctypes
class Klib(ctypes.Structure):
_fields_ = [("createMessage", ctypes.CFUNCTYPE(ctypes.c_char_p))]
class Marcoeckstein(ctypes.Structure):
_fields_ = [("klib", Klib)]
class Com(ctypes.Structure):
_fields_ = [("marcoeckstein", Marcoeckstein)]
class Root(ctypes.Structure):
_fields_ = [("com", Com)]
class Kotlin(ctypes.Structure):
_fields_ = [("root", Root)]
class Libnative_ExportedSymbols(ctypes.Structure):
_fields_ = [("kotlin", Kotlin)]
dll = ctypes.CDLL("path/to/dll")
#region
#The following two lines turned out to be wrong. See below.
dll.libnative_symbols.restype = Libnative_ExportedSymbols
libnative = dll.libnative_symbols()
#endregion
libnative.kotlin.root.com.marcoeckstein.klib.createMessage()
On Windows I get:
OSError: exception: access violation writing 0x00007FFA51556340
On Linux I get:
Segmentation fault (core dumped)
On Windows, I have also tried WINFUNCTYPE instead of CFUNCTYPE, but I got the same error.
The wrapper generator ctypesgen generates the same type for createMessage as I did manually.
So what is going on here? How can I call a field of CFUNCTYPE?
Partial answer (edit)
As jasonharper noted in a comment, I needed to change this code…
dll.libnative_symbols.restype = Libnative_ExportedSymbols
libnative = dll.libnative_symbols()
… to this:
dll.libnative_symbols.restype = ctypes.POINTER(Libnative_ExportedSymbols)
libnative = dll.libnative_symbols().contents
This change turned out to be necessary but not sufficient. I got no error now, but createMessage was not properly called. It returned an empty instance of bytes, but the function had been implemented to always return a non-empty string.
Additional Notes (edit)
- There is a related issue.
- The DLL and header file were automatically created by Kotlin Multiplatform/Native, but this is irrelevant for the question.
Answers:
tl;dr (edit by author of the question)
You cannot ignore the declarations before the struct kotlin. You do not necessarily need exactly fitting declarations for these in Python, but you need at least placeholders:
class Libnative_ExportedSymbols(ctypes.Structure):
_fields_ = [
("funcn", ctypes.CFUNCTYPE(None) * 12), # placeholders
("kotlin", Kotlin),
]
This is already enough to get the solution to work, but it results in a memory leak, because the string returned by createMessage uses unmanaged memory. Thus, we need the pointer to that memory, and we need to free it with the DisposeString function (a service funtion provided by the tool that built the DLL and header files).
class Klib(ctypes.Structure):
_fields_ = [("createMessage", ctypes.CFUNCTYPE(POINTER(ctypes.c_char)))]
// ...
class Libnative_ExportedSymbols(ctypes.Structure):
_fields_ = [
("func", ctypes.CFUNCTYPE(None)),
("DisposeString", ctypes.CFUNCTYPE(None, ctypes.POINTER(ctypes.c_char))),
("funcn", ctypes.CFUNCTYPE(None) * 10),
("kotlin", Kotlin),
]
Now createMessage() does not return the standard Python type bytes anymore, but ctypes.LP_c_char, which we have to convert ourselves:
def to_bytes(p):
return ctypes.c_char_p.from_buffer(p).value
pointer = libnative.kotlin.root.com.marcoeckstein.klib.createMessage()
message = toBytes(pointer) # pointer.contents is only the first char.
Finally, we free the memory:
libnative.DisposeString(pointer)
Original answer
With the actual DLL posted in the comments, I was able to resolve calling createMessage. It is actually the 13th function pointer in the nested structure, so the layout has to match. I didn’t define the full nested structure but just the compatible memory layout.
I figured it out by writing a C .exe (provided below) to see if the DLL worked and what the base address of the structure and the function address of createMessage were, and consulting the linked libnative_api.h.
test.c – C code to validate DLL is working and returning a string.
#include <windows.h>
#include <stdio.h>
#include "libnative_api.h"
typedef libnative_ExportedSymbols* (*PSYM)(void);
int main() {
HMODULE h = LoadLibraryW(L"libnative.dll");
if(h == NULL)
printf("load failedn");
PSYM plibnative_symbols = (PSYM)GetProcAddress(h,"libnative_symbols");
if(plibnative_symbols == NULL)
printf("get failedn");
libnative_ExportedSymbols* lib = plibnative_symbols();
// addresses of the structure and function
printf("%p %pn", lib, lib->kotlin.root.com.marcoeckstein.klib.createMessage);
const char* m = lib->kotlin.root.com.marcoeckstein.klib.createMessage();
printf("msg: %sn", m);
return 0;
}
Output – Note there is an extra string printed by the DLL.
00007FFAB4526340 00007FFAB44E0840
createMessage() was called.
msg: The int is not there.
test.py
import ctypes as ct
# binary-compatible layout.
# createMessage is the 13th function pointer in the nested layout.
# I suspect for Kotlin the DisposeString function should be called,
# so I wrote the return value of createString as an actual pointer
# so Python wouldn't automatically convert to a bytes object and
# lose the pointer value.
class Mylib(ct.Structure):
_fields_ = [("func", ct.CFUNCTYPE(None)), # void pointer placeholder
("DisposeString", ct.CFUNCTYPE(None, ct.POINTER(ct.c_char))),
("funcn", ct.CFUNCTYPE(None) * 10), # 10 more void pointer placeholders
("createMessage", ct.CFUNCTYPE(ct.POINTER(ct.c_char)))]
def to_bytes(p):
'''Return the null-terminated C string at address p as a bytes object.'''
return ct.c_char_p.from_buffer(p).value
def to_address(p):
'''Return the memory address of the ctypes pointer.'''
return ct.c_void_p.from_buffer(p).value
dll = ct.CDLL('./libnative')
dll.libnative_symbols.restype = ct.POINTER(Mylib)
libnative = dll.libnative_symbols()
print(f'{to_address(libnative):016X} {to_address(libnative.contents.createMessage):016X}')
p = libnative.contents.createMessage()
print(to_bytes(p))
libnative.contents.DisposeString(p)
Output – Note the address of the structure and function match the C .exe.
00007FFAB1BF6340 00007FFAB1BB0840
createMessage() was called.
b'The int is not there.'
I have a DLL (.dll for Windows, .so for Linux) and this autogenerated C header file for that DLL (excerpt):
Edit notes
tl;dr: I have updated my header excerpt and Python code because I had left out too much.
Details: I had left out a bunch of declarations in the header that I deemed unimportant but that turned out to be crucial (revealed by the accepted answer). Additionally, I had reduced struct nesting and renamed a struct to reduce clutter. (kotlin.root.com.marcoeckstein.klib was just myLib.) But since the accepted answer was created based on the actual DLL (as requested in the comments and shared outside of Stack Overflow), I have also revised my header excerpt and Python code to be consistent with that DLL and thus the accepted answer.
typedef struct {
// #region
// The details of most of these declarations are not important,
// but as it turned out, their memory requirements in the
// binary layout are:
void (*DisposeStablePointer)(libnative_KNativePtr ptr);
void (*DisposeString)(const char* string);
libnative_KBoolean (*IsInstance)(libnative_KNativePtr ref, const libnative_KType* type);
libnative_kref_kotlin_Byte (*createNullableByte)(libnative_KByte);
libnative_kref_kotlin_Short (*createNullableShort)(libnative_KShort);
libnative_kref_kotlin_Int (*createNullableInt)(libnative_KInt);
libnative_kref_kotlin_Long (*createNullableLong)(libnative_KLong);
libnative_kref_kotlin_Float (*createNullableFloat)(libnative_KFloat);
libnative_kref_kotlin_Double (*createNullableDouble)(libnative_KDouble);
libnative_kref_kotlin_Char (*createNullableChar)(libnative_KChar);
libnative_kref_kotlin_Boolean (*createNullableBoolean)(libnative_KBoolean);
libnative_kref_kotlin_Unit (*createNullableUnit)(void);
// #endregion
struct {
struct {
struct {
struct {
struct {
const char* (*createMessage)();
// More nested structs here
// (irrelevant for this question)
} klib;
} marcoeckstein;
} com;
} root;
} kotlin;
} libnative_ExportedSymbols;
extern libnative_ExportedSymbols* libnative_symbols(void);
I want to call the function createMessage from Python. I have tried this:
import ctypes
class Klib(ctypes.Structure):
_fields_ = [("createMessage", ctypes.CFUNCTYPE(ctypes.c_char_p))]
class Marcoeckstein(ctypes.Structure):
_fields_ = [("klib", Klib)]
class Com(ctypes.Structure):
_fields_ = [("marcoeckstein", Marcoeckstein)]
class Root(ctypes.Structure):
_fields_ = [("com", Com)]
class Kotlin(ctypes.Structure):
_fields_ = [("root", Root)]
class Libnative_ExportedSymbols(ctypes.Structure):
_fields_ = [("kotlin", Kotlin)]
dll = ctypes.CDLL("path/to/dll")
#region
#The following two lines turned out to be wrong. See below.
dll.libnative_symbols.restype = Libnative_ExportedSymbols
libnative = dll.libnative_symbols()
#endregion
libnative.kotlin.root.com.marcoeckstein.klib.createMessage()
On Windows I get:
OSError: exception: access violation writing 0x00007FFA51556340
On Linux I get:
Segmentation fault (core dumped)
On Windows, I have also tried WINFUNCTYPE instead of CFUNCTYPE, but I got the same error.
The wrapper generator ctypesgen generates the same type for createMessage as I did manually.
So what is going on here? How can I call a field of CFUNCTYPE?
Partial answer (edit)
As jasonharper noted in a comment, I needed to change this code…
dll.libnative_symbols.restype = Libnative_ExportedSymbols
libnative = dll.libnative_symbols()
… to this:
dll.libnative_symbols.restype = ctypes.POINTER(Libnative_ExportedSymbols)
libnative = dll.libnative_symbols().contents
This change turned out to be necessary but not sufficient. I got no error now, but createMessage was not properly called. It returned an empty instance of bytes, but the function had been implemented to always return a non-empty string.
Additional Notes (edit)
- There is a related issue.
- The DLL and header file were automatically created by Kotlin Multiplatform/Native, but this is irrelevant for the question.
tl;dr (edit by author of the question)
You cannot ignore the declarations before the struct kotlin. You do not necessarily need exactly fitting declarations for these in Python, but you need at least placeholders:
class Libnative_ExportedSymbols(ctypes.Structure):
_fields_ = [
("funcn", ctypes.CFUNCTYPE(None) * 12), # placeholders
("kotlin", Kotlin),
]
This is already enough to get the solution to work, but it results in a memory leak, because the string returned by createMessage uses unmanaged memory. Thus, we need the pointer to that memory, and we need to free it with the DisposeString function (a service funtion provided by the tool that built the DLL and header files).
class Klib(ctypes.Structure):
_fields_ = [("createMessage", ctypes.CFUNCTYPE(POINTER(ctypes.c_char)))]
// ...
class Libnative_ExportedSymbols(ctypes.Structure):
_fields_ = [
("func", ctypes.CFUNCTYPE(None)),
("DisposeString", ctypes.CFUNCTYPE(None, ctypes.POINTER(ctypes.c_char))),
("funcn", ctypes.CFUNCTYPE(None) * 10),
("kotlin", Kotlin),
]
Now createMessage() does not return the standard Python type bytes anymore, but ctypes.LP_c_char, which we have to convert ourselves:
def to_bytes(p):
return ctypes.c_char_p.from_buffer(p).value
pointer = libnative.kotlin.root.com.marcoeckstein.klib.createMessage()
message = toBytes(pointer) # pointer.contents is only the first char.
Finally, we free the memory:
libnative.DisposeString(pointer)
Original answer
With the actual DLL posted in the comments, I was able to resolve calling createMessage. It is actually the 13th function pointer in the nested structure, so the layout has to match. I didn’t define the full nested structure but just the compatible memory layout.
I figured it out by writing a C .exe (provided below) to see if the DLL worked and what the base address of the structure and the function address of createMessage were, and consulting the linked libnative_api.h.
test.c – C code to validate DLL is working and returning a string.
#include <windows.h>
#include <stdio.h>
#include "libnative_api.h"
typedef libnative_ExportedSymbols* (*PSYM)(void);
int main() {
HMODULE h = LoadLibraryW(L"libnative.dll");
if(h == NULL)
printf("load failedn");
PSYM plibnative_symbols = (PSYM)GetProcAddress(h,"libnative_symbols");
if(plibnative_symbols == NULL)
printf("get failedn");
libnative_ExportedSymbols* lib = plibnative_symbols();
// addresses of the structure and function
printf("%p %pn", lib, lib->kotlin.root.com.marcoeckstein.klib.createMessage);
const char* m = lib->kotlin.root.com.marcoeckstein.klib.createMessage();
printf("msg: %sn", m);
return 0;
}
Output – Note there is an extra string printed by the DLL.
00007FFAB4526340 00007FFAB44E0840
createMessage() was called.
msg: The int is not there.
test.py
import ctypes as ct
# binary-compatible layout.
# createMessage is the 13th function pointer in the nested layout.
# I suspect for Kotlin the DisposeString function should be called,
# so I wrote the return value of createString as an actual pointer
# so Python wouldn't automatically convert to a bytes object and
# lose the pointer value.
class Mylib(ct.Structure):
_fields_ = [("func", ct.CFUNCTYPE(None)), # void pointer placeholder
("DisposeString", ct.CFUNCTYPE(None, ct.POINTER(ct.c_char))),
("funcn", ct.CFUNCTYPE(None) * 10), # 10 more void pointer placeholders
("createMessage", ct.CFUNCTYPE(ct.POINTER(ct.c_char)))]
def to_bytes(p):
'''Return the null-terminated C string at address p as a bytes object.'''
return ct.c_char_p.from_buffer(p).value
def to_address(p):
'''Return the memory address of the ctypes pointer.'''
return ct.c_void_p.from_buffer(p).value
dll = ct.CDLL('./libnative')
dll.libnative_symbols.restype = ct.POINTER(Mylib)
libnative = dll.libnative_symbols()
print(f'{to_address(libnative):016X} {to_address(libnative.contents.createMessage):016X}')
p = libnative.contents.createMessage()
print(to_bytes(p))
libnative.contents.DisposeString(p)
Output – Note the address of the structure and function match the C .exe.
00007FFAB1BF6340 00007FFAB1BB0840
createMessage() was called.
b'The int is not there.'