Skip to content

Adding C-Python compatible API for embedding GraalPy #382

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lokmeinmatz wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Adding C-Python compatible API for embedding GraalPy #382

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bc756ed
WIP: c-embed python compatible lib
lokmeinmatz Jan 6, 2024
02484d5
Progress getting startup working until PyTruffle_InitBuiltinTypesAndS…
lokmeinmatz Feb 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Progress getting startup working until PyTruffle_InitBuiltinTypesAndS… 
…tructs
  • Loading branch information
@lokmeinmatz
lokmeinmatz committed Feb 6, 2024
commit 02484d52fc672d9bea693f9996145327bc13f4c8
7 changes: 7 additions & 0 deletions graalpython/com.oracle.graal.python.c_embed/src/graalpython-embed.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,13 @@ PyAPI_FUNC(void) Py_InitializeEx(int initsigs) {
fprintf(stderr, "graal_create_isolate succeeded\n");

graalpy_init_embed(graal_thread);

fprintf(stderr, "graalpy_init_embed succeeded, acquire GIL\n");
// accuire GIL
PyGILState_STATE gstate;
gstate = PyGILState_Ensure();

fprintf(stderr, "got GIL\n");
}

static void unimplemented(const char* name) {
Expand Down
1 change: 1 addition & 0 deletions graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -940,6 +940,7 @@ protected void initializeMultiThreading(PythonContext context) {

@Override
protected void initializeThread(PythonContext context, Thread thread) {
LOGGER.info("PythonLanguage.initializeThread");
context.attachThread(thread, threadState);
}

Expand Down
2 changes: 2 additions & 0 deletions ...aal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextCEvalBuiltins.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,9 @@ abstract static class PyEval_SaveThread extends CApiNullaryBuiltinNode {

@Specialization
static Object save(@Cached GilNode gil) {
// LOGGER.info("PyEval_SaveThread gil defined: " + (gil != null));
PythonContext context = PythonContext.get(gil);
LOGGER.info(() -> "PyEval_SaveThread context defined: " + (context != null));
Object threadState = PThreadState.getThreadState(PythonLanguage.get(gil), context);
LOGGER.fine("C extension releases GIL");
gil.release(context, true);
Expand Down
17 changes: 10 additions & 7 deletions ...acle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -565,7 +565,7 @@ public static CApiContext ensureCapiWasLoaded(Node node, PythonContext context,
} else {
useNative = false;
}
System.err.println("Loading CAPI from " + capiFile + " as " + (useNative ? "native" : "bitcode"));
LOGGER.info(() -> "Loading CAPI from " + capiFile + " as " + (useNative ? "native" : "bitcode"));
if (useNative) {
context.ensureNFILanguage(node, "NativeModules", "true");
capiSrcBuilder = Source.newBuilder(J_NFI_LANGUAGE, "load(RTLD_GLOBAL) \"" + capiFile.getPath() + "\"", "<libpython>");
Expand All @@ -576,29 +576,32 @@ public static CApiContext ensureCapiWasLoaded(Node node, PythonContext context,
if (!context.getLanguage().getEngineOption(PythonOptions.ExposeInternalSources)) {
capiSrcBuilder.internal(true);
}
LOGGER.config(() -> "loading CAPI from " + capiFile + " as " + (useNative ? "native" : "bitcode"));
LOGGER.info(() -> "loading CAPI from " + capiFile + " as " + (useNative ? "native" : "bitcode"));
CallTarget capiLibraryCallTarget = context.getEnv().parseInternal(capiSrcBuilder.build());

Object capiLibrary = capiLibraryCallTarget.call();
System.err.println("Checking init function");
LOGGER.info(() -> "Checking init function");
Object initFunction = U.readMember(capiLibrary, "initialize_graal_capi");
CApiContext cApiContext = new CApiContext(context, capiLibrary, useNative);
context.setCApiContext(cApiContext);
if (!U.isExecutable(initFunction)) {
Object signature = env.parseInternal(Source.newBuilder(J_NFI_LANGUAGE, "(ENV,(SINT32):POINTER):VOID", "exec").build()).call();
initFunction = SignatureLibrary.getUncached().bind(signature, initFunction);
System.err.println("Bound & calling init function");
LOGGER.info(() -> "Bound & calling init function");
U.execute(initFunction, new GetBuiltin());
} else {
System.err.println("Executing init function");
LOGGER.info(() -> "Executing init function");
U.execute(initFunction, NativePointer.createNull(), new GetBuiltin());
}
System.err.println("Checking builtins");
LOGGER.info(() -> "Checking builtins");
assert CApiCodeGen.assertBuiltins(capiLibrary);
LOGGER.info(() -> "initWrapper");
cApiContext.pyDateTimeCAPICapsule = PyDateTimeCAPIWrapper.initWrapper(context, cApiContext);
LOGGER.info(() -> "runCApiHooks");
context.runCApiHooks();

if (useNative) {
LOGGER.info(() -> "registering native finalizer");
/*
* C++ libraries sometimes declare global objects that have destructors that
* call Py_DECREF. Those destructors are then called during native shutdown,
Expand All @@ -618,7 +621,7 @@ public static CApiContext ensureCapiWasLoaded(Node node, PythonContext context,
LOGGER.warning(() -> "didn't register a native finalizer due to: " + e.getMessage());
}
}

LOGGER.info(() -> "loaded CAPI from " + capiFile + " as " + (useNative ? "native" : "bitcode") + " successfully");
return cApiContext;
} catch (PException e) {
/*
Expand Down
15 changes: 8 additions & 7 deletions ...e.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CEmbedBuiltins.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package com.oracle.graal.python.builtins.objects.cext.capi;

import com.oracle.graal.python.runtime.GilNode;
import com.oracle.graal.python.runtime.PythonContext;
import org.graalvm.nativeimage.IsolateThread;
import org.graalvm.nativeimage.c.function.CEntryPoint;
import org.graalvm.polyglot.Context;
Expand All @@ -13,15 +15,14 @@ public static void GraalPyInitEmbed(IsolateThread thread) {
System.out.println("GraalPyInitEmbed from thread " + thread.rawValue());
pyContext = Context.newBuilder("python")
.allowAllAccess(true)
//
//
//
// .option("python.PythonHome", "/Users/mkind/Dev/Uni/RE23/graal/sdk/mxbuild/darwin-aarch64/PYTHON_NATIVE_STANDALONE_SVM_JAVA21/graalpy-community-24.0.0-dev-macos-aarch64")
.option("python.PythonHome", "/Users/mkind/Dev/Uni/RE23/graal/sdk/mxbuild/darwin-aarch64/PYTHON_NATIVE_STANDALONE_SVM_JAVA21/graalpy-community-24.0.0-dev-macos-aarch64")
.build();
pyContext.initialize("python");
pyContext.enter();
pyContext.eval("python", "import struct; struct.pack('i', 42)");
//CApiContext ctx = CApiContext.ensureCapiWasLoaded();
// System.out.println("GraalPyInitEmbed from thread " + thread.rawValue() + " done, got ctx " + ctx.toString());
GilNode.getUncached().acquire();
CApiContext ctx = CApiContext.ensureCapiWasLoaded();
System.out.println("GraalPyInitEmbed from thread " + thread.rawValue() + " done, got ctx " + ctx.toString());
// get GIL

}
}
2 changes: 1 addition & 1 deletion mx.graalpython/mx_graalpython.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2274,7 +2274,7 @@ def as_license_regex(name):
jar_distributions=['graalpython:GRAALPYTHON-LAUNCHER', 'sdk:MAVEN_DOWNLOADER'],
main_class=GRAALPYTHON_MAIN_CLASS,
build_args=[
'-H:+DetectUserDirectoriesInImageHeap',
'-H:-DetectUserDirectoriesInImageHeap',
'-H:-CopyLanguageResources',
'-Dpolyglot.python.PosixModuleBackend=native',
'-Dpolyglot.python.Sha3ModuleBackend=native',
Expand Down
6 changes: 4 additions & 2 deletions mx.graalpython/suite.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -702,6 +702,7 @@
"bin/modules/_cpython_sre<graalpy_ext:native>",
"bin/modules/_cpython_unicodedata<graalpy_ext:native>",
"bin/modules/_cpython_struct<graalpy_ext:native>",
"bin/modules/_sha3<graalpy_ext:native>",
],
},
},
Expand All @@ -720,6 +721,7 @@
"bin/modules/_cpython_sre<graalpy_ext:native>",
"bin/modules/_cpython_unicodedata<graalpy_ext:native>",
"bin/modules/_cpython_struct<graalpy_ext:native>",
"bin/modules/_sha3<graalpy_ext:native>",
"bin/modules/_testcapi<graalpy_ext:native>",
"bin/modules/_testbuffer<graalpy_ext:native>",
"bin/modules/_testmultiphase<graalpy_ext:native>",
Expand Down Expand Up @@ -929,13 +931,13 @@
"GRAALPYTHON_VERSIONS_RES": {
"type": "dir",
"layout": {
"./graalpy_versions": ["string:<py_ver:binary><graal_ver:binary><dev_tag:none>"]
"./": "dependency:graalpy-versions/graalpy_versions",
},
},
"GRAALPYTHON_VERSIONS_MAIN": {
"type": "dir",
"layout": {
"./graalpy_versions": ["string:<py_ver:binary><graal_ver:binary><dev_tag:none>"],
"./": "dependency:graalpy-versions/graalpy_versions",
},
},

Expand Down