Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

bpo-42972: Fully implement GC protocol for sqlite3 heap types #26104

Merged
merged 14 commits into from May 25, 2021
+190 −72
Merged

bpo-42972: Fully implement GC protocol for sqlite3 heap types #26104

Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
6ece632
sqlite.Connection type now implements GC protocol
erlend-aasland Jan 20, 2021
8660609
sqlite.Cursor type now implements GC protocol
erlend-aasland Jan 20, 2021
13bc519
sqlite.Row type now implements GC protocol
erlend-aasland Jan 20, 2021
4519017
sqlite.PrepareProtocol type now implements GC protocol
erlend-aasland Jan 20, 2021
20da088
sqlite.Statement type now implements GC protocol
erlend-aasland Jan 20, 2021
ca870ac
sqlite3.Cache and sqlite3.Node types now implements GC protocol
erlend-aasland Jan 20, 2021
f5d9873
Remove unneeded self->db = NULL
erlend-aasland May 13, 2021
34928ed
Get rid of unneeded cast in cursor_dealloc
erlend-aasland May 13, 2021
590ff8e
Address review: visit node->key, node->data, and con->isolation_level
erlend-aasland May 23, 2021
b080ff7
Harden cache clear
erlend-aasland May 24, 2021
2bcfa29
Normalize connection_dealloc naming
erlend-aasland May 24, 2021
f8eb733
Address review: visit row PyObjects for consistency
erlend-aasland May 25, 2021
ab4c6e6
Normalize statement dealloc naming
erlend-aasland May 25, 2021
926787d
Clear sqlite3 stmt resources in stmt_clear()
erlend-aasland May 25, 2021
File filter
Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
83 Modules/_sqlite/cache.c
View file
Open in desktop
@@ -44,14 +44,29 @@ pysqlite_new_node(PyObject *key, PyObject *data)
return node;
}

static int
node_traverse(pysqlite_Node *self, visitproc visit, void *arg)
{
Py_VISIT(self->key);
Py_VISIT(self->data);
Py_VISIT(Py_TYPE(self));
This conversation was marked as resolved by erlend-aasland

This comment has been minimized.

Sign in to view
@shihai1991

shihai1991 May 23, 2021
Member

self->key, self->data doesn't be visited in here?

This comment has been minimized.

Sign in to view
@erlend-aasland

erlend-aasland May 23, 2021 •
edited
Author Contributor

self->key, self->data doesn't be visited in here?

Key is just a string, IIRC (I'll check when I'm back on my computer). Data is a statement object IIRC, so that should be visited. I'll fix it. Thanks!

UPDATE: key is a string, data is a pysqlite_Statement.
return 0;
}

static int
node_clear(pysqlite_Node *self)
{
Py_CLEAR(self->key);
Py_CLEAR(self->data);
return 0;
}

static void
pysqlite_node_dealloc(pysqlite_Node *self)
{
PyTypeObject *tp = Py_TYPE(self);

Py_DECREF(self->key);
Py_DECREF(self->data);

PyObject_GC_UnTrack(self);
tp->tp_clear((PyObject *)self);
tp->tp_free(self);
Py_DECREF(tp);
}
@@ -88,31 +103,51 @@ pysqlite_cache_init(pysqlite_Cache *self, PyObject *args, PyObject *kwargs)
return 0;
}

static void
pysqlite_cache_dealloc(pysqlite_Cache *self)
static int
cache_traverse(pysqlite_Cache *self, visitproc visit, void *arg)
{
PyTypeObject *tp = Py_TYPE(self);
pysqlite_Node* node;
pysqlite_Node* delete_node;

if (!self->factory) {
/* constructor failed, just get out of here */
return;
pysqlite_Node *node = self->first;
while (node) {
Py_VISIT(node);
node = node->next;
}
Py_VISIT(self->mapping);
if (self->decref_factory) {
Py_VISIT(self->factory);
}
Py_VISIT(Py_TYPE(self));
return 0;
}

static int
cache_clear(pysqlite_Cache *self)
{
/* iterate over all nodes and deallocate them */
node = self->first;
pysqlite_Node *node = self->first;
self->first = NULL;
while (node) {
delete_node = node;
pysqlite_Node *delete_node = node;
node = node->next;
Py_DECREF(delete_node);
Py_CLEAR(delete_node);
}

if (self->decref_factory) {
Py_DECREF(self->factory);
Py_CLEAR(self->factory);
}
Py_CLEAR(self->mapping);
return 0;
}

static void
pysqlite_cache_dealloc(pysqlite_Cache *self)
{
if (!self->factory) {
/* constructor failed, just get out of here */
return;
}
Py_DECREF(self->mapping);

PyObject_GC_UnTrack(self);
PyTypeObject *tp = Py_TYPE(self);
tp->tp_clear((PyObject *)self);
tp->tp_free(self);
Py_DECREF(tp);
}
@@ -260,14 +295,15 @@ pysqlite_cache_display(pysqlite_Cache *self, PyObject *args)

static PyType_Slot node_slots[] = {
{Py_tp_dealloc, pysqlite_node_dealloc},
{Py_tp_new, PyType_GenericNew},
{Py_tp_traverse, node_traverse},
{Py_tp_clear, node_clear},
{0, NULL},
};

static PyType_Spec node_spec = {
.name = MODULE_NAME ".Node",
.basicsize = sizeof(pysqlite_Node),
.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
.slots = node_slots,
};
PyTypeObject *pysqlite_NodeType = NULL;
@@ -283,15 +319,16 @@ static PyMethodDef cache_methods[] = {
static PyType_Slot cache_slots[] = {
{Py_tp_dealloc, pysqlite_cache_dealloc},
{Py_tp_methods, cache_methods},
{Py_tp_new, PyType_GenericNew},
{Py_tp_init, pysqlite_cache_init},
{Py_tp_traverse, cache_traverse},
{Py_tp_clear, cache_clear},
{0, NULL},
};

static PyType_Spec cache_spec = {
.name = MODULE_NAME ".Cache",
.basicsize = sizeof(pysqlite_Cache),
.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
.slots = cache_slots,
};
PyTypeObject *pysqlite_CacheType = NULL;
58 Modules/_sqlite/connection.c
View file
Open in desktop
@@ -225,28 +225,51 @@ pysqlite_do_all_statements(pysqlite_Connection *self, int action,
}
}

static int
connection_traverse(pysqlite_Connection *self, visitproc visit, void *arg)
{
Py_VISIT(self->statement_cache);
This conversation was marked as resolved by erlend-aasland

This comment has been minimized.

Sign in to view
@shihai1991

shihai1991 May 23, 2021
Member

self->isolation_level should be visited?

This comment has been minimized.

Sign in to view
@erlend-aasland

erlend-aasland May 23, 2021
Author Contributor

self->isolation_level should be visited?

I was under the impression that one does not need to visit str because a ref. cycle is impossible, no?

This comment has been minimized.

Sign in to view
@shihai1991

shihai1991 May 23, 2021
Member

OK. I have no experience in sqlite :(. But I believe that the tp_traverse have no side effect at least :)

This comment has been minimized.

Sign in to view
@erlend-aasland

erlend-aasland May 23, 2021 •
edited
Author Contributor

I've added it for consistency with the clear function :)

This comment has been minimized.

Sign in to view
@shihai1991

shihai1991 May 23, 2021
Member

Got it. Thanks.
Py_VISIT(self->isolation_level);
Py_VISIT(self->function_pinboard_trace_callback);
Py_VISIT(self->function_pinboard_progress_handler);
Py_VISIT(self->function_pinboard_authorizer_cb);
Py_VISIT(self->row_factory);
Py_VISIT(self->text_factory);
Py_VISIT(self->collations);
Py_VISIT(self->statements);
Py_VISIT(self->cursors);
Py_VISIT(Py_TYPE(self));
return 0;
}

static int
connection_clear(pysqlite_Connection *self)
{
Py_CLEAR(self->statement_cache);
Py_CLEAR(self->isolation_level);
Py_CLEAR(self->function_pinboard_trace_callback);
Py_CLEAR(self->function_pinboard_progress_handler);
Py_CLEAR(self->function_pinboard_authorizer_cb);
Py_CLEAR(self->row_factory);
Py_CLEAR(self->text_factory);
Py_CLEAR(self->collations);
Py_CLEAR(self->statements);
Py_CLEAR(self->cursors);
return 0;
}

static void
pysqlite_connection_dealloc(pysqlite_Connection *self)
connection_dealloc(pysqlite_Connection *self)
{
PyTypeObject *tp = Py_TYPE(self);

Py_XDECREF(self->statement_cache);
PyObject_GC_UnTrack(self);
tp->tp_clear((PyObject *)self);

/* Clean up if user has not called .close() explicitly. */
if (self->db) {
sqlite3_close_v2(self->db);
}

Py_XDECREF(self->isolation_level);
Py_XDECREF(self->function_pinboard_trace_callback);
Py_XDECREF(self->function_pinboard_progress_handler);
Py_XDECREF(self->function_pinboard_authorizer_cb);
Py_XDECREF(self->row_factory);
Py_XDECREF(self->text_factory);
Py_XDECREF(self->collations);
Py_XDECREF(self->statements);
Py_XDECREF(self->cursors);

tp->tp_free(self);
Py_DECREF(tp);
}
@@ -1328,7 +1351,7 @@ pysqlite_connection_call(pysqlite_Connection *self, PyObject *args,

_pysqlite_drop_unused_statement_references(self);

statement = PyObject_New(pysqlite_Statement, pysqlite_StatementType);
statement = PyObject_GC_New(pysqlite_Statement, pysqlite_StatementType);
if (!statement) {
return NULL;
}
@@ -1909,21 +1932,22 @@ static struct PyMemberDef connection_members[] =
};

static PyType_Slot connection_slots[] = {
{Py_tp_dealloc, pysqlite_connection_dealloc},
{Py_tp_dealloc, connection_dealloc},
{Py_tp_doc, (void *)connection_doc},
{Py_tp_methods, connection_methods},
{Py_tp_members, connection_members},
{Py_tp_getset, connection_getset},
{Py_tp_new, PyType_GenericNew},
{Py_tp_init, pysqlite_connection_init},
{Py_tp_call, pysqlite_connection_call},
{Py_tp_traverse, connection_traverse},
{Py_tp_clear, connection_clear},
{0, NULL},
};

static PyType_Spec connection_spec = {
.name = MODULE_NAME ".Connection",
.basicsize = sizeof(pysqlite_Connection),
.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
.slots = connection_slots,
};

46 Modules/_sqlite/cursor.c
View file
Open in desktop
@@ -81,27 +81,46 @@ pysqlite_cursor_init_impl(pysqlite_Cursor *self,
return 0;
}

static void pysqlite_cursor_dealloc(pysqlite_Cursor* self)
static int
cursor_traverse(pysqlite_Cursor *self, visitproc visit, void *arg)
{
PyTypeObject *tp = Py_TYPE(self);
Py_VISIT(self->connection);
Py_VISIT(self->row_cast_map);
Py_VISIT(self->row_factory);
Py_VISIT(self->next_row);
Py_VISIT(Py_TYPE(self));
return 0;
}

static int
cursor_clear(pysqlite_Cursor *self)
{
/* Reset the statement if the user has not closed the cursor */
if (self->statement) {
pysqlite_statement_reset(self->statement);
Py_DECREF(self->statement);
Py_CLEAR(self->statement);
}

Py_XDECREF(self->connection);
Py_XDECREF(self->row_cast_map);
Py_XDECREF(self->description);
Py_XDECREF(self->lastrowid);
Py_XDECREF(self->row_factory);
Py_XDECREF(self->next_row);
Py_CLEAR(self->connection);
Py_CLEAR(self->row_cast_map);
Py_CLEAR(self->description);
Py_CLEAR(self->lastrowid);
Py_CLEAR(self->row_factory);
Py_CLEAR(self->next_row);

if (self->in_weakreflist != NULL) {
PyObject_ClearWeakRefs((PyObject*)self);
}

return 0;
}

static void
cursor_dealloc(PyObject *self)
{
PyTypeObject *tp = Py_TYPE(self);
PyObject_GC_UnTrack(self);
tp->tp_clear(self);
tp->tp_free(self);
Py_DECREF(tp);
}
@@ -487,7 +506,7 @@ _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* operation

if (self->statement->in_use) {
Py_SETREF(self->statement,
PyObject_New(pysqlite_Statement, pysqlite_StatementType));
PyObject_GC_New(pysqlite_Statement, pysqlite_StatementType));
if (!self->statement) {
goto error;
}
@@ -1006,21 +1025,22 @@ static const char cursor_doc[] =
PyDoc_STR("SQLite database cursor class.");

static PyType_Slot cursor_slots[] = {
{Py_tp_dealloc, pysqlite_cursor_dealloc},
{Py_tp_dealloc, cursor_dealloc},
{Py_tp_doc, (void *)cursor_doc},
{Py_tp_iter, PyObject_SelfIter},
{Py_tp_iternext, pysqlite_cursor_iternext},
{Py_tp_methods, cursor_methods},
{Py_tp_members, cursor_members},
{Py_tp_new, PyType_GenericNew},
{Py_tp_init, pysqlite_cursor_init},
{Py_tp_traverse, cursor_traverse},
{Py_tp_clear, cursor_clear},
{0, NULL},
};

static PyType_Spec cursor_spec = {
.name = MODULE_NAME ".Cursor",
.basicsize = sizeof(pysqlite_Cursor),
.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
.slots = cursor_slots,
};

13 Modules/_sqlite/prepare_protocol.c
View file
Open in desktop
@@ -30,26 +30,33 @@ pysqlite_prepare_protocol_init(pysqlite_PrepareProtocol *self, PyObject *args,
return 0;
}

static int
pysqlite_prepare_protocol_traverse(PyObject *self, visitproc visit, void *arg)
This conversation was marked as resolved by erlend-aasland

This comment has been minimized.

Sign in to view
@pablogsal

pablogsal May 24, 2021
Member

If this object cannot hold any Python objects, why is implementing the GC interface?

This comment has been minimized.

Sign in to view
@erlend-aasland

erlend-aasland May 25, 2021
Author Contributor

Because we need to visit the type.

This comment has been minimized.

Sign in to view
@erlend-aasland

erlend-aasland May 25, 2021
Author Contributor

Quoting Victor:

A heap type must respect the following 3 conditions to collaborate with the GC:

    Have the Py_TPFLAGS_HAVE_GC flag;
    Define a traverse function (tp_traverse) which visits the type: Py_VISIT(Py_TYPE(self));
    Instances must be tracked by the GC.
{
Py_VISIT(Py_TYPE(self));
return 0;
}

static void
pysqlite_prepare_protocol_dealloc(pysqlite_PrepareProtocol *self)
{
PyTypeObject *tp = Py_TYPE(self);

PyObject_GC_UnTrack(self);
tp->tp_free(self);
Py_DECREF(tp);
}

static PyType_Slot type_slots[] = {
{Py_tp_dealloc, pysqlite_prepare_protocol_dealloc},
{Py_tp_new, PyType_GenericNew},
{Py_tp_init, pysqlite_prepare_protocol_init},
{Py_tp_traverse, pysqlite_prepare_protocol_traverse},
{0, NULL},
};

static PyType_Spec type_spec = {
.name = MODULE_NAME ".PrepareProtocol",
.basicsize = sizeof(pysqlite_PrepareProtocol),
.flags = Py_TPFLAGS_DEFAULT,
.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,
.slots = type_slots,
};