Clean up dead weakrefs to sqlite3 Blob objects

Author: takluyver

Modules/_sqlite/connection.c

@@ -38,6 +38,7 @@
 #include "pycore_pyerrors.h"      // _PyErr_ChainExceptions1()
 #include "pycore_pylifecycle.h"   // _Py_IsInterpreterFinalizing()
 #include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL
+#include "pycore_weakref.h"
 
 #include <stdbool.h>
 
@@ -148,6 +149,7 @@ class _sqlite3.Connection "pysqlite_Connection *" "clinic_state()->ConnectionTyp
 [clinic start generated code]*/
 /*[clinic end generated code: output=da39a3ee5e6b4b0d input=67369db2faf80891]*/
 
+static int _pysqlite_drop_unused_blob_references(pysqlite_Connection* self);
 static void incref_callback_context(callback_context *ctx);
 static void decref_callback_context(callback_context *ctx);
 static void set_callback_context(callback_context **ctx_pp,
@@ -305,6 +307,7 @@ pysqlite_connection_init_impl(pysqlite_Connection *self, PyObject *database,
     self->thread_ident = PyThread_get_thread_ident();
     self->statement_cache = statement_cache;
     self->blobs = blobs;
+    self->created_blobs = 0;
     self->row_factory = Py_NewRef(Py_None);
     self->text_factory = Py_NewRef(&PyUnicode_Type);
     self->trace_ctx = NULL;
@@ -670,6 +673,10 @@ blobopen_impl(pysqlite_Connection *self, const char *table, const char *col,
         goto error;
     }
 
+    if (_pysqlite_drop_unused_blob_references(self) < 0) {
+        goto error;
+    }
+
     return (PyObject *)obj;
 
 error:
@@ -1095,6 +1102,38 @@ final_callback(sqlite3_context *context)
     PyGILState_Release(threadstate);
 }
 
+static int
+_pysqlite_drop_unused_blob_references(pysqlite_Connection* self)
+{
+    /* we only need to do this once in a while */
+    if (self->created_blobs++ < 200) {
+        return 0;
+    }
+
+    self->created_blobs = 0;
+
+    PyObject* new_list = PyList_New(0);
+    if (!new_list) {
+        return -1;
+    }
+
+    assert(PyList_CheckExact(self->blobs));
+    Py_ssize_t imax = PyList_GET_SIZE(self->blobs);
+    for (Py_ssize_t i = 0; i < imax; i++) {
+        PyObject* weakref = PyList_GET_ITEM(self->blobs, i);
+        if (_PyWeakref_IsDead(weakref)) {
+          continue;
+        }
+        if (PyList_Append(new_list, weakref) != 0) {
+            Py_DECREF(new_list);
+            return -1;
+        }
+    }
+
+    Py_SETREF(self->blobs, new_list);
+    return 0;
+}
+
 /* Allocate a UDF/callback context structure. In order to ensure that the state
  * pointer always outlives the callback context, we make sure it owns a
  * reference to the module itself. create_callback_context() is always called

Modules/_sqlite/connection.h

@@ -73,6 +73,10 @@ typedef struct
     /* Lists of weak references to blobs used within this connection */
     PyObject *blobs;
 
+    /* Counter for how many blobs were opened in this connection;
+     * May be reset to 0 at certain intervals. */
+    int created_blobs;
+
     PyObject* row_factory;
 
     /* Determines how bytestrings from SQLite are converted to Python objects: