diff --git a/base_cron_reconnect/README.rst b/base_cron_reconnect/README.rst new file mode 100644 index 00000000000..69b774f65f7 --- /dev/null +++ b/base_cron_reconnect/README.rst @@ -0,0 +1,216 @@ +.. image:: https://odoo-community.org/readme-banner-image + :target: https://odoo-community.org/get-involved?utm_source=readme + :alt: Odoo Community Association + +=================== +Base Cron Reconnect +=================== + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:d59c1769b3cc278a3c0e929b950a6e69ecd782192e76dca30e754b773750d172 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png + :target: https://odoo-community.org/page/development-status + :alt: Beta +.. |badge2| image:: https://img.shields.io/badge/license-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fserver--tools-lightgray.png?logo=github + :target: https://github.com/OCA/server-tools/tree/16.0/base_cron_reconnect + :alt: OCA/server-tools +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/server-tools-16-0/server-tools-16-0-base_cron_reconnect + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/server-tools&target_branch=16.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +When Odoo runs with ``workers = 0`` (threaded mode), scheduled actions are +executed by a small number of long-lived ``cron`` threads spawned once at +server startup (``ThreadedServer.cron_spawn()``). + +If the database connection those threads hold is lost - for example +because PostgreSQL restarts or briefly drops connections during +maintenance or a failover - the affected thread crashes with an +unhandled ``psycopg2.OperationalError`` and is never respawned. Every +scheduled action on that Odoo process then silently stops running, +forever, until someone notices and restarts the whole server. HTTP +requests are unaffected (they borrow a fresh connection per request), so +nothing in the application itself signals that anything is wrong. + +This module patches ``ThreadedServer.cron_thread`` at server startup +(``post_load``) so that a crashed cron thread is logged and retried +after a short delay instead of staying dead, mirroring the retry +pattern Odoo's own ``bus`` module already uses for its long-lived +``LISTEN``/``NOTIFY`` connection (``bus.models.bus.ImDispatch.run()``). +It also logs its own start/stop at ``INFO`` - Odoo's native cron only +logs those at ``DEBUG`` or not at all. See ``USAGE.rst`` for the exact +log lines and recommended alerting. + +This only applies to threaded mode. In prefork mode (``workers`` > 0), +Odoo's own ``PreforkServer`` already respawns a crashed ``WorkerCron`` +process on its own, so this module has no effect there. + +Related upstream reports - not overlooked, but core has taken a +position that automatic recovery isn't in scope for +``ThreadedServer``: + +* https://github.com/odoo/odoo/issues/15666 (2017, fixed for the + pre-``LISTEN``/``NOTIFY`` code of the time; the current call site is + unguarded again) +* https://github.com/odoo/odoo/issues/88984 - closed **Won't fix**, + with an explicit rationale that DB availability is a system + administrator's responsibility in threaded mode +* https://github.com/odoo/odoo/issues/184421 (open) +* https://github.com/odoo/odoo/issues/215164 (open) + +**Table of contents** + +.. contents:: + :local: + +Installation +============ + +You don't need to install this module in the database(s) to enable it. + +But you need to load it server-wide: + +* By starting Odoo with ``--load=base,web,base_cron_reconnect`` + +* Or by updating its configuration file: + +.. code-block:: ini + + [options] + (...) + server_wide_modules = base,web,base_cron_reconnect + +This is required, not optional: the patch has to be in place before +``cron_spawn()`` runs, and ``load_server_wide_modules()`` is the first +statement of ``odoo.service.server.start()`` - guaranteed to run before +any database registry is loaded. A database-installed addon has no such +guarantee (its Python may not even be imported until the first request, +notably with a dynamic ``dbfilter`` setup and no fixed ``--database``), +i.e. potentially after the cron threads already resolved the unpatched +method. Installing this module from the Apps list is harmless (it has +no models or data) but does **not**, by itself, activate anything. + +To verify the patch is active, check the server log at startup for:: + + base_cron_reconnect: patched ThreadedServer.cron_thread to survive a + lost database connection (retry interval: 60s). + +If that line is missing, look for ``Failed to load server-wide module`` +instead - ``load_server_wide_modules()`` silently swallows exceptions +from a module's ``post_load`` hook and boots unpatched. + +Configuration +============= + +The retry interval (in seconds) can optionally be configured. It defaults +to ``60`` (matching Odoo's own cron poll interval) if left unset or if an +invalid value is given. + +Via the configuration file: + +.. code-block:: ini + + [options] + cron_reconnect_retry_interval = 30 + +Or via an environment variable, checked if the option above is not set: + +.. code-block:: shell + + ODOO_CRON_RECONNECT_RETRY_INTERVAL=30 + +An invalid value (non-numeric, or below ``1``) falls back to the default +and logs a warning at startup rather than failing to boot. + +Usage +===== + +There is nothing to click: once loaded server-wide, the module works +silently in the background. + +Odoo's own cron only logs its start at DEBUG (invisible by default) and +nothing at all on a graceful stop. This module logs the lifecycle at +INFO instead, matching the style of ``queue_job``'s jobrunner +("starting"/"stopped"):: + + INFO ... base_cron_reconnect: patched ThreadedServer.cron_thread to + survive a lost database connection (retry interval: 60s). + INFO ... cron0 starting + +What a recovery looks like in the log:: + + WARNING ... cron0 died, most likely due to a lost database connection; + restarting it in 60s. + Traceback (most recent call last): + ... + INFO ... cron0 restarting now. + +And on a clean server shutdown while a retry is pending:: + + INFO ... cron0 stopped gracefully + +Recommended: alert on ``WARNING``-level records from the +``odoo.addons.base_cron_reconnect.hooks`` logger - the point of this +module is as much observability as recovery. A cron thread crashing at +all usually means something worth investigating (a database restart, a +network issue, connection pool exhaustion); this module's own record is +deliberately a ``WARNING``, not an ``ERROR``, since the underlying +disconnect itself is already logged (e.g. by ``odoo.sql_db``) and this +one just confirms the thread is recovering on its own. + +No ``WARNING`` is logged on a normal, clean server restart - the module +tracks the shutdown signal and stays quiet in that case, so the alert +stays low-noise. + +To verify on a test instance: restart the PostgreSQL server (or its +container) while Odoo is running in threaded mode with this module +loaded, and watch the log for the ``WARNING`` above followed by +``cron%d restarting now.`` and normal cron polling resuming. + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Contributors +~~~~~~~~~~~~ + +* Odoo Community Association (OCA) + +Maintainers +~~~~~~~~~~~ + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +This module is part of the `OCA/server-tools `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/base_cron_reconnect/__init__.py b/base_cron_reconnect/__init__.py new file mode 100644 index 00000000000..e3f776c6f81 --- /dev/null +++ b/base_cron_reconnect/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2026 Odoo Community Association (OCA) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +from .hooks import post_load diff --git a/base_cron_reconnect/__manifest__.py b/base_cron_reconnect/__manifest__.py new file mode 100644 index 00000000000..b31e1183257 --- /dev/null +++ b/base_cron_reconnect/__manifest__.py @@ -0,0 +1,17 @@ +# Copyright 2026 Odoo Community Association (OCA) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +{ + "name": "Base Cron Reconnect", + "summary": "Restart threaded-mode cron workers that died on a lost " + "database connection", + "version": "16.0.1.0.0", + "category": "Tools", + "author": "Odoo Community Association (OCA)", + "maintainers": [], + "development_status": "Beta", + "website": "https://github.com/OCA/server-tools", + "license": "AGPL-3", + "depends": ["base"], + "post_load": "post_load", + "installable": True, +} diff --git a/base_cron_reconnect/hooks.py b/base_cron_reconnect/hooks.py new file mode 100644 index 00000000000..674beede051 --- /dev/null +++ b/base_cron_reconnect/hooks.py @@ -0,0 +1,115 @@ +# Copyright 2026 Odoo Community Association (OCA) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +"""Restart threaded-mode (``workers = 0``) cron workers that die on a lost +database connection, instead of leaving them dead until the next restart. + +``ThreadedServer.cron_thread`` has no exception handling around its +polling loop - unlike ``ir_cron._process_jobs()`` (job-level failures) +or ``PreforkServer`` (which respawns a crashed ``WorkerCron`` process). +This wraps ``cron_thread`` in the same catch-log-sleep-retry pattern +Odoo's own ``bus.ImDispatch.run()`` already uses for an equivalent +long-lived connection. No-op under prefork mode. + +Full writeup: ``readme/DESCRIPTION.rst``. Related upstream reports - +core has taken a position that automatic recovery isn't in scope for +``ThreadedServer`` (see #88984): odoo/odoo#15666, #88984, #184421, +#215164. +""" +import logging +import os +import threading + +from odoo.service import server +from odoo.tools import config + +_logger = logging.getLogger(__name__) + +#: Fallback retry interval (seconds) if neither the ``cron_reconnect_retry_interval`` +#: ini option nor the ``ODOO_CRON_RECONNECT_RETRY_INTERVAL`` environment variable is +#: set. Matches ``odoo.service.server.SLEEP_INTERVAL``. +DEFAULT_RETRY_INTERVAL = 60 +CONFIG_KEY = "cron_reconnect_retry_interval" +ENV_VAR = "ODOO_CRON_RECONNECT_RETRY_INTERVAL" + +_PATCH_MARKER = "_cron_reconnect_patched" + +#: Set once the server starts shutting down, so the wrapper can tell a +#: shutdown-induced crash (expected, quiet) from a real one (logged and +#: retried). +_stop_event = threading.Event() + + +def _get_retry_interval(): + """Return the configured retry interval, defaulting defensively.""" + raw = config.get(CONFIG_KEY) or os.environ.get(ENV_VAR) + if raw is None: + return DEFAULT_RETRY_INTERVAL + try: + value = int(raw) + except (TypeError, ValueError): + _logger.warning( + "Invalid %s/%s value %r, falling back to %ss.", + CONFIG_KEY, + ENV_VAR, + raw, + DEFAULT_RETRY_INTERVAL, + ) + return DEFAULT_RETRY_INTERVAL + return max(1, value) + + +def post_load(): + """Patch ``ThreadedServer.cron_thread`` for auto-reconnect resilience. + + Safe to call more than once (idempotent) and safe to run under prefork + mode (no-op there, since ``ThreadedServer`` is never used). + """ + if getattr(server.ThreadedServer.cron_thread, _PATCH_MARKER, False): + return + + original_cron_thread = server.ThreadedServer.cron_thread + retry_interval = _get_retry_interval() + + # Make the wrapper below stop retrying as soon as the server starts a + # clean shutdown, instead of logging a false-alarm WARNING on every + # restart (``ThreadedServer.stop()`` closes all database connections + # right after running the on-stop hooks, which reliably makes the + # underlying cron_thread call raise). + server.CommonServer.on_stop(_stop_event.set) + + def cron_thread(self, number): + # original_cron_thread() never returns under normal operation (its + # own inner loop is `while True:`, with no break/return) - it can + # only ever exit by raising. So the only case to handle here is an + # exception; a clean return is not a real possibility today. + # + # Odoo logs cron start/poll only at DEBUG (invisible by default) + # and nothing at all on stop. Log the lifecycle at INFO instead, + # matching queue_job's jobrunner ("starting"/"graceful stop + # requested"/"stopped"). + _logger.info("cron%d starting", number) + while not _stop_event.is_set(): + try: + original_cron_thread(self, number) + except Exception: + if _stop_event.is_set(): + break + _logger.warning( + "cron%d died, most likely due to a lost database " + "connection; restarting it in %ss.", + number, + retry_interval, + exc_info=True, + ) + _stop_event.wait(retry_interval) + if not _stop_event.is_set(): + _logger.info("cron%d restarting now.", number) + _logger.info("cron%d stopped gracefully", number) + + setattr(cron_thread, _PATCH_MARKER, True) + server.ThreadedServer.cron_thread = cron_thread + _logger.info( + "base_cron_reconnect: patched ThreadedServer.cron_thread to survive " + "a lost database connection (retry interval: %ss).", + retry_interval, + ) diff --git a/base_cron_reconnect/readme/CONFIGURE.rst b/base_cron_reconnect/readme/CONFIGURE.rst new file mode 100644 index 00000000000..c6c86de10ee --- /dev/null +++ b/base_cron_reconnect/readme/CONFIGURE.rst @@ -0,0 +1,19 @@ +The retry interval (in seconds) can optionally be configured. It defaults +to ``60`` (matching Odoo's own cron poll interval) if left unset or if an +invalid value is given. + +Via the configuration file: + +.. code-block:: ini + + [options] + cron_reconnect_retry_interval = 30 + +Or via an environment variable, checked if the option above is not set: + +.. code-block:: shell + + ODOO_CRON_RECONNECT_RETRY_INTERVAL=30 + +An invalid value (non-numeric, or below ``1``) falls back to the default +and logs a warning at startup rather than failing to boot. diff --git a/base_cron_reconnect/readme/CONTRIBUTORS.rst b/base_cron_reconnect/readme/CONTRIBUTORS.rst new file mode 100644 index 00000000000..60e3b42b4fd --- /dev/null +++ b/base_cron_reconnect/readme/CONTRIBUTORS.rst @@ -0,0 +1 @@ +* Odoo Community Association (OCA) diff --git a/base_cron_reconnect/readme/DESCRIPTION.rst b/base_cron_reconnect/readme/DESCRIPTION.rst new file mode 100644 index 00000000000..6e846c9481a --- /dev/null +++ b/base_cron_reconnect/readme/DESCRIPTION.rst @@ -0,0 +1,38 @@ +When Odoo runs with ``workers = 0`` (threaded mode), scheduled actions are +executed by a small number of long-lived ``cron`` threads spawned once at +server startup (``ThreadedServer.cron_spawn()``). + +If the database connection those threads hold is lost - for example +because PostgreSQL restarts or briefly drops connections during +maintenance or a failover - the affected thread crashes with an +unhandled ``psycopg2.OperationalError`` and is never respawned. Every +scheduled action on that Odoo process then silently stops running, +forever, until someone notices and restarts the whole server. HTTP +requests are unaffected (they borrow a fresh connection per request), so +nothing in the application itself signals that anything is wrong. + +This module patches ``ThreadedServer.cron_thread`` at server startup +(``post_load``) so that a crashed cron thread is logged and retried +after a short delay instead of staying dead, mirroring the retry +pattern Odoo's own ``bus`` module already uses for its long-lived +``LISTEN``/``NOTIFY`` connection (``bus.models.bus.ImDispatch.run()``). +It also logs its own start/stop at ``INFO`` - Odoo's native cron only +logs those at ``DEBUG`` or not at all. See ``USAGE.rst`` for the exact +log lines and recommended alerting. + +This only applies to threaded mode. In prefork mode (``workers`` > 0), +Odoo's own ``PreforkServer`` already respawns a crashed ``WorkerCron`` +process on its own, so this module has no effect there. + +Related upstream reports - not overlooked, but core has taken a +position that automatic recovery isn't in scope for +``ThreadedServer``: + +* https://github.com/odoo/odoo/issues/15666 (2017, fixed for the + pre-``LISTEN``/``NOTIFY`` code of the time; the current call site is + unguarded again) +* https://github.com/odoo/odoo/issues/88984 - closed **Won't fix**, + with an explicit rationale that DB availability is a system + administrator's responsibility in threaded mode +* https://github.com/odoo/odoo/issues/184421 (open) +* https://github.com/odoo/odoo/issues/215164 (open) diff --git a/base_cron_reconnect/readme/INSTALL.rst b/base_cron_reconnect/readme/INSTALL.rst new file mode 100644 index 00000000000..a8835fbb86a --- /dev/null +++ b/base_cron_reconnect/readme/INSTALL.rst @@ -0,0 +1,32 @@ +You don't need to install this module in the database(s) to enable it. + +But you need to load it server-wide: + +* By starting Odoo with ``--load=base,web,base_cron_reconnect`` + +* Or by updating its configuration file: + +.. code-block:: ini + + [options] + (...) + server_wide_modules = base,web,base_cron_reconnect + +This is required, not optional: the patch has to be in place before +``cron_spawn()`` runs, and ``load_server_wide_modules()`` is the first +statement of ``odoo.service.server.start()`` - guaranteed to run before +any database registry is loaded. A database-installed addon has no such +guarantee (its Python may not even be imported until the first request, +notably with a dynamic ``dbfilter`` setup and no fixed ``--database``), +i.e. potentially after the cron threads already resolved the unpatched +method. Installing this module from the Apps list is harmless (it has +no models or data) but does **not**, by itself, activate anything. + +To verify the patch is active, check the server log at startup for:: + + base_cron_reconnect: patched ThreadedServer.cron_thread to survive a + lost database connection (retry interval: 60s). + +If that line is missing, look for ``Failed to load server-wide module`` +instead - ``load_server_wide_modules()`` silently swallows exceptions +from a module's ``post_load`` hook and boots unpatched. diff --git a/base_cron_reconnect/readme/USAGE.rst b/base_cron_reconnect/readme/USAGE.rst new file mode 100644 index 00000000000..ceacb357ad2 --- /dev/null +++ b/base_cron_reconnect/readme/USAGE.rst @@ -0,0 +1,41 @@ +There is nothing to click: once loaded server-wide, the module works +silently in the background. + +Odoo's own cron only logs its start at DEBUG (invisible by default) and +nothing at all on a graceful stop. This module logs the lifecycle at +INFO instead, matching the style of ``queue_job``'s jobrunner +("starting"/"stopped"):: + + INFO ... base_cron_reconnect: patched ThreadedServer.cron_thread to + survive a lost database connection (retry interval: 60s). + INFO ... cron0 starting + +What a recovery looks like in the log:: + + WARNING ... cron0 died, most likely due to a lost database connection; + restarting it in 60s. + Traceback (most recent call last): + ... + INFO ... cron0 restarting now. + +And on a clean server shutdown while a retry is pending:: + + INFO ... cron0 stopped gracefully + +Recommended: alert on ``WARNING``-level records from the +``odoo.addons.base_cron_reconnect.hooks`` logger - the point of this +module is as much observability as recovery. A cron thread crashing at +all usually means something worth investigating (a database restart, a +network issue, connection pool exhaustion); this module's own record is +deliberately a ``WARNING``, not an ``ERROR``, since the underlying +disconnect itself is already logged (e.g. by ``odoo.sql_db``) and this +one just confirms the thread is recovering on its own. + +No ``WARNING`` is logged on a normal, clean server restart - the module +tracks the shutdown signal and stays quiet in that case, so the alert +stays low-noise. + +To verify on a test instance: restart the PostgreSQL server (or its +container) while Odoo is running in threaded mode with this module +loaded, and watch the log for the ``WARNING`` above followed by +``cron%d restarting now.`` and normal cron polling resuming. diff --git a/base_cron_reconnect/static/description/icon.png b/base_cron_reconnect/static/description/icon.png new file mode 100644 index 00000000000..3a0328b516c Binary files /dev/null and b/base_cron_reconnect/static/description/icon.png differ diff --git a/base_cron_reconnect/static/description/index.html b/base_cron_reconnect/static/description/index.html new file mode 100644 index 00000000000..42c63211e83 --- /dev/null +++ b/base_cron_reconnect/static/description/index.html @@ -0,0 +1,548 @@ + + + + + +README.rst + + + +
+ + + +Odoo Community Association + +
+

Base Cron Reconnect

+ +

Beta License: AGPL-3 OCA/server-tools Translate me on Weblate Try me on Runboat

+

When Odoo runs with workers = 0 (threaded mode), scheduled actions are +executed by a small number of long-lived cron threads spawned once at +server startup (ThreadedServer.cron_spawn()).

+

If the database connection those threads hold is lost - for example +because PostgreSQL restarts or briefly drops connections during +maintenance or a failover - the affected thread crashes with an +unhandled psycopg2.OperationalError and is never respawned. Every +scheduled action on that Odoo process then silently stops running, +forever, until someone notices and restarts the whole server. HTTP +requests are unaffected (they borrow a fresh connection per request), so +nothing in the application itself signals that anything is wrong.

+

This module patches ThreadedServer.cron_thread at server startup +(post_load) so that a crashed cron thread is logged and retried +after a short delay instead of staying dead, mirroring the retry +pattern Odoo’s own bus module already uses for its long-lived +LISTEN/NOTIFY connection (bus.models.bus.ImDispatch.run()). +It also logs its own start/stop at INFO - Odoo’s native cron only +logs those at DEBUG or not at all. See USAGE.rst for the exact +log lines and recommended alerting.

+

This only applies to threaded mode. In prefork mode (workers > 0), +Odoo’s own PreforkServer already respawns a crashed WorkerCron +process on its own, so this module has no effect there.

+

Related upstream reports - not overlooked, but core has taken a +position that automatic recovery isn’t in scope for +ThreadedServer:

+ +

Table of contents

+ +
+

Installation

+

You don’t need to install this module in the database(s) to enable it.

+

But you need to load it server-wide:

+
    +
  • By starting Odoo with --load=base,web,base_cron_reconnect
    • +
  • Or by updating its configuration file:
    • +
+
+[options]
+(...)
+server_wide_modules = base,web,base_cron_reconnect
+
+

This is required, not optional: the patch has to be in place before +cron_spawn() runs, and load_server_wide_modules() is the first +statement of odoo.service.server.start() - guaranteed to run before +any database registry is loaded. A database-installed addon has no such +guarantee (its Python may not even be imported until the first request, +notably with a dynamic dbfilter setup and no fixed --database), +i.e. potentially after the cron threads already resolved the unpatched +method. Installing this module from the Apps list is harmless (it has +no models or data) but does not, by itself, activate anything.

+

To verify the patch is active, check the server log at startup for:

+
+base_cron_reconnect: patched ThreadedServer.cron_thread to survive a
+lost database connection (retry interval: 60s).
+
+

If that line is missing, look for Failed to load server-wide module +instead - load_server_wide_modules() silently swallows exceptions +from a module’s post_load hook and boots unpatched.

+
+
+

Configuration

+

The retry interval (in seconds) can optionally be configured. It defaults +to 60 (matching Odoo’s own cron poll interval) if left unset or if an +invalid value is given.

+

Via the configuration file:

+
+[options]
+cron_reconnect_retry_interval = 30
+
+

Or via an environment variable, checked if the option above is not set:

+
+ODOO_CRON_RECONNECT_RETRY_INTERVAL=30
+
+

An invalid value (non-numeric, or below 1) falls back to the default +and logs a warning at startup rather than failing to boot.

+
+
+

Usage

+

There is nothing to click: once loaded server-wide, the module works +silently in the background.

+

Odoo’s own cron only logs its start at DEBUG (invisible by default) and +nothing at all on a graceful stop. This module logs the lifecycle at +INFO instead, matching the style of queue_job’s jobrunner +(“starting”/”stopped”):

+
+INFO ... base_cron_reconnect: patched ThreadedServer.cron_thread to
+survive a lost database connection (retry interval: 60s).
+INFO ... cron0 starting
+
+

What a recovery looks like in the log:

+
+WARNING ... cron0 died, most likely due to a lost database connection;
+restarting it in 60s.
+Traceback (most recent call last):
+...
+INFO ... cron0 restarting now.
+
+

And on a clean server shutdown while a retry is pending:

+
+INFO ... cron0 stopped gracefully
+
+

Recommended: alert on WARNING-level records from the +odoo.addons.base_cron_reconnect.hooks logger - the point of this +module is as much observability as recovery. A cron thread crashing at +all usually means something worth investigating (a database restart, a +network issue, connection pool exhaustion); this module’s own record is +deliberately a WARNING, not an ERROR, since the underlying +disconnect itself is already logged (e.g. by odoo.sql_db) and this +one just confirms the thread is recovering on its own.

+

No WARNING is logged on a normal, clean server restart - the module +tracks the shutdown signal and stays quiet in that case, so the alert +stays low-noise.

+

To verify on a test instance: restart the PostgreSQL server (or its +container) while Odoo is running in threaded mode with this module +loaded, and watch the log for the WARNING above followed by +cron%d restarting now. and normal cron polling resuming.

+
+
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Contributors

+
    +
  • Odoo Community Association (OCA)
    • +
+
+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

This module is part of the OCA/server-tools project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+
+ + diff --git a/base_cron_reconnect/tests/__init__.py b/base_cron_reconnect/tests/__init__.py new file mode 100644 index 00000000000..2b72f59df1a --- /dev/null +++ b/base_cron_reconnect/tests/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2026 Odoo Community Association (OCA) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +from . import test_cron_reconnect diff --git a/base_cron_reconnect/tests/test_cron_reconnect.py b/base_cron_reconnect/tests/test_cron_reconnect.py new file mode 100644 index 00000000000..2bd140d77c2 --- /dev/null +++ b/base_cron_reconnect/tests/test_cron_reconnect.py @@ -0,0 +1,177 @@ +# Copyright 2026 Odoo Community Association (OCA) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +import logging +import os +import unittest +from unittest import mock + +from odoo.tests.common import BaseCase + +from .. import hooks + + +class TestCronReconnect(BaseCase): + """No ORM/database needed: this module has no models, only a + server_wide post_load hook that patches a class attribute. Using + ``BaseCase`` rather than plain ``unittest.TestCase`` only for its + ``MetaCase`` metaclass, which sets ``test_tags``/``test_module``/ + ``test_class``. Without those, Odoo's own test runner + (``TagsSelector.check()``, odoo/tests/tag_selector.py) silently + excludes every test in this class whenever it's invoked with + tag-based filtering - including module-scoped filters like + ``--test-tags=/base_cron_reconnect``, which OCA CI commonly uses. + These tests still pass fine locally via plain + ``python -m unittest``, which never applies Odoo's tag filter at + all - that's what masked the gap during development. + """ + + def setUp(self): + super().setUp() + self._original_cron_thread = hooks.server.ThreadedServer.cron_thread + self._original_on_stop_funcs = list(hooks.server.CommonServer._on_stop_funcs) + hooks._stop_event.clear() + self.addCleanup(self._restore) + + def _restore(self): + hooks.server.ThreadedServer.cron_thread = self._original_cron_thread + hooks.server.CommonServer._on_stop_funcs[:] = self._original_on_stop_funcs + hooks._stop_event.clear() + + def test_crash_is_retried_not_propagated(self): + """A cron_thread that raises is retried, not left dead.""" + calls = [] + + def fake_cron_thread(self_, number): + calls.append(number) + if len(calls) < 3: + raise ConnectionError("connection to server ... failed") + # third call "succeeds": stop the supervisor loop cleanly. + hooks._stop_event.set() + + hooks.server.ThreadedServer.cron_thread = fake_cron_thread + + # NOTE: pass the numeric logging.INFO, not the string "INFO". + # odoo/netsvc.py does `logging.addLevelName(logging.RUNBOT, "INFO")`, + # which hijacks the *name* "INFO" to mean level 25 process-wide. + # assertLogs(level="INFO") would resolve the string through that + # hijacked mapping and silently capture nothing from a plain + # logger.info() call (level 20). + with mock.patch.object( + hooks._stop_event, "wait", return_value=False + ) as wait_mock, self.assertLogs(hooks._logger.name, level=logging.INFO) as logs: + hooks.post_load() + hooks.server.ThreadedServer.cron_thread(None, 0) + + self.assertEqual(len(calls), 3) + self.assertEqual( + wait_mock.call_args_list, + [ + mock.call(hooks.DEFAULT_RETRY_INTERVAL), + mock.call(hooks.DEFAULT_RETRY_INTERVAL), + ], + ) + warning_records = [r for r in logs.records if r.levelname == "WARNING"] + # post_load() itself logs one INFO confirmation line; filter it out + # to isolate the "restarting now" records we actually care about. + restart_records = [ + r for r in logs.records if "restarting now" in r.getMessage() + ] + self.assertEqual(len(warning_records), 2) + for record in warning_records: + self.assertIsNotNone(record.exc_info) + self.assertIn("cron0", record.getMessage()) + # One "restarting now" per retry that actually happens (not logged + # if a shutdown is what ends the wait instead - see the dedicated + # test below). + self.assertEqual(len(restart_records), 2) + + def test_no_restart_log_if_shutdown_wins_the_race(self): + """No 'restarting now' log if _stop_event fires during the wait.""" + calls = [] + + def fake_cron_thread(self_, number): + calls.append(number) + raise ConnectionError("connection to server ... failed") + + hooks.server.ThreadedServer.cron_thread = fake_cron_thread + + def wait_and_shutdown(timeout): + # Simulate a clean server shutdown winning the race against + # the retry wait: the event becomes set while we're "waiting". + hooks._stop_event.set() + return True + + with mock.patch.object( + hooks._stop_event, "wait", side_effect=wait_and_shutdown + ), self.assertLogs(hooks._logger.name, level=logging.INFO) as logs: + hooks.post_load() + hooks.server.ThreadedServer.cron_thread(None, 2) + + self.assertEqual(len(calls), 1) + restart_records = [ + r for r in logs.records if "restarting now" in r.getMessage() + ] + stopped_records = [ + r for r in logs.records if "stopped gracefully" in r.getMessage() + ] + self.assertEqual(restart_records, []) + self.assertEqual(len(stopped_records), 1) + + def test_lifecycle_logs_starting_and_stopped(self): + """cronN logs 'starting' and 'stopped gracefully' at INFO level, + matching queue_job's jobrunner lifecycle logging - Odoo's own + cron_spawn()/cron_thread() only log the equivalent at DEBUG (start) + or not at all (stop). + """ + + def fake_cron_thread(self_, number): + hooks._stop_event.set() + + hooks.server.ThreadedServer.cron_thread = fake_cron_thread + + with self.assertLogs(hooks._logger.name, level=logging.INFO) as logs: + hooks.post_load() + hooks.server.ThreadedServer.cron_thread(None, 3) + + messages = [r.getMessage() for r in logs.records] + self.assertIn("cron3 starting", messages) + self.assertIn("cron3 stopped gracefully", messages) + + def test_post_load_is_idempotent(self): + """Calling post_load() more than once must not double-wrap.""" + + def sentinel(self_, number): + pass + + hooks.server.ThreadedServer.cron_thread = sentinel + + hooks.post_load() + patched = hooks.server.ThreadedServer.cron_thread + self.assertIsNot(patched, sentinel) + self.assertTrue(getattr(patched, hooks._PATCH_MARKER, False)) + + on_stop_count = len(hooks.server.CommonServer._on_stop_funcs) + + hooks.post_load() + hooks.post_load() + + self.assertIs(hooks.server.ThreadedServer.cron_thread, patched) + self.assertEqual(len(hooks.server.CommonServer._on_stop_funcs), on_stop_count) + + def test_retry_interval_from_config(self): + with mock.patch.dict(hooks.config.options, {hooks.CONFIG_KEY: "5"}): + self.assertEqual(hooks._get_retry_interval(), 5) + + with mock.patch.dict(hooks.config.options, {hooks.CONFIG_KEY: "not-a-number"}): + self.assertEqual(hooks._get_retry_interval(), hooks.DEFAULT_RETRY_INTERVAL) + + with mock.patch.dict(hooks.config.options, {hooks.CONFIG_KEY: "0"}): + self.assertEqual(hooks._get_retry_interval(), 1) + + with mock.patch.dict(hooks.config.options, {}, clear=True): + with mock.patch.dict(os.environ, {hooks.ENV_VAR: "7"}): + self.assertEqual(hooks._get_retry_interval(), 7) + + +if __name__ == "__main__": + unittest.main() diff --git a/setup/base_cron_reconnect/odoo/addons/base_cron_reconnect b/setup/base_cron_reconnect/odoo/addons/base_cron_reconnect new file mode 120000 index 00000000000..7b6c89cd704 --- /dev/null +++ b/setup/base_cron_reconnect/odoo/addons/base_cron_reconnect @@ -0,0 +1 @@ +../../../../base_cron_reconnect \ No newline at end of file diff --git a/setup/base_cron_reconnect/setup.py b/setup/base_cron_reconnect/setup.py new file mode 100644 index 00000000000..28c57bb6403 --- /dev/null +++ b/setup/base_cron_reconnect/setup.py @@ -0,0 +1,6 @@ +import setuptools + +setuptools.setup( + setup_requires=['setuptools-odoo'], + odoo_addon=True, +)