diff --git a/.gitignore b/.gitignore index b7467aa9..436b173a 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ venv .cursor *~ -.*.swp +*.sw[op] +.*.sw[op] -DS_Store +.DS_Store diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 06fac5da..51e4c815 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,21 +5,12 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - # Ruff version. rev: v0.15.12 hooks: - # Run the linter. - id: ruff-check args: [ --fix ] - # Run the formatter. - id: ruff-format - repo: https://github.com/tox-dev/pyproject-fmt rev: "v2.21.2" hooks: - id: pyproject-fmt - - #- repo: https://github.com/pre-commit/mirrors-clang-format - # rev: v15.0.7 - # hooks: - # - id: clang-format - # args: ["--sort-includes=false"] diff --git a/README.md b/README.md index 49ace25f..56646b71 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,18 @@ RecSync ======= The record synchronizer project includes two parts. -A client [RecCaster](https://github.com/ChannelFinder/reccaster) which runing -as part of an EPICS IOC, and a server ([RecCeiver](./server/README.md)) -which is a stand alone daemon. Together they work to ensure the the server(s) +A client, [RecCaster](https://github.com/ChannelFinder/reccaster), which runs +as part of an EPICS IOC, and a server, [RecCeiver](./server/README.md), +which is a stand alone daemon. Together they work to ensure that the server(s) have a complete list of all records currently provided by the client IOCs. +This repository holds the RecCeiver source in `server/`, and describes the +protocol the two speak below. The RecCaster source was hosted here as well +until June 2026, when it moved to +[ChannelFinder/reccaster](https://github.com/ChannelFinder/reccaster); +`client/` is now a stub pointing there. + Information Uploaded -------------------- @@ -25,7 +31,8 @@ RecCeiver Usage The RecCeiver server in the `server/` directory is a Python script using the [Twisted][twisted] networking -library. It requires Python 3.6 or above and Twisted >= 12.0. +library. It requires Python 3.6 or above. The supported Twisted +version depends on the Python version; see `server/pyproject.toml`. [twisted]: http://twistedmatrix.com/ @@ -34,10 +41,11 @@ make client information available to one or more plugins. See `server/demo.conf` for an example configuration. -Currently two plugins are provided: `show` which print client -information to screen/log, and `db` which writes into a SQL -database (currently only sqlite3 supported). -The SQL table schema used is defined in `server/recceiver.sqlite3`. +Currently three plugins are provided: `show` which prints client +information to screen/log, `db` which writes into a SQL +database (currently only sqlite3 supported), and `cf` which writes +into [ChannelFinder](https://github.com/ChannelFinder/ChannelFinderService). +The SQL table schema used by `db` is defined in `server/recceiver.sqlite3`. Theory of Operation =================== diff --git a/client/README.md b/client/README.md new file mode 100644 index 00000000..09dc2860 --- /dev/null +++ b/client/README.md @@ -0,0 +1,11 @@ +RecCaster +========= + +The RecCaster source moved to +[ChannelFinder/reccaster](https://github.com/ChannelFinder/reccaster) in June +2026. Nothing is built from this directory; the `Makefile` here exists only to +report the move to anyone building an older checkout. + +```bash +git clone https://github.com/ChannelFinder/reccaster +``` diff --git a/debian/README b/debian/README deleted file mode 100644 index a1ed37c7..00000000 --- a/debian/README +++ /dev/null @@ -1,19 +0,0 @@ -Setup for Debian. - -The init script /etc/init.d/recceiver is a no-op unless the config file -(default /etc/recceiver.conf) exists. A sample is provided as -/etc/recceiver.conf.example. - -If the sqlite3 DB backend is used then the .db file must be initialized as follows. -This file must be read/write-able by the user account under which the recceiver daemon runs. -By default this is "recceiver" - -$ sqlite3 -init usr/share/doc/recceiver/recceiver.sqlite3 /path/to/somefile.db .exit -$ sudo chown -R recceiver /path/to -$ sudo chmod -R u+rw recceiver /path/to - -"/path/to/somefile.db" is a place holder. Whatever path is used -must by specified with "dbname" in the config file. - -Sqlite needs to create temporary files in the directory containing a .db, -so must have write permission. diff --git a/debian/compat b/debian/compat deleted file mode 100644 index ec635144..00000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -9 diff --git a/debian/control b/debian/control deleted file mode 100644 index 5398e746..00000000 --- a/debian/control +++ /dev/null @@ -1,47 +0,0 @@ -Source: recsync -Section: libdevel -Priority: extra -Maintainer: Dylan Maxwell -Build-Depends: debhelper (>= 9), dh-python, epics-debhelper (>= 8.14~), - epics-dev, - python-all-dev, python-setuptools, - python-twisted-core, -Standards-Version: 3.9.6 -XS-Python-Version: >= 2.7 -Homepage: https://github.com/ChannelFinder/recsync -Vcs-Git: https://github.com/ChannelFinder/recsync.git -Vcs-Browser: https://github.com/ChannelFinder/recsync - -Package: epics-recsync-dev -Architecture: any -Depends: libreccaster1 (= ${binary:Version}), - ${shlibs:Depends}, ${misc:Depends}, - ${epics:Depends}, -Suggests: recceiver, -Description: recsync client - The client component of recsync. Pushes record list and meta-info - to a server (recceiver). - . - This package contains headers and libraries needed at build time. - -Package: libreccaster1 -Architecture: any -Depends: ${shlibs:Depends}, ${misc:Depends}, -Description: recsync client - The client component of recsync. Pushes record list and meta-info - to a server (recceiver). - . - This package contains runtime libraries - -Package: recceiver -Architecture: all -Section: admin -Depends: ${shlibs:Depends}, ${python:Depends}, ${misc:Depends}, - adduser, lsb-base, -Suggests: epics-recsync-dev, -XB-Python-Version: ${python:Versions} -Description: recsync client - The client component of recsync. Pushes record list and meta-info - to a server (recceiver). - . - This package contains the recsync server component. diff --git a/debian/copyright b/debian/copyright deleted file mode 100644 index b1a1be21..00000000 --- a/debian/copyright +++ /dev/null @@ -1,70 +0,0 @@ -Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ -Upstream-Name: recsync -Upstream-Contact: Michael Davidsaver -Source: http://some.url/ - -Files: * -Copyright: Copyright 2016 Michael Davidsaver -License: EPICS - recsync is distributed subject to the following license conditions: - . - SOFTWARE LICENSE AGREEMENT - Software: recsync - . - 1. The "Software", below, refers to recsync (in either source code, or - binary form and accompanying documentation). Each licensee is - addressed as "you" or "Licensee." - . - 2. The copyright holders shown above and their third-party licensors - hereby grant Licensee a royalty-free nonexclusive license, subject to - the limitations stated herein and U.S. Government license rights. - . - 3. You may modify and make a copy or copies of the Software for use - within your organization, if you meet the following conditions: - a. Copies in source code must include the copyright notice and this - Software License Agreement. - b. Copies in binary form must include the copyright notice and this - Software License Agreement in the documentation and/or other - materials provided with the copy. - . - 4. You may modify a copy or copies of the Software or any portion of it, - thus forming a work based on the Software, and distribute copies of - such work outside your organization, if you meet all of the following - conditions: - a. Copies in source code must include the copyright notice and this - Software License Agreement; - b. Copies in binary form must include the copyright notice and this - Software License Agreement in the documentation and/or other - materials provided with the copy; - c. Modified copies and works based on the Software must carry - prominent notices stating that you changed specified portions of - the Software. - . - 5. Portions of the Software resulted from work developed under a U.S. - Government contract and are subject to the following license: the - Government is granted for itself and others acting on its behalf a - paid-up, nonexclusive, irrevocable worldwide license in this computer - software to reproduce, prepare derivative works, and perform publicly - and display publicly. - . - 6. WARRANTY DISCLAIMER. THE SOFTWARE IS SUPPLIED "AS IS" WITHOUT WARRANTY - OF ANY KIND. THE COPYRIGHT HOLDERS, THEIR THIRD PARTY LICENSORS, THE - UNITED STATES, THE UNITED STATES DEPARTMENT OF ENERGY, AND THEIR - EMPLOYEES: (1.0) DISCLAIM ANY WARRANTIES, EXPRESS OR IMPLIED, INCLUDING - BUT NOT LIMITED TO ANY IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS - FOR A PARTICULAR PURPOSE, TITLE OR NON-INFRINGEMENT, (2) DO NOT ASSUME - ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, - OR USEFULNESS OF THE SOFTWARE, (3) DO NOT REPRESENT THAT USE OF THE - SOFTWARE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS, (4) DO NOT WARRANT - THAT THE SOFTWARE WILL FUNCTION UNINTERRUPTED, THAT IT IS ERROR-FREE - OR THAT ANY ERRORS WILL BE CORRECTED. - . - 7. LIMITATION OF LIABILITY. IN NO EVENT WILL THE COPYRIGHT HOLDERS, THEIR - THIRD PARTY LICENSORS, THE UNITED STATES, THE UNITED STATES DEPARTMENT - OF ENERGY, OR THEIR EMPLOYEES: BE LIABLE FOR ANY INDIRECT, INCIDENTAL, - CONSEQUENTIAL, SPECIAL OR PUNITIVE DAMAGES OF ANY KIND OR NATURE, - INCLUDING BUT NOT LIMITED TO LOSS OF PROFITS OR LOSS OF DATA, FOR ANY - REASON WHATSOEVER, WHETHER SUCH LIABILITY IS ASSERTED ON THE BASIS OF - CONTRACT, TORT (INCLUDING NEGLIGENCE OR STRICT LIABILITY), OR - OTHERWISE, EVEN IF ANY OF SAID PARTIES HAS BEEN WARNED OF THE - POSSIBILITY OF SUCH LOSS OR DAMAGES. diff --git a/debian/recceiver.init b/debian/recceiver.init deleted file mode 100755 index 451a9867..00000000 --- a/debian/recceiver.init +++ /dev/null @@ -1,105 +0,0 @@ -#! /bin/sh -# -### BEGIN INIT INFO -# Provides: recceiver -# Required-Start: $remote_fs $local_fs $network $syslog $time -# Required-Stop: $remote_fs $local_fs $network $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: recsync server -# Description: Collects record lists from IOCs -### END INIT INFO - -PATH=/sbin:/bin:/usr/sbin:/usr/bin -DAEMON=/usr/bin/twistd -DNAME=twistd -NAME=recceiver -DESC="recsync server" - -# defaults -RUN=no -RUN_AS_USER=recceiver -RUN_AS_GROUP=nogroup - -CONF_FILE="/etc/recceiver.conf" -PID_FILE="/var/run/recceiver/$NAME.pid" -LOG_FILE="/var/log/recceiver.log" - -# Include admin settings if available -if [ -f /etc/default/recceiver ] ; then - . /etc/default/recceiver -fi - -# "-r poll" due to bug in epoll reactor which causes fail on startup -DAEMON_OPTS="-r poll --logfile=$LOG_FILE --pidfile=$PID_FILE --uid=$RUN_AS_USER --gid=$RUN_AS_GROUP --umask=0022" -DAEMON_OPTS="$DAEMON_OPTS recceiver -f $CONF_FILE" - -SSD_OPTS="-q --pidfile $PID_FILE" -SSD_OPTS="$SSD_OPTS --name $DNAME" -SSD_OPTS="$SSD_OPTS --startas $DAEMON" - -set -e - -. /lib/lsb/init-functions - -serv_start() { - log_daemon_msg "Starting recsync server" - [ -d "/var/run/recceiver" ] || install -d "$(dirname "$PID_FILE")" - chown -R $RUN_AS_USER:$RUN_AS_GROUP "$(dirname "$PID_FILE")" - if [ -f "$CONF_FILE" ] - then - log_progress_msg "recceiver" - start-stop-daemon --start $SSD_OPTS -- $DAEMON_OPTS - else - log_warning_msg "No config file ($CONF_FILE) skipping" - fi - log_end_msg $? -} - -serv_stop() { - log_daemon_msg "Stopping recsync server" recceiver - start-stop-daemon --stop $SSD_OPTS - log_end_msg $? -} - -serv_status() { - #Debian >= 7... - #start-stop-daemon --status $SSD_OPTS - #Debian <= 6 (and others) - status_of_proc -p "$PID_FILE" "" "$NAME" -} - -serv_restart() { - serv_stop && sleep 1 && serv_start -} - -case "$1" in - start) - serv_start - RET=$? - ;; - stop) - serv_stop - RET=$? - ;; - force-reload) - # check whether $DAEMON is running. If so, restart - serv_status && serv_restart - RET=$? - ;; - restart) - serv_restart - RET=$? - ;; - status) - serv_status - RET=$? - ;; - *) - N=/etc/init.d/$NAME - echo "Usage: $N {start|stop|restart|force-reload}" >&2 - exit 1 - ;; -esac - -exit ${RET:=42} diff --git a/debian/recceiver.install b/debian/recceiver.install deleted file mode 100644 index f9dfea18..00000000 --- a/debian/recceiver.install +++ /dev/null @@ -1,4 +0,0 @@ -usr/lib/python2.7/dist-packages/recceiver* -usr/lib/python2.7/dist-packages/twisted/plugins/recceiver_plugin.py* -server/recceiver.sqlite3 usr/share/doc/recceiver/ -debian/README usr/share/doc/recceiver/ diff --git a/debian/recceiver.postinst b/debian/recceiver.postinst deleted file mode 100644 index 43098870..00000000 --- a/debian/recceiver.postinst +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -set -e - -if [ "$1" = "configure" ]; then - if ! grep -q '^recceiver:' /etc/passwd; then - adduser --system --home / --ingroup nogroup \ - --no-create-home --disabled-password recceiver > /dev/null 2>&1 - fi -fi - -#DEBHELPER# diff --git a/debian/recceiver.postrm b/debian/recceiver.postrm deleted file mode 100644 index 9cbb27c9..00000000 --- a/debian/recceiver.postrm +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -set -e - -#DEBHELPER# - -if [ "$1" = "purge" ]; then - if grep -q '^recceiver:' /etc/passwd; then - echo "Removing user recceiver" - userdel recceiver || echo "WARNING: problem removing user recceiver" - fi -fi diff --git a/debian/recceiver.triggers b/debian/recceiver.triggers deleted file mode 100644 index 419035c2..00000000 --- a/debian/recceiver.triggers +++ /dev/null @@ -1 +0,0 @@ -activate twisted-plugins-cache diff --git a/debian/rules b/debian/rules deleted file mode 100755 index eae3db2d..00000000 --- a/debian/rules +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/make -f - -export DH_VERBOSE=1 -export SKIPDEMO=YES - -# ABI version set manually -export SHRLIB_VERSION=1 - -%: - dh $@ --with epics --with python2 --parallel - -override_dh_auto_clean: - echo "EPICS_BASE=/usr/lib/epics" > client/configure/RELEASE.local - dh_auto_clean -Sepicsmake -Dclient - dh_auto_clean -Spybuild -Dserver - rm -f client/configure/RELEASE.local - -override_dh_auto_configure: - echo "EPICS_BASE=/usr/lib/epics" > client/configure/RELEASE.local - dh_auto_configure -Sepicsmake -Dclient - dh_auto_configure -Spybuild -Dserver - -override_dh_auto_build: - dh_auto_build -Sepicsmake -Dclient - dh_auto_build -Spybuild -Dserver - -override_dh_auto_install: - dh_auto_install -Sepicsmake -Dclient - dh_auto_install -Spybuild -Dserver - install -d debian/recceiver/etc - cp server/demo.conf debian/recceiver/etc/recceiver.conf - -override_dh_install: - rm -f debian/tmp/usr/lib/epics/configure/RELEASE.local - dh_install --fail-missing - -override_dh_installinit: - dh_installinit --error-handler=true diff --git a/debian/source/format b/debian/source/format deleted file mode 100644 index 89ae9db8..00000000 --- a/debian/source/format +++ /dev/null @@ -1 +0,0 @@ -3.0 (native) diff --git a/debian/source/options b/debian/source/options deleted file mode 100644 index 62d91781..00000000 --- a/debian/source/options +++ /dev/null @@ -1 +0,0 @@ -compression-level = 9 diff --git a/server/.dockerignore b/server/.dockerignore index 42f280f0..3f66f16d 100644 --- a/server/.dockerignore +++ b/server/.dockerignore @@ -5,7 +5,7 @@ # Docker -docker-compose.yml +compose.yml Dockerfile .docker .dockerignore diff --git a/server/README.md b/server/README.md index 1fd90eaf..3f4af7ed 100644 --- a/server/README.md +++ b/server/README.md @@ -1,6 +1,6 @@ # Recceiver -Application for talking between IOCs (via [reccaster](../client)) and ChannelFinder (via [pyCFClient](https://github.com/ChannelFinder/pyCFClient)). +Application for talking between IOCs (via [reccaster](https://github.com/ChannelFinder/reccaster)) and ChannelFinder (via [pyCFClient](https://github.com/ChannelFinder/pyCFClient)). Written using [twistd](https://twisted.org/). diff --git a/server/compose.yml b/server/compose.yml index a1b5f178..183cfe34 100644 --- a/server/compose.yml +++ b/server/compose.yml @@ -8,10 +8,16 @@ services: condition: service_healthy restart: true healthcheck: - test: netstat | grep cf - interval: 10s - timeout: 30s - retries: 3 + # Ready means the TCP listener is open, which is what an IOC needs before + # it can act on an announcement. RecCeiver binds a random port unless + # bind is set, so match on the listening state instead of a port number. + # Connecting to the port would register as an IOC connection and churn + # the maxActive accounting on every probe. + test: netstat -lnt | grep -q LISTEN + interval: 5s + timeout: 3s + retries: 5 + start_period: 10s environment: RECCEIVER_RECCEIVER_PROCS: cf RECCEIVER_RECCEIVER_LOGLEVEL: DEBUG diff --git a/server/recceiver/announcer.py b/server/recceiver/announcer.py index eaba18ba..22ad8cce 100644 --- a/server/recceiver/announcer.py +++ b/server/recceiver/announcer.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import logging import socket diff --git a/server/recceiver/application.py b/server/recceiver/application.py index d47bfff6..31e7e382 100644 --- a/server/recceiver/application.py +++ b/server/recceiver/application.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import logging import random diff --git a/server/recceiver/dbstore.py b/server/recceiver/dbstore.py index a8bb80e5..a8171339 100644 --- a/server/recceiver/dbstore.py +++ b/server/recceiver/dbstore.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import itertools import logging diff --git a/server/recceiver/interfaces.py b/server/recceiver/interfaces.py index b9e4e992..6df2a7f4 100644 --- a/server/recceiver/interfaces.py +++ b/server/recceiver/interfaces.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - from twisted.application import service from zope.interface import Attribute, Interface diff --git a/server/recceiver/metrics.py b/server/recceiver/metrics.py index ca5f2a3f..f940b2db 100644 --- a/server/recceiver/metrics.py +++ b/server/recceiver/metrics.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import logging _log = logging.getLogger(__name__) diff --git a/server/recceiver/processors.py b/server/recceiver/processors.py index d1c32a3e..a5be62d5 100644 --- a/server/recceiver/processors.py +++ b/server/recceiver/processors.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import configparser as ConfigParser import logging import os diff --git a/server/recceiver/protocol/announce.py b/server/recceiver/protocol/announce.py index afdb6c54..f9768566 100644 --- a/server/recceiver/protocol/announce.py +++ b/server/recceiver/protocol/announce.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Framework-neutral RecSync UDP announce packet.""" import socket diff --git a/server/recceiver/protocol/messages.py b/server/recceiver/protocol/messages.py index 979f6097..58566e02 100644 --- a/server/recceiver/protocol/messages.py +++ b/server/recceiver/protocol/messages.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Framework-neutral RecSync TCP protocol messages.""" import struct diff --git a/server/recceiver/recast.py b/server/recceiver/recast.py index be4af974..94dfa00a 100644 --- a/server/recceiver/recast.py +++ b/server/recceiver/recast.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import collections import logging import random diff --git a/server/recceiver_full.conf b/server/recceiver_full.conf index 6a35ed1d..410f6ee0 100644 --- a/server/recceiver_full.conf +++ b/server/recceiver_full.conf @@ -41,6 +41,9 @@ tcptimeout = 15.0 # Doesn't effect IOC clients commitInterval = 5.0 +# Maximum records per commit. Default is 5000; set to 0 for no limit. +#commitSizeLimit = 5000 + # Maximum concurrent "active" clients # to allow. maxActive = 20 @@ -129,6 +132,7 @@ verifySSL = True pushMaxRetries = 10 # Whether to retry polling indefinitely until success. Default is False. +# Enabling this holds the global commit lock until CF recovers; all other IOC commits are blocked. pushAlwaysRetry = False # Interval in seconds between periodic CF status log lines (0 to disable) diff --git a/server/tests/test_protocol.py b/server/tests/unit/test_protocol.py similarity index 100% rename from server/tests/test_protocol.py rename to server/tests/unit/test_protocol.py diff --git a/server/tests/test_recast.py b/server/tests/unit/test_recast_session.py similarity index 100% rename from server/tests/test_recast.py rename to server/tests/unit/test_recast_session.py diff --git a/server/twisted/plugins/recceiver_plugin.py b/server/twisted/plugins/recceiver_plugin.py index 6b43f8b5..94f019fc 100644 --- a/server/twisted/plugins/recceiver_plugin.py +++ b/server/twisted/plugins/recceiver_plugin.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - from recceiver import dbstore, processors from recceiver.application import Maker from recceiver.cf.processor import CFProcessor