Create SMuRF Hammer Agent

docs/agents/smurf_hammer.rst

@@ -0,0 +1,105 @@
+.. highlight:: rst
+
+.. _smurf_hammer:
+
+==================
+SMuRF Hammer Agent
+==================
+
+.. image:: https://img.shields.io/badge/AI-assisted-orange
+   :alt: Agent written with AI assitance
+
+The SMuRF Hammer Agent wraps sodetlib's ``jackhammer hammer`` CLI command as an
+OCS agent. It operates on the crate controlled by the SMuRF server to which it
+is deployed and accepts the usual ``jackhammer`` options. Slots for which the
+hammer fails will be reported in the session data. There is also a monitoring
+process to expose the configuration state of each slot.
+
+.. argparse::
+    :filename: ../socs/agents/smurf_hammer/agent.py
+    :func: add_agent_args
+    :prog: python3 agent.py
+
+Dependencies
+--------------
+
+The SMuRF Hammer Agent requires the following packages:
+
+    - `sodetlib <https://github.com/simonsobs/sodetlib>`_
+
+jackhammer requires EPICS to be installed on the host. The easiest
+way to do this is to follow the installation steps present in the
+`Dockerfile from smurf_docker <https://github.com/simonsobs/smurf_dockers/blob/v0.0.9/smurf_base/Dockerfile>`_
+To build that version of EPICS I found I needed to use the following
+compiler flags:
+
+.. code-block:: bash
+
+    -std=c++98 -fno-access-control
+
+Note that it is also necessary to set the EPICS environment variables
+from the Dockerfile in the host's .bashrc and also the following:
+
+.. code-block:: bash
+
+    EPICS_CA_ADDR_LIST=127.255.255.255
+    EPICS_CA_MAX_ARRAY_BYTES=80000000
+
+Additionally, ``socs`` should be installed with the ``pysmurf`` group:
+
+.. code-block:: bash
+
+    $ pip install -U socs[pysmurf]
+
+Configuration File Examples
+------------------------------
+Below are configuration examples for the ocs config file and for the
+docker compose service.
+
+OCS Site Config
+`````````````````
+Here is an example of an agent configuration block for the ocs-site-config
+file::
+
+    {'agent-class': 'SmurfHammerAgent',
+     'instance-id': 'smurf-hammer',
+     'arguments': []},
+
+To suppress the auto-starting monitor process::
+
+    {'agent-class': 'SmurfHammerAgent',
+     'instance-id': 'smurf-hammer',
+     'arguments': ['--no-processes']},
+
+Description
+--------------
+
+The agent exposes two operations:
+
+**hammer** (task)
+    Resets and reconfigures the specified SMuRF slots by calling
+    ``sodetlib.hammers.jackhammer.hammer()``. The sequence reboots the
+    carriers, waits for EPICS connectivity, and runs pysmurf setup. Failures
+    at each stage are caught per-slot, and the remaining slots continue.
+
+**monitor** (process)
+    Continuously polls each slot's EPICS registers using ``epics.caget()``
+    directly (bypassing pysmurf, so it works even when the system is
+    degraded). Two registers are queried per slot every 10 seconds:
+
+    - ``AMCc.SmurfApplication.SystemConfigured`` -- whether pysmurf setup
+      has completed.
+    - ``AMCc.SmurfApplication.ConfiguringInProgress`` -- whether setup is
+      currently running.
+
+    Results are published to the ``system_configured`` OCS feed for HK
+    archival and Grafana. The monitor auto-starts on agent launch unless
+    ``--no-processes`` is passed.
+
+    Unreachable slots are recorded as ``None`` in session data and ``-1``
+    in the feed.
+
+Agent API
+-----------
+.. autoclass:: socs.agents.smurf_hammer.agent.SmurfHammerAgent
+    :members:

docs/conf.py

@@ -108,6 +108,8 @@
                         'sodetlib',
                         'sodetlib.operations',
                         'sodetlib.det_config',
+                        'sodetlib.hammers',
+                        'sodetlib.hammers.jackhammer',
                         'src',
                         'src.pid_controller',
                         'cv2',

docs/index.rst

@@ -78,6 +78,7 @@ API Reference        Full API documentation for core parts of the SOCS library.
     agents/rtsp_camera
     agents/scpi_psu
     agents/smurf_crate_monitor
+    agents/smurf_hammer
     agents/smurf_timing_card
     agents/srs_cg635
     agents/stimulator_encoder

socs/agents/smurf_hammer/agent.py

@@ -0,0 +1,258 @@
+# Agent developed with AI assistance.
+
+import argparse
+import os
+import time
+import traceback
+
+import epics
+import txaio
+from ocs import ocs_agent, site_config
+from ocs.ocs_twisted import TimeoutLock
+from sodetlib.hammers.jackhammer import hammer, sys_config
+
+
+class SmurfHammerAgent:
+    """Agent to execute the sodetlib jackhammer hammer sequence and
+    monitor the configured status of each SMuRF slot.
+
+    Parameters
+    ----------
+    agent : OCSAgent
+        OCSAgent object which forms this Agent.
+    """
+
+    def __init__(self, agent):
+        self.agent = agent
+        self.log = agent.log
+        self.lock = TimeoutLock()
+        self._monitor_running = False
+
+        # get slots from config file
+        self.slot_order = list(sys_config['slot_order'])
+
+        self.agent.register_feed('system_configured',
+                                 record=True,
+                                 buffer_time=1)
+
+    @ocs_agent.param('slots', default=None)
+    @ocs_agent.param('no_reboot', default=False, type=bool)
+    @ocs_agent.param('dump_logs', default=False, type=bool)
+    @ocs_agent.param('skip_setup', default=False, type=bool)
+    @ocs_agent.param('dump_rogue', default=False, type=bool)
+    def hammer(self, session, params):
+        """hammer(slots=None, no_reboot=False, dump_logs=False, \
+                  skip_setup=False, dump_rogue=False)
+
+        **Task** - Execute the jackhammer hammer sequence to reset and
+        reconfigure SMuRF slots. This replicates the ``jackhammer hammer``
+        CLI command.
+
+        Individual slot failures are isolated so that the remaining
+        slots can still complete successfully.
+
+        Parameters
+        ----------
+        slots : list of int, optional
+            Slot numbers to hammer. Defaults to all slots defined in
+            the sys_config.
+        no_reboot : bool
+            If True, perform a soft reset without rebooting the carriers.
+        dump_logs : bool
+            If True, dump docker logs before hammering.
+        skip_setup : bool
+            If True, skip pysmurf setup after reboot.
+        dump_rogue : bool
+            If True, dump the rogue tree before hammering.
+
+        Notes
+        -----
+        The session data object reports per-slot results::
+
+            >>> response.session['data']
+            {'slots': [2, 3],
+             'reboot': True,
+             'succeeded_slots': [2],
+             'failed_slots': {3: 'EPICS connection timed out ...'},
+             'error': None}
+        """
+
+        session.data = {
+            'slots': params.get('slots'),
+            'reboot': not params['no_reboot'],
+            'succeeded_slots': [],
+            'failed_slots': {},
+            'error': None,
+            'timestamp': time.time(),
+        }
+
+        try:
+            result = hammer(
+                slots=params['slots'],
+                no_reboot=params['no_reboot'],
+                no_dump=not params['dump_logs'],
+                skip_setup=params['skip_setup'],
+                dump_rogue=params['dump_rogue'],
+            )
+        except Exception as e:
+            self.log.error(
+                "Hammer failed: {error}\n{traceback}",
+                error=e,
+                traceback=traceback.format_exc()
+            )
+            session.data['error'] = f"{e}"
+            return False, f"Hammer failed: {e}"
+
+        session.data['succeeded_slots'] = result['succeeded']
+        session.data['failed_slots'] = result['failed']
+
+        succeeded = result['succeeded']
+        failed = result['failed']
+
+        if not succeeded:
+            return False, f"All slots failed: {failed}"
+        if failed:
+            return False, (
+                f"Partial success: slots {succeeded} succeeded, "
+                f"slots {list(failed.keys())} failed"
+            )
+        return True, f"Successfully hammered slots {succeeded}"
+
+    @ocs_agent.param('_')
+    def monitor(self, session, params):
+        """monitor()
+
+        **Process** - Continuously monitor the configuration status of
+        each SMuRF slot by querying two EPICS registers:
+
+        - ``AMCc.SmurfApplication.SystemConfigured`` — whether setup
+          has completed successfully.
+        - ``AMCc.SmurfApplication.ConfiguringInProgress`` — whether
+          setup is currently running.
+
+        Notes
+        -----
+        The session data object reports per-slot status::
+
+            >>> response.session['data']
+            {'timestamp': 1721234567.0,
+             'slots': {
+                 2: {'configured': True, 'configuring': False},
+                 3: {'configured': False, 'configuring': True},
+                 5: {'configured': None, 'configuring': None},
+             }}
+
+        For each register, ``True``/``False`` reflect the register
+        value and ``None`` means the EPICS query timed out (slot
+        unreachable).
+        """
+        self._monitor_running = True
+        session.data = {}
+
+        while self._monitor_running:
+            slot_status = {}
+            start = time.time()
+            feed_data = {
+                'block_name': 'system_configured',
+                'timestamp': start,
+                'data': {},
+            }
+
+            pv_names = []
+            for slot in self.slot_order:
+                epics_root = f'smurf_server_s{slot}'
+                pv_names.append(f'{epics_root}:AMCc:SmurfApplication:SystemConfigured')
+                pv_names.append(f'{epics_root}:AMCc:SmurfApplication:ConfiguringInProgress')
+
+            try:
+                values = epics.caget_many(pv_names, timeout=5, connection_timeout=5)
+            except Exception as e:
+                self.log.error(f"EPICS query failed with: {e}")
+                values = [None] * len(pv_names)
+
+            for i, slot in enumerate(self.slot_order):
+                val_configured = values[2 * i]
+                val_configuring = values[2 * i + 1]
+
+                if val_configured is None:
+                    configured = None
+                    feed_data['data'][f'configured_s{slot}'] = -1
+                else:
+                    configured = bool(int(val_configured))
+                    feed_data['data'][f'configured_s{slot}'] = int(configured)
+
+                if val_configuring is None:
+                    configuring = None
+                    feed_data['data'][f'configuring_s{slot}'] = -1
+                else:
+                    configuring = bool(int(val_configuring))
+                    feed_data['data'][f'configuring_s{slot}'] = int(configuring)
+
+                slot_status[slot] = {
+                    'configured': configured,
+                    'configuring': configuring,
+                }
+
+            session.data = {
+                'timestamp': feed_data['timestamp'],
+                'slots': slot_status,
+            }
+
+            if feed_data['data']:
+                self.agent.publish_to_feed('system_configured', feed_data)
+
+            self.log.debug(
+                "Slot status: {status}", status=slot_status
+            )
+
+            # aim for 10s between samples
+            wait = 10 - (time.time() - start)
+            if wait > 0:
+                time.sleep(wait)
+
+        return True, 'Monitor exited cleanly.'
+
+    def _stop_monitor(self, session, params):
+        self._monitor_running = False
+        session.set_status('stopping')
+        return True, 'Stopping monitor.'
+
+
+def add_agent_args(parser_in=None):
+    if parser_in is None:
+        parser_in = argparse.ArgumentParser()
+    pgroup = parser_in.add_argument_group('Agent Options')
+    pgroup.add_argument('--no-processes', action='store_true',
+                        default=False,
+                        help="Do not auto-start the monitor process.")
+    return parser_in
+
+
+def main(args=None):
+    # set up logging
+    txaio.use_twisted()
+    txaio.start_logging(level=os.environ.get("LOGLEVEL", "info"))
+
+    parser = add_agent_args()
+    args = site_config.parse_args(agent_class='SmurfHammerAgent',
+                                  parser=parser,
+                                  args=args)
+
+    startup = not args.no_processes
+
+    agent, runner = ocs_agent.init_site_agent(args)
+    p = SmurfHammerAgent(agent)
+
+    agent.register_process('monitor',
+                           p.monitor,
+                           p._stop_monitor,
+                           blocking=True,
+                           startup=startup)
+
+    agent.register_task('hammer', p.hammer)
+
+    runner.run(agent, auto_reconnect=True)
+
+
+if __name__ == "__main__":
+    main()

socs/plugin.py

@@ -41,6 +41,7 @@
     'RTSPCameraAgent': {'module': 'socs.agents.rtsp_camera.agent', 'entry_point': 'main'},
     'ScpiPsuAgent': {'module': 'socs.agents.scpi_psu.agent', 'entry_point': 'main'},
     'SmurfFileEmulator': {'module': 'socs.agents.smurf_file_emulator.agent', 'entry_point': 'main'},
+    'SmurfHammerAgent': {'module': 'socs.agents.smurf_hammer.agent', 'entry_point': 'main'},
     'SmurfStreamSimulator': {'module': 'socs.agents.smurf_stream_simulator.agent', 'entry_point': 'main'},
     'SmurfTimingCardAgent': {'module': 'socs.agents.smurf_timing_card.agent', 'entry_point': 'main'},
     'SRSCG635Agent': {'module': 'socs.agents.srs_cg635.agent', 'entry_point': 'main'},