docs(protocol): threading, lifecycle and config contracts match the host

pj_base/include/pj_base/data_source_protocol.h

@@ -8,7 +8,9 @@
  *     See pj_base/plugin_data_api.h. append_arrow_ipc is removed.
  *   - Every vtable slot is PJ_NOEXCEPT. Trampolines that drop exceptions
  *     through the ABI boundary are now a compile-time error in C++.
- *   - Every slot carries a thread-class tag (// [main-thread], etc.).
+ *   - Every slot carries a thread-class tag (// [main-thread], etc.), except
+ *     start/stop, whose rule none of the tags expresses; see the threading
+ *     table on PJ_data_source_vtable_t.
  *
  * The host obtains the plugin's vtable via `PJ_get_data_source_vtable()`
  * and drives the plugin through: create -> bind(registry) -> load_config
@@ -63,13 +65,15 @@ extern "C" {
 #endif
 
 /**
- * Plugin lifecycle state machine.
+ * Plugin lifecycle states, as reported through notify_state / current_state.
  *
- * Valid transitions:
- *   idle -> configuring -> starting -> running -> stopping -> stopped
- *   running -> paused -> running  (if SUPPORTS_PAUSE capability set)
- *   any -> failed  (terminal)
- *   stopped is terminal — create a new instance to restart.
+ * ADVISORY: a host may ignore notify_state and never call current_state (the
+ * reference host does both), so a plugin must not depend on either for
+ * correctness. What a host acts on is the start() return value, request_stop,
+ * is_stop_requested and complete_ingest. A plugin is not required to visit
+ * every state; the SDK bases report starting, then running (stream) or
+ * stopped/failed (file). stopped and failed are terminal for the instance:
+ * the host never calls start() twice and creates a new instance to restart.
  */
 typedef enum {
   PJ_DATA_SOURCE_STATE_IDLE = 0,
@@ -255,7 +259,8 @@ typedef struct PJ_ingest_completion_t {
  * raw message payloads. The per-slot thread tag governs: the host may drive
  * start()/stop() on one thread and poll() (with its [stream-thread]
  * callbacks) on another, so a host must treat [stream-thread] callbacks as
- * arriving off its UI thread and marshal accordingly.
+ * arriving off its UI thread and marshal accordingly. When a plugin may call
+ * the [stream-thread] slots is narrowed in PJ_data_source_vtable_t.
  *
  * Fallible calls take a `PJ_error_t* out_error` which the callee populates
  * on failure. Callers may pass NULL if they don't need the detail.
@@ -287,7 +292,8 @@ typedef struct PJ_data_source_runtime_host_vtable_t {
   /** [thread-safe] Returns true if the host has requested the plugin to stop. */
   bool (*is_stop_requested)(void* ctx) PJ_NOEXCEPT;
 
-  /** [thread-safe] Inform the host that the plugin has transitioned to @p state. */
+  /** [thread-safe] Inform the host that the plugin has transitioned to @p state.
+   *  Advisory: a host may ignore it (see PJ_data_source_state_t). */
   void (*notify_state)(void* ctx, PJ_data_source_state_t state) PJ_NOEXCEPT;
 
   /**
@@ -308,7 +314,7 @@ typedef struct PJ_data_source_runtime_host_vtable_t {
       PJ_error_t* out_error) PJ_NOEXCEPT;
 
   /**
-   * [main-thread] Display a modal message box to the user and wait for
+   * [thread-safe] Display a modal message box to the user and wait for
    * their response. BLOCKS until the user closes the dialog. The host is
    * responsible for showing the dialog on the UI thread in a thread-safe
    * manner; the plugin may call from any thread and the host will marshal.
@@ -499,6 +505,43 @@ typedef struct {
  *   create  -> bind(registry) -> load_config (optional)
  *           -> start -> poll* -> stop -> destroy
  *
+ * Threading (normative). The host never runs two of create, bind,
+ * load_config, save_config, get_dialog, start, poll, stop and destroy
+ * concurrently on one instance, and each happens-after the previous returned;
+ * consecutive calls may come from different threads.
+ *
+ *   create, destroy, capabilities,  GUI thread, never while start/poll/stop
+ *   bind, load_config, save_config, runs. load_config may be called several
+ *   get_dialog                      times before start; save_config also
+ *                                   after start returned.
+ *   start                           Once. Any host thread: a FINITE_IMPORT
+ *                                   start may run on a worker and block for
+ *                                   the whole import (the host calls no
+ *                                   poll); a CONTINUOUS_STREAM start may run
+ *                                   on the GUI thread and must return
+ *                                   promptly.
+ *   poll                            CONTINUOUS_STREAM only, after start
+ *                                   returned true; repeatedly, from one host
+ *                                   thread that need not be start's.
+ *   stop                            Possibly on poll's thread, after the
+ *                                   last poll returned. May be skipped: a
+ *                                   host may destroy after a failed start or
+ *                                   a finished import without calling stop,
+ *                                   so release resources in destroy as well.
+ *   get_plugin_extension            Any thread, concurrently with anything.
+ *   set_active_topics (extension)   GUI thread, concurrently with poll and
+ *                                   with the plugin's own threads.
+ *   pause, resume, current_state    Advisory; a host may never call them.
+ *                                   If called: GUI thread, never while
+ *                                   start/poll/stop runs.
+ *
+ * Plugin -> host: [stream-thread] slots of the write hosts and of the runtime
+ * host may be called only while a start, poll or stop call on this instance
+ * is in progress, from one thread at a time (a plugin thread that the call
+ * waits on counts; a thread still producing after the call returned does not:
+ * between polls the host touches the same host objects). [thread-safe] slots
+ * may be called from any thread while the instance exists.
+ *
  * Fallible slots take a PJ_error_t* out-param which the callee populates
  * on failure. Callers may pass NULL to discard error detail. Every slot
  * is PJ_NOEXCEPT; exceptions from the implementation must be caught
@@ -556,23 +599,60 @@ typedef struct PJ_data_source_vtable_t {
    * On success, returns true and writes to @p out_json a view over a
    * plugin-owned string that remains valid until the next call to this
    * function on the same ctx.
+   *
+   * The output must be a UTF-8 JSON object that load_config accepts; a host
+   * persists it (e.g. in a layout) and later passes it back verbatim apart
+   * from the host-written keys listed under load_config. It may carry
+   * "filepath" and "_parser_config" back (a file layout replays this output,
+   * so that is how its parser config survives), never "__pj_saved_filepath".
    */
   bool (*save_config)(void* ctx, PJ_string_view_t* out_json, PJ_error_t* out_error) PJ_NOEXCEPT;
-  /** [main-thread] Restore plugin configuration from JSON. */
+  /**
+   * [main-thread] Restore plugin configuration from JSON.
+   *
+   * @p config_json is either empty, meaning "no saved config" (load_config
+   * is still called, e.g. on a stream's first run, and must keep defaults),
+   * or a UTF-8 JSON object: the output of this plugin's save_config or of its
+   * embedded dialog's save_config (after an accepted dialog), with host keys
+   * added or rewritten, or host keys alone. It may have been written by an
+   * OLDER version of the plugin, which load_config must still accept. Top-level
+   * keys starting with "_" belong to the host; the plugin must not define
+   * its own. Keys the reference host writes:
+   *   "filepath"            — file sources: the file to import, rewritten
+   *                           on layout replay to the file actually opened.
+   *   "_parser_config"      — string: the embedded parser dialog's config;
+   *                           forward it as parser_config_json in
+   *                           ensure_parser_binding (the host does not).
+   *   "__pj_saved_filepath" — layout replay only: the absolute path that
+   *                           "filepath" replaced, to rebase sidecar files.
+   * "__pj_fanout" is host-defined but plugin-written: an array of complete
+   * config strings in an accepted file-source config, asking the host to
+   * import each with its own instance. May be called several times before
+   * start.
+   */
   bool (*load_config)(void* ctx, PJ_string_view_t config_json, PJ_error_t* out_error) PJ_NOEXCEPT;
 
-  /** [main-thread] Begin data acquisition. May spawn stream threads internally. */
+  /** Begin data acquisition. Thread: see the table above (not necessarily the
+   *  GUI thread). May spawn plugin threads. If it fails, release what it
+   *  acquired: the host may destroy the instance without calling stop. */
   bool (*start)(void* ctx, PJ_error_t* out_error) PJ_NOEXCEPT;
-  /** [main-thread] Stop data acquisition. Must be idempotent. Failures are not reportable. */
+  /** Stop data acquisition. Thread: see the table above (may be poll's
+   *  thread). Must be idempotent. Failures are not reportable. */
   void (*stop)(void* ctx) PJ_NOEXCEPT;
-  /** [main-thread] Pause a running source. Returns false + error if unsupported. */
+  /** [main-thread] Advisory: a host may never call it and pauses ingest
+   *  host-side instead. Pause a running source. Returns false + error if
+   *  unsupported. */
   bool (*pause)(void* ctx, PJ_error_t* out_error) PJ_NOEXCEPT;
-  /** [main-thread] Resume a paused source. Returns false + error if unsupported. */
+  /** [main-thread] Advisory, as pause. Resume a paused source. Returns
+   *  false + error if unsupported. */
   bool (*resume)(void* ctx, PJ_error_t* out_error) PJ_NOEXCEPT;
-  /** [stream-thread] Called periodically by the host while running. */
+  /** [stream-thread] Called periodically by the host while running, from one
+   *  host thread (see the table above). Must not block. */
   bool (*poll)(void* ctx, PJ_error_t* out_error) PJ_NOEXCEPT;
 
-  /** [thread-safe] Return the plugin's current lifecycle state. */
+  /** [main-thread] Advisory: a host may never call it. Return the plugin's
+   *  current lifecycle state. Not [thread-safe]: the SDK bases keep the state
+   *  unsynchronised, so never call it while start/poll/stop runs. */
   PJ_data_source_state_t (*current_state)(void* ctx) PJ_NOEXCEPT;
 
   /**
@@ -634,8 +714,10 @@ typedef struct PJ_data_source_vtable_t {
 typedef struct {
   uint32_t struct_size; /**< sizeof(PJ_topic_subscription_v1_t) — for forward-compatible growth. */
   /**
-   * [main-thread] Set the full active-topic set. Entered on the host thread;
-   * the plugin marshals to its own stream thread. Returns false + populates
+   * [main-thread] Set the full active-topic set. Entered on the GUI thread,
+   * concurrently with poll (also once right after start, before the first
+   * poll); the plugin stores the set in a mutex-protected latest-wins slot
+   * that its stream thread drains, never a command queue. Returns false + populates
    * out_error on failure. Names the source does not recognize are ignored
    * (not an error).
    */

pj_base/include/pj_base/plugin_data_api.h

@@ -442,7 +442,9 @@ typedef struct {
  *                    object carries a SHARED error buffer whose pointer is
  *                    handed back through PJ_error_t*, so two threads must not
  *                    be inside slots of the SAME host object concurrently.
- *                    Serialise with your own mutex if you fan out.
+ *                    Serialise with your own mutex if you fan out. A
+ *                    DataSource narrows this further: only while a host
+ *                    start/poll/stop call is in progress (data_source_protocol.h).
  *   [thread-safe]    Any thread, concurrently.
  * ========================================================================== */
 
@@ -611,9 +613,10 @@ typedef struct {
  * for exactly this).
  *
  * The two ABI-appended tail slots (register_object_topic_on_dataset,
- * set_object_topic_retention) deliberately keep a conservative [main-thread]
- * tag: no host implements them yet — every known vtable leaves them NULL — so
- * there is no implementation to audit. Re-tag when the first one lands.
+ * set_object_topic_retention) follow the same rule. In the reference host the
+ * first takes the engine lock only for its dataset lookup; both otherwise go
+ * through ObjectStore's locks, like register_object_topic, and share its
+ * error buffer.
  *
  * read_series_arrow: caller zero-initialises both out structs. Host fills
  * them (allocates buffers, sets release callbacks). On success the caller
@@ -686,7 +689,7 @@ typedef struct PJ_toolbox_host_vtable_t {
       void* ctx, PJ_object_topic_handle_t topic, int64_t timestamp_ns, const uint8_t* data, uint64_t size,
       PJ_error_t* out_error) PJ_NOEXCEPT;
 
-  /* [main-thread] Register an object topic on an EXISTING dataset, addressed by
+  /* [stream-thread] Register an object topic on an EXISTING dataset, addressed by
    * its DatasetId, rather than under a data source the toolbox created. This is
    * the ANNOTATION path: a toolbox attaches objects (e.g. plot markers) to data
    * loaded by another source. Idempotent — if a topic with this name already
@@ -701,7 +704,7 @@ typedef struct PJ_toolbox_host_vtable_t {
       void* ctx, uint32_t dataset_id, PJ_string_view_t topic_name, PJ_string_view_t metadata_json,
       PJ_object_topic_handle_t* out_handle, PJ_error_t* out_error) PJ_NOEXCEPT;
 
-  /* [main-thread] Bound an object topic's retention to keep at most `max_entries`
+  /* [stream-thread] Bound an object topic's retention to keep at most `max_entries`
    * of the most recent snapshots (0 = unlimited). Generic capability: a producer
    * that republishes a whole set under one topic at a sentinel timestamp (e.g.
    * plot markers) sets max_entries=1 so superseded snapshots are evicted instead

pj_base/include/pj_base/sdk/data_source_patterns.hpp

@@ -10,8 +10,9 @@
  *  - **StreamSourceBase** — long-lived streaming sources.
  *    Override: extraCapabilities(), onStart(), onPoll(), onStop().
  *
- * Both manage the lifecycle state machine automatically — derived classes
- * only implement the domain-specific work.
+ * Both report lifecycle state automatically (advisory; see
+ * PJ_data_source_state_t) — derived classes only implement the
+ * domain-specific work.
  *
  * A StreamSourceBase with its own receive thread should NOT hand-roll the
  * thread -> onPoll() buffer: pj_plugins/sdk/streaming_source.hpp ships
@@ -55,6 +56,7 @@ namespace PJ {
  *
  * Manages the full lifecycle state machine. The derived class implements
  * importData() — all file I/O, parsing, and write-host calls happen there.
+ * It runs inside start(), which a host may call on a worker thread.
  *
  * ## Config convention
  *
@@ -121,18 +123,23 @@ class StreamSourceBase : public DataSourcePluginBase {
   /// Return additional capability flags (e.g. kCapabilityDirectIngest).
   virtual uint64_t extraCapabilities() const = 0;
 
-  /// Called from start(). Open connections, allocate resources.
+  /// Called from start(), possibly on the GUI thread. Open connections,
+  /// allocate resources. On failure, release what it acquired: onStop() is
+  /// not called and the host may destroy the instance without stop().
   virtual Status onStart() = 0;
 
   /// Called periodically by the host's polling thread.
   /// MUST NOT BLOCK — drain buffered data and return immediately.
   /// Do not call recv(), read(), or any syscall that may wait.
   /// If your source has a receive thread, swap-drain a buffer here.
-  /// Host methods (appendRecord, pushMessage) may only be called from this method.
+  /// [stream-thread] host methods (appendRecord, pushMessage, ...) may only be
+  /// called from onStart/onPoll/onStop, never from a thread you spawned;
+  /// [thread-safe] ones (reportMessage, requestStop, isStopRequested) from any.
   virtual Status onPoll() = 0;
 
-  /// Called from stop(). Close connections, free resources.
-  /// Must be idempotent.
+  /// Called from stop(), possibly on the polling thread rather than the one
+  /// that ran onStart(). Close connections, join your threads, free
+  /// resources. Must be idempotent.
   virtual void onStop() = 0;
 
   Status start() final;

pj_base/include/pj_base/sdk/toolbox_plugin_base.hpp

@@ -159,7 +159,8 @@ class ToolboxPluginBase {
     return PJ_borrowed_dialog_t{nullptr, nullptr};
   }
 
-  /// React to data appended to the datastore. Default is no-op.
+  /// React to data appended to the datastore. Default is no-op. Advisory: a
+  /// host may never call it, so do not depend on it for correctness.
   virtual void onDataChanged() {}
 
   /// Return a pointer to a static plugin-exposed extension for @p id, or

pj_base/include/pj_base/toolbox_protocol.h

@@ -89,7 +89,9 @@ typedef struct PJ_toolbox_runtime_host_vtable_t {
    * by the toolbox write host's create_data_source (== the dataset id).
    * On success fills `out_host` with a standard data-source runtime host
    * fat pointer: ensure_parser_binding / push_message on it behave exactly
-   * as they do for file/stream DataSource plugins. The context stays valid
+   * as they do for file/stream DataSource plugins. Its [stream-thread] slots
+   * may be called from any thread (typically a toolbox worker), one thread at
+   * a time per context; there is no start/poll/stop window. The context stays valid
    * until release_parser_ingest or host teardown.
    * Returns false (with out_error populated) if `data_source_id` is not a live
    * data source or parser ingest is not configured on this host; `out_host` is
@@ -124,6 +126,23 @@ typedef struct {
  * Typical lifecycle: create -> bind(registry) -> load_config (optional)
  *                    -> [user interacts] -> save_config -> destroy.
  * Every slot is PJ_NOEXCEPT.
+ *
+ * Threading (normative). Host -> plugin: every slot except
+ * get_plugin_extension is called on the GUI thread, one at a time per
+ * instance; get_plugin_extension may be called from any thread.
+ * on_data_changed is advisory: a host may never call it.
+ *
+ * Plugin -> host, by service:
+ *   pj.toolbox_runtime.v1      report_message, notify_data_changed and the
+ *                              parser-ingest create/release/discard slots:
+ *                              any thread, concurrently. The ingest context
+ *                              they return: one thread at a time.
+ *   pj.toolbox_write.v1        [stream-thread]: any thread, typically the
+ *                              toolbox's own worker, one thread at a time per
+ *                              host object (see plugin_data_api.h).
+ *   every other service        [main-thread] unless its slot says otherwise:
+ *                              call from inside a host->plugin call or a
+ *                              dialog callback, not from a plugin thread.
  */
 typedef struct PJ_toolbox_vtable_t {
   uint32_t protocol_version;
@@ -168,7 +187,9 @@ typedef struct PJ_toolbox_vtable_t {
   PJ_borrowed_dialog_t (*get_dialog)(void* ctx) PJ_NOEXCEPT;
 
   /** [main-thread] Notify the plugin that new records have been appended
-   *  to the datastore. */
+   *  to the datastore. Advisory: a host is not required to call it (the
+   *  reference host currently does not), so a plugin must not depend on it
+   *  for correctness; re-read the catalog when it needs fresh data. */
   void (*on_data_changed)(void* ctx) PJ_NOEXCEPT;
 
   /** [thread-safe] Query a plugin-exposed extension by reverse-DNS id.

pj_plugins/docs/ARCHITECTURE.md

@@ -218,7 +218,10 @@ previously-circulated pre-v4 design included):
   enforced at compile time. Trampolines catch and translate internally.
 - **Thread-class tags on every slot.** Every function-pointer field in
   the ABI headers carries a `[main-thread]` / `[stream-thread]` /
-  `[thread-safe]` comment. Host-side runtime checking is optional
+  `[thread-safe]` comment (DataSource `start`/`stop` state their rule in
+  words). Each family's plugin vtable also carries a normative per-slot
+  threading table (`data_source_protocol.h`, `toolbox_protocol.h`).
+  Host-side runtime checking is optional
   (reserved for a future `"pj.thread_check.v1"` service).
 - **Embedded-manifest plugin discovery.** Each DSO exports a
   family-specific protocol vtable with embedded metadata (`manifest_json`