services = defaultServices(lateStartupProbe).collect {
if (it instanceof MessageBrokerService) {
return messageBroker
}
@@ -92,14 +98,24 @@ class ManagerStartupTest extends Specification implements ManagerContainerTrait
stopContainer()
}
- def "MQTT ingress waits for event processing and reconciles monitored connection state"() {
+ def "public listeners open after all services start"() {
+ expect: "both endpoints stay closed until the ingress phase and accept connections after startup"
+ verifyAll {
+ !messageBroker.mqttPortOpenBeforeCamel
+ !lateStartupProbe.httpPortOpen
+ !lateStartupProbe.mqttPortOpen
+ canConnectToPort(lateStartupProbe.httpPort)
+ canConnectToPort(mqttPort)
+ }
+ }
+
+ def "MQTT connection monitoring reconciles state during startup"() {
given: "the startup probe ran under executor saturation with stale online values in storage"
def conditions = statusConditions()
MQTT_IOClient client
- expect: "the MQTT port stayed closed until Camel started and both stored statuses are offline"
+ expect: "startup completed without consumer waits and both stored statuses are offline"
verifyAll {
- !messageBroker.mqttPortOpenBeforeCamel
mqttBroker.startupConsumerTimeouts == 0
connectionStatuses(storage, mqttBroker.monitoredAssetId) == [false, false]
}
@@ -161,7 +177,7 @@ class ManagerStartupTest extends Specification implements ManagerContainerTrait
return conditions
}
- private static boolean canConnectToMqttPort(int port) {
+ private static boolean canConnectToPort(int port) {
try {
new Socket().withCloseable {
it.connect(new InetSocketAddress("127.0.0.1", port), 1000)
@@ -258,7 +274,7 @@ class ManagerStartupTest extends Specification implements ManagerContainerTrait
submittedAssetId = storage.merge(asset).id
withSaturatedExecutor(container) {
- mqttPortOpenBeforeCamel = canConnectToMqttPort(mqttPort)
+ mqttPortOpenBeforeCamel = canConnectToPort(mqttPort)
int timeoutsBeforeSubmission = direct.startupConsumerTimeouts
container.getService(AssetProcessingService.class)
.sendAttributeEvent(new AttributeEvent(submittedAssetId, "submitted", true))
@@ -272,6 +288,35 @@ class ManagerStartupTest extends Specification implements ManagerContainerTrait
}
}
+ private static class LateStartupProbe implements ContainerService {
+ final int httpPort
+ private final int mqttPort
+ boolean httpPortOpen
+ boolean mqttPortOpen
+
+ LateStartupProbe(int httpPort, int mqttPort) {
+ this.httpPort = httpPort
+ this.mqttPort = mqttPort
+ }
+
+ @Override
+ int getPriority() {
+ Integer.MAX_VALUE
+ }
+
+ @Override
+ void init(Container container) {}
+
+ @Override
+ void start(Container container) {
+ httpPortOpen = canConnectToPort(httpPort)
+ mqttPortOpen = canConnectToPort(mqttPort)
+ }
+
+ @Override
+ void stop(Container container) {}
+ }
+
private static class StartupDirectComponent extends DirectComponent {
volatile Thread startupThread
int startupConsumerTimeouts
From 4f1bc5d260944ce1d2fa11db2a52fee8c9011eb7 Mon Sep 17 00:00:00 2001
From: Eric Bariaux <375613+ebariaux@users.noreply.github.com>
Date: Thu, 19 Nov 2026 09:51:13 +0100
Subject: [PATCH 3/4] Add an internal producer startup phase
Refs #3587
---
.../org/openremote/container/Container.java | 5 ++
.../openremote/model/ContainerService.java | 54 ++++++++++++-------
2 files changed, 41 insertions(+), 18 deletions(-)
diff --git a/container/src/main/java/org/openremote/container/Container.java b/container/src/main/java/org/openremote/container/Container.java
index 891e347d5d..4bbbea0769 100644
--- a/container/src/main/java/org/openremote/container/Container.java
+++ b/container/src/main/java/org/openremote/container/Container.java
@@ -197,6 +197,11 @@ public synchronized void start() throws Exception {
service.start(Container.this);
}
+ LOG.log(INFO, "Starting runtime event producers");
+ for (ContainerService service : getServices()) {
+ service.startProducers(this);
+ }
+
LOG.log(INFO, "Starting runtime ingress");
for (ContainerService service : getServices()) {
service.startIngress(this);
diff --git a/model/src/main/java/org/openremote/model/ContainerService.java b/model/src/main/java/org/openremote/model/ContainerService.java
index 72902e571b..f5482b7ee0 100644
--- a/model/src/main/java/org/openremote/model/ContainerService.java
+++ b/model/src/main/java/org/openremote/model/ContainerService.java
@@ -26,21 +26,38 @@
* Container} using the auto service discovery mechanism. If the container is started with an
* explicit list of services then the insertion order is used.
*
- * Service startup lifecycle:
+ *
The container runs each startup phase in registry order and completes it for all services
+ * before moving to the next.
*
- *
- * - {@link #init} in registry insertion order
- *
- {@link #start} in registry insertion order
- *
- {@link #startIngress} in registry insertion order
- *
+ *
+ * Service startup phases
+ *
+ *
+ * | Phase |
+ * Responsibility |
+ *
+ *
+ *
+ *
+ * | {@link #init} |
+ * Resolve dependencies and register routes, handlers, and subscriptions. |
+ *
+ *
+ * | {@link #start} |
+ * Start internal services and event consumers, including the message broker. |
+ *
+ *
+ * | {@link #startProducers} |
+ * Enable internal event producers and complete startup writes required before ingress. |
+ *
+ *
+ * | {@link #startIngress} |
+ * Open public listeners and accept external traffic. |
+ *
+ *
+ *
*
- * The container completes each startup phase for all services before moving to the next.
- *
- *
Service shutdown lifecycle:
- *
- *
- * - {@link #stop} in reverse registry order
- *
+ * On shutdown, the container calls {@link #stop} in reverse registry order.
*/
@TsIgnore
public interface ContainerService {
@@ -66,16 +83,17 @@ default int getPriority() {
void init(Container container) throws Exception;
/**
- * After initialization, services are started in the order they have been added to the container
- * (if container started with explicit list of services) otherwise they are started in order of
- * {@link #getPriority}.
+ * Starts internal services and event consumers after every service has completed {@link #init}.
*/
void start(Container container) throws Exception;
/**
- * Opens public listeners after every service has completed {@link #start}. Services should prepare
- * their handlers and event consumers during {@link #init} and {@link #start}.
+ * Starts internal event producers after every service has completed {@link #start}. Complete
+ * writes required for public traffic before returning from this hook.
*/
+ default void startProducers(Container container) throws Exception {}
+
+ /** Opens public listeners after every service has completed {@link #startProducers}. */
default void startIngress(Container container) throws Exception {}
/**
From eb6e1c30a8d4302d36af8ecb1daeba6811729721 Mon Sep 17 00:00:00 2001
From: Eric Bariaux <375613+ebariaux@users.noreply.github.com>
Date: Thu, 19 Nov 2026 09:51:24 +0100
Subject: [PATCH 4/4] Fail fast when attribute event processing is unavailable
Run built-in startup producers after Camel is ready and cover the ordering with integration tests.
Refs #3587
---
.../message/MessageBrokerService.java | 13 +-
.../manager/asset/AssetProcessingService.java | 56 ++++++--
.../ApplyPredictedDataPointsService.java | 3 +
.../mqtt/ConnectionMonitorHandler.java | 116 +++++++++------
.../manager/mqtt/MQTTBrokerService.java | 9 ++
.../openremote/manager/mqtt/MQTTHandler.java | 3 +
.../manager/rules/RulesService.java | 42 ++++--
.../test/system/ManagerStartupTest.groovy | 134 ++++++++++++++----
8 files changed, 273 insertions(+), 103 deletions(-)
diff --git a/container/src/main/java/org/openremote/container/message/MessageBrokerService.java b/container/src/main/java/org/openremote/container/message/MessageBrokerService.java
index e92b751893..940555a06c 100644
--- a/container/src/main/java/org/openremote/container/message/MessageBrokerService.java
+++ b/container/src/main/java/org/openremote/container/message/MessageBrokerService.java
@@ -53,7 +53,7 @@ public class MessageBrokerService implements ContainerService {
protected ProducerTemplate producerTemplate = context.createProducerTemplate();
protected FluentProducerTemplate fluentProducerTemplate = context.createFluentProducerTemplate();
- private boolean isStarted = false;
+ private volatile boolean isStarted;
@Override
public int getPriority() {
@@ -151,12 +151,11 @@ public void start(Container container) throws Exception {
@Override
public void stop(Container container) throws Exception {
- try {
- if (context != null) {
- context.stop();
- }
- } finally {
- isStarted = false;
+ // We want to indicate service is stopped before actually stopping it so callers will not be
+ // able to push new events during shutdown
+ isStarted = false;
+ if (context != null) {
+ context.stop();
}
}
diff --git a/manager/src/main/java/org/openremote/manager/asset/AssetProcessingService.java b/manager/src/main/java/org/openremote/manager/asset/AssetProcessingService.java
index 089ca47276..d26dfd9ad7 100644
--- a/manager/src/main/java/org/openremote/manager/asset/AssetProcessingService.java
+++ b/manager/src/main/java/org/openremote/manager/asset/AssetProcessingService.java
@@ -31,6 +31,8 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.RejectedExecutionException;
import org.apache.camel.Exchange;
+import org.apache.camel.FluentProducerTemplate;
+import org.apache.camel.ServiceStatus;
import org.apache.camel.builder.RouteBuilder;
import org.apache.camel.builder.RouteConfigurationBuilder;
import org.openremote.container.message.MessageBrokerService;
@@ -91,6 +93,9 @@ public class AssetProcessingService extends RouteBuilder implements ContainerSer
public static final String ATTRIBUTE_EVENT_ROUTE_CONFIG_ID = "attributeEvent";
public static final int PRIORITY = AssetStorageService.PRIORITY + 1000;
public static final String ATTRIBUTE_EVENT_PROCESSOR = "direct://AttributeEventProcessor";
+ private static final String ATTRIBUTE_EVENT_ROUTE_ID = "AttributeEvent-Processor";
+ private static final String ATTRIBUTE_EVENT_SEND_ENDPOINT =
+ ATTRIBUTE_EVENT_PROCESSOR + "?block=false";
private static final System.Logger LOG = System.getLogger(AssetProcessingService.class.getName());
protected final List eventInterceptors = new ArrayList<>();
protected TimerService timerService;
@@ -329,7 +334,7 @@ public void configure() throws Exception {
// All user authorisation checks MUST have been carried out before events reach this queue
from(ATTRIBUTE_EVENT_PROCESSOR)
- .routeId("AttributeEvent-Processor")
+ .routeId(ATTRIBUTE_EVENT_ROUTE_ID)
.routeConfigurationId(ATTRIBUTE_EVENT_ROUTE_CONFIG_ID)
.threads()
.executorService(executorService)
@@ -382,24 +387,59 @@ public void addEventInterceptor(AttributeEventInterceptor eventInterceptor) {
eventInterceptors.sort(Comparator.comparingInt(AttributeEventInterceptor::getPriority));
}
- /** Send internal attribute change events into the {@link #ATTRIBUTE_EVENT_PROCESSOR}. */
+ /**
+ * Send an internal attribute change event for asynchronous processing.
+ *
+ * @throws IllegalStateException if attribute event processing is unavailable
+ */
public void sendAttributeEvent(AttributeEvent attributeEvent) {
sendAttributeEvent(attributeEvent, null);
}
- /** Send internal attribute change events into the {@link #ATTRIBUTE_EVENT_PROCESSOR}. */
+ /**
+ * Send an internal attribute change event for asynchronous processing.
+ *
+ * @throws IllegalStateException if attribute event processing is unavailable
+ */
public void sendAttributeEvent(AttributeEvent attributeEvent, String source) {
- attributeEvent.setSource(source);
+ prepareAttributeEvent(attributeEvent, source).asyncSend();
+ }
+
+ /**
+ * Process an internal attribute change before continuing startup work that depends on the value.
+ *
+ * @throws IllegalStateException if attribute event processing is unavailable
+ * @throws AssetProcessingException if the attribute event cannot be processed
+ */
+ public void sendAttributeEventAndWait(AttributeEvent attributeEvent, String source) {
+ Object result = prepareAttributeEvent(attributeEvent, source).request();
+ if (result instanceof RuntimeException failure) {
+ throw failure;
+ }
+ }
+
+ private FluentProducerTemplate prepareAttributeEvent(
+ AttributeEvent attributeEvent, String source) {
+ if (messageBrokerService == null
+ || !messageBrokerService.isStarted()
+ || messageBrokerService
+ .getContext()
+ .getRouteController()
+ .getRouteStatus(ATTRIBUTE_EVENT_ROUTE_ID)
+ != ServiceStatus.Started) {
+ throw new IllegalStateException("Attribute event processor is not ready");
+ }
- // Set event source time if not already set
+ attributeEvent.setSource(source);
if (attributeEvent.getTimestamp() <= 0) {
attributeEvent.setTimestamp(timerService.getCurrentTimeMillis());
}
- messageBrokerService
+
+ // Fail promptly if the consumer stops between the readiness check and the send.
+ return messageBrokerService
.getFluentProducerTemplate()
.withBody(attributeEvent)
- .to(ATTRIBUTE_EVENT_PROCESSOR)
- .asyncSend();
+ .to(ATTRIBUTE_EVENT_SEND_ENDPOINT);
}
/**
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/ApplyPredictedDataPointsService.java b/manager/src/main/java/org/openremote/manager/datapoint/ApplyPredictedDataPointsService.java
index 657bbd0fd6..8c2307ca1a 100644
--- a/manager/src/main/java/org/openremote/manager/datapoint/ApplyPredictedDataPointsService.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/ApplyPredictedDataPointsService.java
@@ -89,7 +89,10 @@ public void start(Container container) throws Exception {
clientEventService.addSubscription(
AssetPredictedDatapointEvent.class, null, this::onPredictedDatapointsChanged);
clientEventService.addSubscription(AttributeEvent.class, null, this::onAttributeEvent);
+ }
+ @Override
+ public void startProducers(Container container) {
loadInitialAttributes();
scheduleNext();
}
diff --git a/manager/src/main/java/org/openremote/manager/mqtt/ConnectionMonitorHandler.java b/manager/src/main/java/org/openremote/manager/mqtt/ConnectionMonitorHandler.java
index dabf016997..28c6617b92 100644
--- a/manager/src/main/java/org/openremote/manager/mqtt/ConnectionMonitorHandler.java
+++ b/manager/src/main/java/org/openremote/manager/mqtt/ConnectionMonitorHandler.java
@@ -29,17 +29,17 @@
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutorService;
+import java.util.function.BiConsumer;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import org.apache.activemq.artemis.core.config.Configuration;
import org.apache.activemq.artemis.spi.core.protocol.RemotingConnection;
import org.apache.camel.builder.RouteBuilder;
-import org.openremote.container.message.MessageBrokerService;
import org.openremote.container.persistence.PersistenceService;
import org.openremote.container.security.AuthContext;
import org.openremote.container.security.IdentityProvider;
+import org.openremote.manager.asset.AssetProcessingException;
import org.openremote.manager.asset.AssetProcessingService;
import org.openremote.manager.asset.AssetStorageService;
import org.openremote.manager.gateway.GatewayService;
@@ -71,7 +71,6 @@ public class ConnectionMonitorHandler extends MQTTHandler {
protected static final Logger LOG = SyslogCategory.getLogger(API, ConnectionMonitorHandler.class);
protected MQTTBrokerService mqttBrokerService;
- protected ExecutorService executorService;
protected AssetStorageService assetStorageService;
protected AssetProcessingService assetProcessingService;
protected GatewayService gatewayService;
@@ -82,13 +81,11 @@ public class ConnectionMonitorHandler extends MQTTHandler {
@Override
public void init(Container container, Configuration serverConfiguration) throws Exception {
super.init(container, serverConfiguration);
- executorService = container.getExecutor();
mqttBrokerService = container.getService(MQTTBrokerService.class);
assetStorageService = container.getService(AssetStorageService.class);
assetProcessingService = container.getService(AssetProcessingService.class);
gatewayService = container.getService(GatewayService.class);
persistenceService = container.getService(PersistenceService.class);
- MessageBrokerService messageBrokerService = container.getService(MessageBrokerService.class);
messageBrokerService
.getContext()
.addRoutes(
@@ -149,37 +146,48 @@ public void configure() throws Exception {
@Override
public void start(Container container) throws Exception {
- // Don't do super start as don't need to publish or subscribe
- // super.start(container);
-
- // Don't block start
- executorService.submit(
- () -> {
- // Get all assets that have attributes with user connected meta and initialise them
- List> assets =
- assetStorageService.findAll(
- new AssetQuery()
- .attributes(
- new AttributePredicate()
- .meta(new NameValuePredicate(USER_CONNECTED, null))));
-
- Map>> realmAttributeMap =
- assets.stream().collect(Collectors.groupingBy(Asset::getRealm));
-
- realmAttributeMap.forEach(
- (realm, realmAssets) -> {
- List>> assetIdsAttrs =
- realmAssets.stream()
- .flatMap(
- asset ->
- asset.getAttributes().stream()
- .filter(ConnectionMonitorHandler::attributeMatches)
- .map(
- attr ->
- new Pair>(asset.getId(), attr)))
- .toList();
-
- addSessionAttributes(realm, assetIdsAttrs);
+ // This handler has no internal MQTT session to start.
+ }
+
+ @Override
+ public void startProducers(Container container) {
+ // Get all assets that have attributes with user connected meta and initialise them
+ List> assets =
+ assetStorageService.findAll(
+ new AssetQuery()
+ .attributes(
+ new AttributePredicate().meta(new NameValuePredicate(USER_CONNECTED, null))));
+
+ Map>> realmAttributeMap =
+ assets.stream().collect(Collectors.groupingBy(Asset::getRealm));
+
+ realmAttributeMap.forEach(
+ (realm, realmAssets) -> {
+ List>> assetIdsAttrs =
+ realmAssets.stream()
+ .flatMap(
+ asset ->
+ asset.getAttributes().stream()
+ .filter(ConnectionMonitorHandler::attributeMatches)
+ .map(attr -> new Pair>(asset.getId(), attr)))
+ .toList();
+
+ forEachSessionAttribute(
+ realm,
+ assetIdsAttrs,
+ (userID, attributeRef) -> {
+ registerSessionAttribute(userID, attributeRef);
+ boolean connected = !mqttBrokerService.getUserConnections(userID).isEmpty();
+ // Finish the initial write before the public acceptor can receive a connection.
+ try {
+ assetProcessingService.sendAttributeEventAndWait(
+ new AttributeEvent(attributeRef, connected), getClass().getSimpleName());
+ } catch (AssetProcessingException e) {
+ LOG.log(
+ Level.WARNING,
+ "Could not initialize connection status for " + attributeRef,
+ e);
+ }
});
});
}
@@ -265,6 +273,13 @@ public void onUnsubscribe(RemotingConnection connection, Topic topic) {}
protected void addSessionAttributes(
String realm, List>> assetIdsAttrs) {
+ forEachSessionAttribute(realm, assetIdsAttrs, this::addSessionAttribute);
+ }
+
+ private void forEachSessionAttribute(
+ String realm,
+ List>> assetIdsAttrs,
+ BiConsumer action) {
LOG.finest(
"Adding '"
+ assetIdsAttrs.size()
@@ -275,12 +290,8 @@ protected void addSessionAttributes(
assetIdsAttrs.stream()
.map(assetIdAttr -> assetIdAttr.getValue().getMetaValue(USER_CONNECTED).orElse(null))
.filter(Objects::nonNull)
+ .map(ConnectionMonitorHandler::toServiceAccountUsername)
.distinct()
- .map(
- username ->
- username.startsWith(User.SERVICE_ACCOUNT_PREFIX)
- ? username
- : User.SERVICE_ACCOUNT_PREFIX + username)
.toList();
// Convert usernames to userIds
@@ -294,7 +305,7 @@ protected void addSessionAttributes(
.ifPresent(
username -> {
String userID =
- userIds.get(usernames.indexOf(User.SERVICE_ACCOUNT_PREFIX + username));
+ userIds.get(usernames.indexOf(toServiceAccountUsername(username)));
if (userID == null) {
LOG.warning(
@@ -303,22 +314,32 @@ protected void addSessionAttributes(
+ ", username="
+ username);
} else {
- addSessionAttribute(
+ action.accept(
userID,
new AttributeRef(assetIdAttr.key, assetIdAttr.getValue().getName()));
}
}));
}
+ private static String toServiceAccountUsername(String username) {
+ return username.startsWith(User.SERVICE_ACCOUNT_PREFIX)
+ ? username
+ : User.SERVICE_ACCOUNT_PREFIX + username;
+ }
+
protected void addSessionAttribute(String userID, AttributeRef attributeRef) {
LOG.finest("Adding userID '" + userID + "' monitoring for attribute: " + attributeRef);
updateUserConnectedStatus(
userID,
Collections.singletonList(attributeRef),
!mqttBrokerService.getUserConnections(userID).isEmpty());
- Set refs =
- userIDAttributeRefs.computeIfAbsent(userID, ID -> ConcurrentHashMap.newKeySet());
- refs.add(attributeRef);
+ registerSessionAttribute(userID, attributeRef);
+ }
+
+ private void registerSessionAttribute(String userID, AttributeRef attributeRef) {
+ userIDAttributeRefs
+ .computeIfAbsent(userID, id -> ConcurrentHashMap.newKeySet())
+ .add(attributeRef);
}
protected void removeSessionAttribute(String userID, AttributeRef attributeRef) {
@@ -334,6 +355,9 @@ protected void removeSessionAttribute(String userID, AttributeRef attributeRef)
protected void updateUserConnectedStatus(
String userID, Collection attributeRefs, boolean connected) {
+ if (!messageBrokerService.isStarted()) {
+ return;
+ }
// Only update statuses if this is the only connection
Set connections = mqttBrokerService.getUserConnections(userID);
diff --git a/manager/src/main/java/org/openremote/manager/mqtt/MQTTBrokerService.java b/manager/src/main/java/org/openremote/manager/mqtt/MQTTBrokerService.java
index b1265b5609..c7809c9122 100644
--- a/manager/src/main/java/org/openremote/manager/mqtt/MQTTBrokerService.java
+++ b/manager/src/main/java/org/openremote/manager/mqtt/MQTTBrokerService.java
@@ -375,6 +375,15 @@ public void start(Container container) throws Exception {
}
}
+ @Override
+ public void startProducers(Container container) throws Exception {
+ if (active) {
+ for (MQTTHandler handler : customHandlers) {
+ handler.startProducers(container);
+ }
+ }
+ }
+
@Override
public void startIngress(Container container) throws Exception {
if (active) {
diff --git a/manager/src/main/java/org/openremote/manager/mqtt/MQTTHandler.java b/manager/src/main/java/org/openremote/manager/mqtt/MQTTHandler.java
index 33702615a6..e3f6f9d8c6 100644
--- a/manager/src/main/java/org/openremote/manager/mqtt/MQTTHandler.java
+++ b/manager/src/main/java/org/openremote/manager/mqtt/MQTTHandler.java
@@ -123,6 +123,9 @@ public void start(Container container) throws Exception {
}
}
+ /** Starts work that needs running event consumers before the public MQTT acceptor opens. */
+ public void startProducers(Container container) throws Exception {}
+
/** Called when the system stops to allow for any cleanup. */
public void stop() throws Exception {
if (clientSession != null) {
diff --git a/manager/src/main/java/org/openremote/manager/rules/RulesService.java b/manager/src/main/java/org/openremote/manager/rules/RulesService.java
index 3710466db4..17aace28b5 100644
--- a/manager/src/main/java/org/openremote/manager/rules/RulesService.java
+++ b/manager/src/main/java/org/openremote/manager/rules/RulesService.java
@@ -154,7 +154,8 @@ public class RulesService extends RouteBuilder implements ContainerService {
protected long quickFireMillis;
protected boolean groovyRulesExecutionEnabled;
protected boolean initDone;
- protected boolean startDone;
+ private boolean producerPhaseReached;
+ protected volatile boolean startDone;
protected io.micrometer.core.instrument.Timer rulesFiringTimer;
@Override
@@ -164,6 +165,7 @@ public int getPriority() {
@Override
public void init(Container container) throws Exception {
+ producerPhaseReached = false;
executorService = container.getExecutor();
scheduledExecutorService = container.getScheduledExecutor();
timerService = container.getService(TimerService.class);
@@ -359,8 +361,21 @@ AssetRuleset.class, new RulesetQuery().setEnabledOnly(true).setFullyPopulate(tru
});
});
- // Start the engines
+ // Restart the engines immediately when only the rules service is restarted.
+ if (producerPhaseReached && container.getService(MessageBrokerService.class).isStarted()) {
+ startEngines();
+ }
+ }
+
+ @Override
+ public void startProducers(Container container) {
+ if (!startDone) {
+ startEngines();
+ }
+ producerPhaseReached = true;
+ }
+ private void startEngines() {
synchronized (ENGINE_LOCK) {
RulesEngine> globalRulesEngine = globalEngine.get();
if (globalRulesEngine != null) {
@@ -382,6 +397,7 @@ AssetRuleset.class, new RulesetQuery().setEnabledOnly(true).setFullyPopulate(tru
@Override
public void stop(Container container) throws Exception {
+ startDone = false;
for (GeofenceAssetAdapter geofenceAssetAdapter : geofenceAssetAdapters) {
try {
geofenceAssetAdapter.stop(container);
@@ -596,16 +612,11 @@ protected void processRulesetChange(Ruleset ruleset, PersistenceEvent.Cause caus
undeployAssetRuleset((AssetRuleset) ruleset);
}
} else {
- if (ruleset instanceof GlobalRuleset) {
-
- RulesEngine engine = deployGlobalRuleset((GlobalRuleset) ruleset);
- engine.start();
-
- } else if (ruleset instanceof RealmRuleset) {
-
- RulesEngine engine = deployRealmRuleset((RealmRuleset) ruleset);
- engine.start();
-
+ RulesEngine> engine;
+ if (ruleset instanceof GlobalRuleset globalRuleset) {
+ engine = deployGlobalRuleset(globalRuleset);
+ } else if (ruleset instanceof RealmRuleset realmRuleset) {
+ engine = deployRealmRuleset(realmRuleset);
} else if (ruleset instanceof AssetRuleset) {
// Must reload from the database, the ruleset might not be completely hydrated on CREATE or
@@ -615,7 +626,12 @@ protected void processRulesetChange(Ruleset ruleset, PersistenceEvent.Cause caus
undeployAssetRuleset((AssetRuleset) ruleset);
return;
}
- RulesEngine engine = deployAssetRuleset(assetRuleset);
+ engine = deployAssetRuleset(assetRuleset);
+ } else {
+ return;
+ }
+
+ if (startDone) {
engine.start();
}
}
diff --git a/test/src/test/groovy/org/openremote/test/system/ManagerStartupTest.groovy b/test/src/test/groovy/org/openremote/test/system/ManagerStartupTest.groovy
index 8782d99102..46d3b37640 100644
--- a/test/src/test/groovy/org/openremote/test/system/ManagerStartupTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/system/ManagerStartupTest.groovy
@@ -22,9 +22,12 @@ import org.apache.camel.component.direct.DirectComponent
import org.apache.camel.component.direct.DirectConsumer
import org.openremote.agent.protocol.mqtt.MQTT_IOClient
import org.openremote.container.message.MessageBrokerService
+import org.openremote.container.timer.TimerService
import org.openremote.manager.asset.AssetProcessingService
import org.openremote.manager.asset.AssetStorageService
+import org.openremote.manager.datapoint.AssetPredictedDatapointService
import org.openremote.manager.mqtt.MQTTBrokerService
+import org.openremote.manager.rules.RulesService
import org.openremote.manager.setup.SetupService
import org.openremote.model.Container
import org.openremote.model.ContainerService
@@ -34,6 +37,7 @@ import org.openremote.model.attribute.Attribute
import org.openremote.model.attribute.AttributeEvent
import org.openremote.model.attribute.MetaItem
import org.openremote.model.auth.UsernamePassword
+import org.openremote.model.datapoint.ValueDatapoint
import org.openremote.model.util.UniqueIdentifierGenerator
import org.openremote.setup.integration.KeycloakTestSetup
import org.openremote.test.ManagerContainerTrait
@@ -51,6 +55,9 @@ import static org.openremote.container.web.WebService.OR_WEBSERVER_LISTEN_PORT
import static org.openremote.manager.asset.AssetProcessingService.ATTRIBUTE_EVENT_PROCESSOR
import static org.openremote.manager.mqtt.MQTTBrokerService.MQTT_SERVER_LISTEN_HOST
import static org.openremote.manager.mqtt.MQTTBrokerService.MQTT_SERVER_LISTEN_PORT
+import static org.openremote.model.security.User.SERVICE_ACCOUNT_PREFIX
+import static org.openremote.model.value.MetaItemType.APPLY_PREDICTED_DATA_POINTS
+import static org.openremote.model.value.MetaItemType.HAS_PREDICTED_DATA_POINTS
import static org.openremote.model.value.MetaItemType.USER_CONNECTED
import static org.openremote.model.value.ValueType.BOOLEAN
@@ -80,7 +87,7 @@ class ManagerStartupTest extends Specification implements ManagerContainerTrait
(OR_EXECUTOR_THREADS_MAX): "1"
]
lateStartupProbe = new LateStartupProbe(config[OR_WEBSERVER_LISTEN_PORT].toInteger(), mqttPort)
- Iterable services = defaultServices(lateStartupProbe).collect {
+ List services = defaultServices(lateStartupProbe).collect {
if (it instanceof MessageBrokerService) {
return messageBroker
}
@@ -89,6 +96,9 @@ class ManagerStartupTest extends Specification implements ManagerContainerTrait
}
return it
}
+ // Exercise a custom service order in which Camel is already running when rules are prepared.
+ services.remove(messageBroker)
+ services.add(services.findIndexOf { it instanceof RulesService }, messageBroker)
def manager = startContainer(config, services)
storage = manager.getService(AssetStorageService.class)
@@ -98,12 +108,20 @@ class ManagerStartupTest extends Specification implements ManagerContainerTrait
stopContainer()
}
- def "public listeners open after all services start"() {
- expect: "both endpoints stay closed until the ingress phase and accept connections after startup"
+ def "internal producers start after consumers and before public listeners"() {
+ expect: "rules wait for the producer phase even when Camel starts first"
+ verifyAll {
+ !lateStartupProbe.rulesStartedDuringStart
+ lateStartupProbe.rulesStartedDuringProducerPhase
+ }
+
+ and: "both endpoints stay closed through the producer phase and accept connections after startup"
verifyAll {
!messageBroker.mqttPortOpenBeforeCamel
- !lateStartupProbe.httpPortOpen
- !lateStartupProbe.mqttPortOpen
+ !lateStartupProbe.httpPortOpenDuringStart
+ !lateStartupProbe.mqttPortOpenDuringStart
+ !lateStartupProbe.httpPortOpenDuringProducerPhase
+ !lateStartupProbe.mqttPortOpenDuringProducerPhase
canConnectToPort(lateStartupProbe.httpPort)
canConnectToPort(mqttPort)
}
@@ -122,7 +140,7 @@ class ManagerStartupTest extends Specification implements ManagerContainerTrait
when: "the monitored client connects after startup"
client = new MQTT_IOClient(UniqueIdentifierGenerator.generateId(), "127.0.0.1", mqttPort,
- false, true, mqttBroker.credentials, null, null)
+ false, true, mqttBroker.credentials, null, null)
client.connect()
then: "both monitored attributes report the client online"
@@ -143,17 +161,51 @@ class ManagerStartupTest extends Specification implements ManagerContainerTrait
client?.disconnect()
}
- def "a Java attribute submission during startup is processed after Camel starts"() {
- expect: "the submission returned without waiting for the consumer and processing started after Camel"
- def startupConsumerTimeouts = messageBroker.submissionConsumerTimeouts
- assert startupConsumerTimeouts == 0 :
- "Submitted attribute after startup was ${attributeValue(storage, messageBroker.submittedAssetId, "submitted")}"
- messageBroker.submittedValueBeforeCamel == false
+ def "a Java attribute submission fails fast before Camel starts and can be retried"() {
+ given:
+ def assetId = messageBroker.submittedAssetId
+ def conditions = statusConditions()
+
+ expect: "the early submission was rejected without waiting for a consumer"
+ verifyAll {
+ messageBroker.submissionConsumerTimeouts == 0
+ messageBroker.submissionFailure?.message == "Attribute event processor is not ready"
+ attributeValue(storage, assetId, "submitted") == false
+ }
+
+ when: "the caller retries after Camel starts"
+ container.getService(AssetProcessingService.class)
+ .sendAttributeEvent(new AttributeEvent(assetId, "submitted", true))
+
+ then: "the retried event is applied"
+ conditions.eventually {
+ assert attributeValue(storage, assetId, "submitted") == true
+ }
+ }
- and: "the event is applied to the asset"
+ def "a Java attribute submission fails fast while only its route is stopped"() {
+ given:
+ def routeController = messageBroker.context.routeController
+ routeController.stopRoute("AttributeEvent-Processor")
+
+ when:
+ container.getService(AssetProcessingService.class)
+ .sendAttributeEvent(new AttributeEvent(messageBroker.submittedAssetId, "submitted", true))
+
+ then:
+ IllegalStateException failure = thrown()
+ failure.message == "Attribute event processor is not ready"
+ messageBroker.isStarted()
+
+ cleanup:
+ routeController?.startRoute("AttributeEvent-Processor")
+ }
+
+ def "a prediction already due during startup is applied once Camel is ready"() {
+ expect:
def conditions = statusConditions()
conditions.eventually {
- assert attributeValue(storage, messageBroker.submittedAssetId, "submitted") == true
+ assert attributeValue(storage, mqttBroker.monitoredAssetId, "startupPrediction") == true
}
}
@@ -167,8 +219,10 @@ class ManagerStartupTest extends Specification implements ManagerContainerTrait
private static List connectionStatuses(AssetStorageService storage, String assetId) {
def asset = storage.find(assetId)
- [asset.getAttribute("firstConnected").get().value.orElse(null),
- asset.getAttribute("secondConnected").get().value.orElse(null)]
+ [
+ asset.getAttribute("firstConnected").get().value.orElse(null),
+ asset.getAttribute("secondConnected").get().value.orElse(null)
+ ]
}
private static PollingConditions statusConditions() {
@@ -195,7 +249,7 @@ class ManagerStartupTest extends Specification implements ManagerContainerTrait
try {
new PollingConditions(timeout: 10).eventually {
- if (workerOccupied.count > 0) {
+ if (workerOccupied.count> 0) {
container.executor.execute {
// Retry if another startup task occupies the worker and this submission runs inline.
if (Thread.currentThread() != startupThread) {
@@ -231,11 +285,18 @@ class ManagerStartupTest extends Specification implements ManagerContainerTrait
def realm = keycloak.realmBuilding.name
def user = keycloak.serviceUser
credentials = new UsernamePassword("${realm}:${user.username}".toString(), user.secret)
- def asset = new ThingAsset("MQTT connection status").setRealm(realm).addAttributes(
- new Attribute<>("firstConnected", BOOLEAN, true).addMeta(new MetaItem<>(USER_CONNECTED, user.username)),
- new Attribute<>("secondConnected", BOOLEAN, true).addMeta(new MetaItem<>(USER_CONNECTED, user.username))
+ long now = container.getService(TimerService.class).getCurrentTimeMillis()
+ def asset = new ThingAsset("Startup attributes").setRealm(realm).addAttributes(
+ new Attribute<>("firstConnected", BOOLEAN, true).addMeta(new MetaItem<>(USER_CONNECTED, user.username)),
+ new Attribute<>("secondConnected", BOOLEAN, true)
+ .addMeta(new MetaItem<>(USER_CONNECTED, SERVICE_ACCOUNT_PREFIX + user.username)),
+ new Attribute<>("startupPrediction", BOOLEAN, false).setTimestamp(now - 2000)
+ .addOrReplaceMeta(new MetaItem<>(HAS_PREDICTED_DATA_POINTS, true),
+ new MetaItem<>(APPLY_PREDICTED_DATA_POINTS, true))
)
monitoredAssetId = container.getService(AssetStorageService.class).merge(asset).id
+ container.getService(AssetPredictedDatapointService.class)
+ .updateValues(monitoredAssetId, "startupPrediction", [new ValueDatapoint<>(now - 1000, true)])
int timeoutsBeforeMqtt = direct.startupConsumerTimeouts
withSaturatedExecutor(container) {
@@ -250,7 +311,7 @@ class ManagerStartupTest extends Specification implements ManagerContainerTrait
private final int mqttPort
boolean mqttPortOpenBeforeCamel
String submittedAssetId
- Boolean submittedValueBeforeCamel
+ IllegalStateException submissionFailure
int submissionConsumerTimeouts
StartupMessageBroker(int mqttPort) {
@@ -270,17 +331,20 @@ class ManagerStartupTest extends Specification implements ManagerContainerTrait
def keycloak = keycloakSetup(container)
def storage = container.getService(AssetStorageService.class)
def asset = new ThingAsset("Startup attribute event").setRealm(keycloak.realmBuilding.name)
- .addAttributes(new Attribute<>("submitted", BOOLEAN, false))
+ .addAttributes(new Attribute<>("submitted", BOOLEAN, false))
submittedAssetId = storage.merge(asset).id
withSaturatedExecutor(container) {
mqttPortOpenBeforeCamel = canConnectToPort(mqttPort)
int timeoutsBeforeSubmission = direct.startupConsumerTimeouts
- container.getService(AssetProcessingService.class)
- .sendAttributeEvent(new AttributeEvent(submittedAssetId, "submitted", true))
+ try {
+ container.getService(AssetProcessingService.class)
+ .sendAttributeEvent(new AttributeEvent(submittedAssetId, "submitted", true))
+ } catch (IllegalStateException failure) {
+ submissionFailure = failure
+ }
submissionConsumerTimeouts = direct.startupConsumerTimeouts - timeoutsBeforeSubmission
}
- submittedValueBeforeCamel = attributeValue(storage, submittedAssetId, "submitted")
super.start(container)
} finally {
direct.startupThread = null
@@ -291,8 +355,12 @@ class ManagerStartupTest extends Specification implements ManagerContainerTrait
private static class LateStartupProbe implements ContainerService {
final int httpPort
private final int mqttPort
- boolean httpPortOpen
- boolean mqttPortOpen
+ boolean rulesStartedDuringStart
+ boolean rulesStartedDuringProducerPhase
+ boolean httpPortOpenDuringStart
+ boolean mqttPortOpenDuringStart
+ boolean httpPortOpenDuringProducerPhase
+ boolean mqttPortOpenDuringProducerPhase
LateStartupProbe(int httpPort, int mqttPort) {
this.httpPort = httpPort
@@ -309,8 +377,16 @@ class ManagerStartupTest extends Specification implements ManagerContainerTrait
@Override
void start(Container container) {
- httpPortOpen = canConnectToPort(httpPort)
- mqttPortOpen = canConnectToPort(mqttPort)
+ rulesStartedDuringStart = container.getService(RulesService.class).startDone
+ httpPortOpenDuringStart = canConnectToPort(httpPort)
+ mqttPortOpenDuringStart = canConnectToPort(mqttPort)
+ }
+
+ @Override
+ void startProducers(Container container) {
+ rulesStartedDuringProducerPhase = container.getService(RulesService.class).startDone
+ httpPortOpenDuringProducerPhase = canConnectToPort(httpPort)
+ mqttPortOpenDuringProducerPhase = canConnectToPort(mqttPort)
}
@Override