diff --git a/container/src/main/java/org/openremote/container/Container.java b/container/src/main/java/org/openremote/container/Container.java index 1570b07ce4..4bbbea0769 100644 --- a/container/src/main/java/org/openremote/container/Container.java +++ b/container/src/main/java/org/openremote/container/Container.java @@ -196,6 +196,16 @@ public synchronized void start() throws Exception { LOG.log(INFO, "Starting service: " + service.getClass().getName()); service.start(Container.this); } + + LOG.log(INFO, "Starting runtime event producers"); + for (ContainerService service : getServices()) { + service.startProducers(this); + } + + LOG.log(INFO, "Starting runtime ingress"); + for (ContainerService service : getServices()) { + service.startIngress(this); + } } catch (Exception ex) { LOG.log(ERROR, ">>> Runtime container startup failed", ex); stop(true); diff --git a/container/src/main/java/org/openremote/container/message/MessageBrokerService.java b/container/src/main/java/org/openremote/container/message/MessageBrokerService.java index e92b751893..940555a06c 100644 --- a/container/src/main/java/org/openremote/container/message/MessageBrokerService.java +++ b/container/src/main/java/org/openremote/container/message/MessageBrokerService.java @@ -53,7 +53,7 @@ public class MessageBrokerService implements ContainerService { protected ProducerTemplate producerTemplate = context.createProducerTemplate(); protected FluentProducerTemplate fluentProducerTemplate = context.createFluentProducerTemplate(); - private boolean isStarted = false; + private volatile boolean isStarted; @Override public int getPriority() { @@ -151,12 +151,11 @@ public void start(Container container) throws Exception { @Override public void stop(Container container) throws Exception { - try { - if (context != null) { - context.stop(); - } - } finally { - isStarted = false; + // We want to indicate service is stopped before actually stopping it so callers will not be + // able to push new events during shutdown + isStarted = false; + if (context != null) { + context.stop(); } } diff --git a/container/src/main/java/org/openremote/container/web/WebService.java b/container/src/main/java/org/openremote/container/web/WebService.java index 2a8b54202a..023fbc5348 100644 --- a/container/src/main/java/org/openremote/container/web/WebService.java +++ b/container/src/main/java/org/openremote/container/web/WebService.java @@ -182,7 +182,10 @@ public void init(Container container) throws Exception { } @Override - public void start(Container container) throws Exception { + public void start(Container container) throws Exception {} + + @Override + public void startIngress(Container container) throws Exception { if (undertow != null) { undertow.start(); LOG.log(INFO, "Webserver ready on http://" + host + ":" + port); diff --git a/manager/src/main/java/org/openremote/manager/asset/AssetProcessingService.java b/manager/src/main/java/org/openremote/manager/asset/AssetProcessingService.java index 089ca47276..d26dfd9ad7 100644 --- a/manager/src/main/java/org/openremote/manager/asset/AssetProcessingService.java +++ b/manager/src/main/java/org/openremote/manager/asset/AssetProcessingService.java @@ -31,6 +31,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; import org.apache.camel.Exchange; +import org.apache.camel.FluentProducerTemplate; +import org.apache.camel.ServiceStatus; import org.apache.camel.builder.RouteBuilder; import org.apache.camel.builder.RouteConfigurationBuilder; import org.openremote.container.message.MessageBrokerService; @@ -91,6 +93,9 @@ public class AssetProcessingService extends RouteBuilder implements ContainerSer public static final String ATTRIBUTE_EVENT_ROUTE_CONFIG_ID = "attributeEvent"; public static final int PRIORITY = AssetStorageService.PRIORITY + 1000; public static final String ATTRIBUTE_EVENT_PROCESSOR = "direct://AttributeEventProcessor"; + private static final String ATTRIBUTE_EVENT_ROUTE_ID = "AttributeEvent-Processor"; + private static final String ATTRIBUTE_EVENT_SEND_ENDPOINT = + ATTRIBUTE_EVENT_PROCESSOR + "?block=false"; private static final System.Logger LOG = System.getLogger(AssetProcessingService.class.getName()); protected final List eventInterceptors = new ArrayList<>(); protected TimerService timerService; @@ -329,7 +334,7 @@ public void configure() throws Exception { // All user authorisation checks MUST have been carried out before events reach this queue from(ATTRIBUTE_EVENT_PROCESSOR) - .routeId("AttributeEvent-Processor") + .routeId(ATTRIBUTE_EVENT_ROUTE_ID) .routeConfigurationId(ATTRIBUTE_EVENT_ROUTE_CONFIG_ID) .threads() .executorService(executorService) @@ -382,24 +387,59 @@ public void addEventInterceptor(AttributeEventInterceptor eventInterceptor) { eventInterceptors.sort(Comparator.comparingInt(AttributeEventInterceptor::getPriority)); } - /** Send internal attribute change events into the {@link #ATTRIBUTE_EVENT_PROCESSOR}. */ + /** + * Send an internal attribute change event for asynchronous processing. + * + * @throws IllegalStateException if attribute event processing is unavailable + */ public void sendAttributeEvent(AttributeEvent attributeEvent) { sendAttributeEvent(attributeEvent, null); } - /** Send internal attribute change events into the {@link #ATTRIBUTE_EVENT_PROCESSOR}. */ + /** + * Send an internal attribute change event for asynchronous processing. + * + * @throws IllegalStateException if attribute event processing is unavailable + */ public void sendAttributeEvent(AttributeEvent attributeEvent, String source) { - attributeEvent.setSource(source); + prepareAttributeEvent(attributeEvent, source).asyncSend(); + } + + /** + * Process an internal attribute change before continuing startup work that depends on the value. + * + * @throws IllegalStateException if attribute event processing is unavailable + * @throws AssetProcessingException if the attribute event cannot be processed + */ + public void sendAttributeEventAndWait(AttributeEvent attributeEvent, String source) { + Object result = prepareAttributeEvent(attributeEvent, source).request(); + if (result instanceof RuntimeException failure) { + throw failure; + } + } + + private FluentProducerTemplate prepareAttributeEvent( + AttributeEvent attributeEvent, String source) { + if (messageBrokerService == null + || !messageBrokerService.isStarted() + || messageBrokerService + .getContext() + .getRouteController() + .getRouteStatus(ATTRIBUTE_EVENT_ROUTE_ID) + != ServiceStatus.Started) { + throw new IllegalStateException("Attribute event processor is not ready"); + } - // Set event source time if not already set + attributeEvent.setSource(source); if (attributeEvent.getTimestamp() <= 0) { attributeEvent.setTimestamp(timerService.getCurrentTimeMillis()); } - messageBrokerService + + // Fail promptly if the consumer stops between the readiness check and the send. + return messageBrokerService .getFluentProducerTemplate() .withBody(attributeEvent) - .to(ATTRIBUTE_EVENT_PROCESSOR) - .asyncSend(); + .to(ATTRIBUTE_EVENT_SEND_ENDPOINT); } /** diff --git a/manager/src/main/java/org/openremote/manager/datapoint/ApplyPredictedDataPointsService.java b/manager/src/main/java/org/openremote/manager/datapoint/ApplyPredictedDataPointsService.java index 657bbd0fd6..8c2307ca1a 100644 --- a/manager/src/main/java/org/openremote/manager/datapoint/ApplyPredictedDataPointsService.java +++ b/manager/src/main/java/org/openremote/manager/datapoint/ApplyPredictedDataPointsService.java @@ -89,7 +89,10 @@ public void start(Container container) throws Exception { clientEventService.addSubscription( AssetPredictedDatapointEvent.class, null, this::onPredictedDatapointsChanged); clientEventService.addSubscription(AttributeEvent.class, null, this::onAttributeEvent); + } + @Override + public void startProducers(Container container) { loadInitialAttributes(); scheduleNext(); } diff --git a/manager/src/main/java/org/openremote/manager/mqtt/ConnectionMonitorHandler.java b/manager/src/main/java/org/openremote/manager/mqtt/ConnectionMonitorHandler.java index dabf016997..28c6617b92 100644 --- a/manager/src/main/java/org/openremote/manager/mqtt/ConnectionMonitorHandler.java +++ b/manager/src/main/java/org/openremote/manager/mqtt/ConnectionMonitorHandler.java @@ -29,17 +29,17 @@ import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; +import java.util.function.BiConsumer; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; import org.apache.activemq.artemis.core.config.Configuration; import org.apache.activemq.artemis.spi.core.protocol.RemotingConnection; import org.apache.camel.builder.RouteBuilder; -import org.openremote.container.message.MessageBrokerService; import org.openremote.container.persistence.PersistenceService; import org.openremote.container.security.AuthContext; import org.openremote.container.security.IdentityProvider; +import org.openremote.manager.asset.AssetProcessingException; import org.openremote.manager.asset.AssetProcessingService; import org.openremote.manager.asset.AssetStorageService; import org.openremote.manager.gateway.GatewayService; @@ -71,7 +71,6 @@ public class ConnectionMonitorHandler extends MQTTHandler { protected static final Logger LOG = SyslogCategory.getLogger(API, ConnectionMonitorHandler.class); protected MQTTBrokerService mqttBrokerService; - protected ExecutorService executorService; protected AssetStorageService assetStorageService; protected AssetProcessingService assetProcessingService; protected GatewayService gatewayService; @@ -82,13 +81,11 @@ public class ConnectionMonitorHandler extends MQTTHandler { @Override public void init(Container container, Configuration serverConfiguration) throws Exception { super.init(container, serverConfiguration); - executorService = container.getExecutor(); mqttBrokerService = container.getService(MQTTBrokerService.class); assetStorageService = container.getService(AssetStorageService.class); assetProcessingService = container.getService(AssetProcessingService.class); gatewayService = container.getService(GatewayService.class); persistenceService = container.getService(PersistenceService.class); - MessageBrokerService messageBrokerService = container.getService(MessageBrokerService.class); messageBrokerService .getContext() .addRoutes( @@ -149,37 +146,48 @@ public void configure() throws Exception { @Override public void start(Container container) throws Exception { - // Don't do super start as don't need to publish or subscribe - // super.start(container); - - // Don't block start - executorService.submit( - () -> { - // Get all assets that have attributes with user connected meta and initialise them - List> assets = - assetStorageService.findAll( - new AssetQuery() - .attributes( - new AttributePredicate() - .meta(new NameValuePredicate(USER_CONNECTED, null)))); - - Map>> realmAttributeMap = - assets.stream().collect(Collectors.groupingBy(Asset::getRealm)); - - realmAttributeMap.forEach( - (realm, realmAssets) -> { - List>> assetIdsAttrs = - realmAssets.stream() - .flatMap( - asset -> - asset.getAttributes().stream() - .filter(ConnectionMonitorHandler::attributeMatches) - .map( - attr -> - new Pair>(asset.getId(), attr))) - .toList(); - - addSessionAttributes(realm, assetIdsAttrs); + // This handler has no internal MQTT session to start. + } + + @Override + public void startProducers(Container container) { + // Get all assets that have attributes with user connected meta and initialise them + List> assets = + assetStorageService.findAll( + new AssetQuery() + .attributes( + new AttributePredicate().meta(new NameValuePredicate(USER_CONNECTED, null)))); + + Map>> realmAttributeMap = + assets.stream().collect(Collectors.groupingBy(Asset::getRealm)); + + realmAttributeMap.forEach( + (realm, realmAssets) -> { + List>> assetIdsAttrs = + realmAssets.stream() + .flatMap( + asset -> + asset.getAttributes().stream() + .filter(ConnectionMonitorHandler::attributeMatches) + .map(attr -> new Pair>(asset.getId(), attr))) + .toList(); + + forEachSessionAttribute( + realm, + assetIdsAttrs, + (userID, attributeRef) -> { + registerSessionAttribute(userID, attributeRef); + boolean connected = !mqttBrokerService.getUserConnections(userID).isEmpty(); + // Finish the initial write before the public acceptor can receive a connection. + try { + assetProcessingService.sendAttributeEventAndWait( + new AttributeEvent(attributeRef, connected), getClass().getSimpleName()); + } catch (AssetProcessingException e) { + LOG.log( + Level.WARNING, + "Could not initialize connection status for " + attributeRef, + e); + } }); }); } @@ -265,6 +273,13 @@ public void onUnsubscribe(RemotingConnection connection, Topic topic) {} protected void addSessionAttributes( String realm, List>> assetIdsAttrs) { + forEachSessionAttribute(realm, assetIdsAttrs, this::addSessionAttribute); + } + + private void forEachSessionAttribute( + String realm, + List>> assetIdsAttrs, + BiConsumer action) { LOG.finest( "Adding '" + assetIdsAttrs.size() @@ -275,12 +290,8 @@ protected void addSessionAttributes( assetIdsAttrs.stream() .map(assetIdAttr -> assetIdAttr.getValue().getMetaValue(USER_CONNECTED).orElse(null)) .filter(Objects::nonNull) + .map(ConnectionMonitorHandler::toServiceAccountUsername) .distinct() - .map( - username -> - username.startsWith(User.SERVICE_ACCOUNT_PREFIX) - ? username - : User.SERVICE_ACCOUNT_PREFIX + username) .toList(); // Convert usernames to userIds @@ -294,7 +305,7 @@ protected void addSessionAttributes( .ifPresent( username -> { String userID = - userIds.get(usernames.indexOf(User.SERVICE_ACCOUNT_PREFIX + username)); + userIds.get(usernames.indexOf(toServiceAccountUsername(username))); if (userID == null) { LOG.warning( @@ -303,22 +314,32 @@ protected void addSessionAttributes( + ", username=" + username); } else { - addSessionAttribute( + action.accept( userID, new AttributeRef(assetIdAttr.key, assetIdAttr.getValue().getName())); } })); } + private static String toServiceAccountUsername(String username) { + return username.startsWith(User.SERVICE_ACCOUNT_PREFIX) + ? username + : User.SERVICE_ACCOUNT_PREFIX + username; + } + protected void addSessionAttribute(String userID, AttributeRef attributeRef) { LOG.finest("Adding userID '" + userID + "' monitoring for attribute: " + attributeRef); updateUserConnectedStatus( userID, Collections.singletonList(attributeRef), !mqttBrokerService.getUserConnections(userID).isEmpty()); - Set refs = - userIDAttributeRefs.computeIfAbsent(userID, ID -> ConcurrentHashMap.newKeySet()); - refs.add(attributeRef); + registerSessionAttribute(userID, attributeRef); + } + + private void registerSessionAttribute(String userID, AttributeRef attributeRef) { + userIDAttributeRefs + .computeIfAbsent(userID, id -> ConcurrentHashMap.newKeySet()) + .add(attributeRef); } protected void removeSessionAttribute(String userID, AttributeRef attributeRef) { @@ -334,6 +355,9 @@ protected void removeSessionAttribute(String userID, AttributeRef attributeRef) protected void updateUserConnectedStatus( String userID, Collection attributeRefs, boolean connected) { + if (!messageBrokerService.isStarted()) { + return; + } // Only update statuses if this is the only connection Set connections = mqttBrokerService.getUserConnections(userID); diff --git a/manager/src/main/java/org/openremote/manager/mqtt/MQTTBrokerService.java b/manager/src/main/java/org/openremote/manager/mqtt/MQTTBrokerService.java index 45a2c2cca1..c7809c9122 100644 --- a/manager/src/main/java/org/openremote/manager/mqtt/MQTTBrokerService.java +++ b/manager/src/main/java/org/openremote/manager/mqtt/MQTTBrokerService.java @@ -20,6 +20,7 @@ import static java.lang.System.Logger.Level.*; import static java.util.stream.StreamSupport.stream; +import static org.apache.activemq.artemis.core.remoting.impl.netty.TransportConstants.AUTO_START; import static org.openremote.container.persistence.PersistenceService.PERSISTENCE_TOPIC; import static org.openremote.container.security.IdentityProvider.getTokenPrincipal; import static org.openremote.model.syslog.SyslogCategory.API; @@ -102,6 +103,7 @@ public class MQTTBrokerService extends RouteBuilder public static final int PRIORITY = MED_PRIORITY; public static final String MQTT_SERVER_LISTEN_HOST = "MQTT_SERVER_LISTEN_HOST"; public static final String MQTT_SERVER_LISTEN_PORT = "MQTT_SERVER_LISTEN_PORT"; + private static final String MQTT_TCP_ACCEPTOR = "tcp"; // Allow 5 min durable session but this will not enable retained topics etc. as we delete queues // aggressively for now public static final int DEFAULT_SESSION_EXPIRY_MILLIS = 300000; @@ -192,6 +194,7 @@ public void init(Container container) throws Exception { .setHost(host) .setPort(port) .setParameter("protocols", "MQTT") + .setParameter(AUTO_START, "false") .setParameter( "allowLinkStealing", "false") // Preventing this ensures previous connection/session is properly cleaned @@ -200,7 +203,7 @@ public void init(Container container) throws Exception { "defaultMqttSessionExpiryInterval", Integer.toString(DEFAULT_SESSION_EXPIRY_MILLIS)) .build() .toString(); - serverConfiguration.addAcceptorConfiguration("tcp", serverURI); + serverConfiguration.addAcceptorConfiguration(MQTT_TCP_ACCEPTOR, serverURI); serverConfiguration.registerBrokerPlugin(this); if (container.getMeterRegistry() != null) { serverConfiguration.setMetricsConfiguration( @@ -299,7 +302,7 @@ public void start(Container container) throws Exception { return; } - // Start the broker + // Start the broker and its in-process acceptor so handlers can create internal sessions. server = new EmbeddedActiveMQ(); server.setConfiguration(serverConfiguration); securityManager = new ActiveMQORSecurityManager(this, executorService, identityService); @@ -372,6 +375,23 @@ public void start(Container container) throws Exception { } } + @Override + public void startProducers(Container container) throws Exception { + if (active) { + for (MQTTHandler handler : customHandlers) { + handler.startProducers(container); + } + } + } + + @Override + public void startIngress(Container container) throws Exception { + if (active) { + server.getActiveMQServer().getRemotingService().getAcceptor(MQTT_TCP_ACCEPTOR).start(); + LOG.log(INFO, "MQTT listener ready on " + host + ":" + port); + } + } + @SuppressWarnings("unchecked") @Override public void configure() throws Exception { diff --git a/manager/src/main/java/org/openremote/manager/mqtt/MQTTHandler.java b/manager/src/main/java/org/openremote/manager/mqtt/MQTTHandler.java index 33702615a6..e3f6f9d8c6 100644 --- a/manager/src/main/java/org/openremote/manager/mqtt/MQTTHandler.java +++ b/manager/src/main/java/org/openremote/manager/mqtt/MQTTHandler.java @@ -123,6 +123,9 @@ public void start(Container container) throws Exception { } } + /** Starts work that needs running event consumers before the public MQTT acceptor opens. */ + public void startProducers(Container container) throws Exception {} + /** Called when the system stops to allow for any cleanup. */ public void stop() throws Exception { if (clientSession != null) { diff --git a/manager/src/main/java/org/openremote/manager/rules/RulesService.java b/manager/src/main/java/org/openremote/manager/rules/RulesService.java index 3710466db4..17aace28b5 100644 --- a/manager/src/main/java/org/openremote/manager/rules/RulesService.java +++ b/manager/src/main/java/org/openremote/manager/rules/RulesService.java @@ -154,7 +154,8 @@ public class RulesService extends RouteBuilder implements ContainerService { protected long quickFireMillis; protected boolean groovyRulesExecutionEnabled; protected boolean initDone; - protected boolean startDone; + private boolean producerPhaseReached; + protected volatile boolean startDone; protected io.micrometer.core.instrument.Timer rulesFiringTimer; @Override @@ -164,6 +165,7 @@ public int getPriority() { @Override public void init(Container container) throws Exception { + producerPhaseReached = false; executorService = container.getExecutor(); scheduledExecutorService = container.getScheduledExecutor(); timerService = container.getService(TimerService.class); @@ -359,8 +361,21 @@ AssetRuleset.class, new RulesetQuery().setEnabledOnly(true).setFullyPopulate(tru }); }); - // Start the engines + // Restart the engines immediately when only the rules service is restarted. + if (producerPhaseReached && container.getService(MessageBrokerService.class).isStarted()) { + startEngines(); + } + } + + @Override + public void startProducers(Container container) { + if (!startDone) { + startEngines(); + } + producerPhaseReached = true; + } + private void startEngines() { synchronized (ENGINE_LOCK) { RulesEngine globalRulesEngine = globalEngine.get(); if (globalRulesEngine != null) { @@ -382,6 +397,7 @@ AssetRuleset.class, new RulesetQuery().setEnabledOnly(true).setFullyPopulate(tru @Override public void stop(Container container) throws Exception { + startDone = false; for (GeofenceAssetAdapter geofenceAssetAdapter : geofenceAssetAdapters) { try { geofenceAssetAdapter.stop(container); @@ -596,16 +612,11 @@ protected void processRulesetChange(Ruleset ruleset, PersistenceEvent.Cause caus undeployAssetRuleset((AssetRuleset) ruleset); } } else { - if (ruleset instanceof GlobalRuleset) { - - RulesEngine engine = deployGlobalRuleset((GlobalRuleset) ruleset); - engine.start(); - - } else if (ruleset instanceof RealmRuleset) { - - RulesEngine engine = deployRealmRuleset((RealmRuleset) ruleset); - engine.start(); - + RulesEngine engine; + if (ruleset instanceof GlobalRuleset globalRuleset) { + engine = deployGlobalRuleset(globalRuleset); + } else if (ruleset instanceof RealmRuleset realmRuleset) { + engine = deployRealmRuleset(realmRuleset); } else if (ruleset instanceof AssetRuleset) { // Must reload from the database, the ruleset might not be completely hydrated on CREATE or @@ -615,7 +626,12 @@ protected void processRulesetChange(Ruleset ruleset, PersistenceEvent.Cause caus undeployAssetRuleset((AssetRuleset) ruleset); return; } - RulesEngine engine = deployAssetRuleset(assetRuleset); + engine = deployAssetRuleset(assetRuleset); + } else { + return; + } + + if (startDone) { engine.start(); } } diff --git a/model/src/main/java/org/openremote/model/ContainerService.java b/model/src/main/java/org/openremote/model/ContainerService.java index 3f8679c14e..f5482b7ee0 100644 --- a/model/src/main/java/org/openremote/model/ContainerService.java +++ b/model/src/main/java/org/openremote/model/ContainerService.java @@ -26,18 +26,38 @@ * Container} using the auto service discovery mechanism. If the container is started with an * explicit list of services then the insertion order is used. * - *

Service startup lifecycle: + *

The container runs each startup phase in registry order and completes it for all services + * before moving to the next. * - *

    - *
  1. {@link #init} in registry insertion order - *
  2. {@link #start} in registry insertion order - *
+ * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Service startup phases
PhaseResponsibility
{@link #init}Resolve dependencies and register routes, handlers, and subscriptions.
{@link #start}Start internal services and event consumers, including the message broker.
{@link #startProducers}Enable internal event producers and complete startup writes required before ingress.
{@link #startIngress}Open public listeners and accept external traffic.
* - *

Service shutdown lifecycle: - * - *

    - *
  1. {@link #stop} in reverse registry order - *
+ *

On shutdown, the container calls {@link #stop} in reverse registry order. */ @TsIgnore public interface ContainerService { @@ -48,8 +68,8 @@ public interface ContainerService { int LOW_PRIORITY = Integer.MAX_VALUE - 1000; /** - * Gets the priority of this service which is used to determine initialization order when services - * are auto discovered; services with a lower priority are initialized and started first. + * Gets the priority of this service, which determines the order within each startup phase when + * services are auto-discovered. Services with a lower priority run first. */ default int getPriority() { return DEFAULT_PRIORITY; @@ -63,12 +83,19 @@ default int getPriority() { void init(Container container) throws Exception; /** - * After initialization, services are started in the order they have been added to the container - * (if container started with explicit list of services) otherwise they are started in order of - * {@link #getPriority}. + * Starts internal services and event consumers after every service has completed {@link #init}. */ void start(Container container) throws Exception; + /** + * Starts internal event producers after every service has completed {@link #start}. Complete + * writes required for public traffic before returning from this hook. + */ + default void startProducers(Container container) throws Exception {} + + /** Opens public listeners after every service has completed {@link #startProducers}. */ + default void startIngress(Container container) throws Exception {} + /** * When the container is shutting down, it stops all services in the reverse order they were * started. diff --git a/test/src/test/groovy/org/openremote/test/system/ManagerStartupTest.groovy b/test/src/test/groovy/org/openremote/test/system/ManagerStartupTest.groovy new file mode 100644 index 0000000000..46d3b37640 --- /dev/null +++ b/test/src/test/groovy/org/openremote/test/system/ManagerStartupTest.groovy @@ -0,0 +1,414 @@ +/* + * Copyright 2026, OpenRemote Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +package org.openremote.test.system + +import org.apache.camel.component.direct.DirectComponent +import org.apache.camel.component.direct.DirectConsumer +import org.openremote.agent.protocol.mqtt.MQTT_IOClient +import org.openremote.container.message.MessageBrokerService +import org.openremote.container.timer.TimerService +import org.openremote.manager.asset.AssetProcessingService +import org.openremote.manager.asset.AssetStorageService +import org.openremote.manager.datapoint.AssetPredictedDatapointService +import org.openremote.manager.mqtt.MQTTBrokerService +import org.openremote.manager.rules.RulesService +import org.openremote.manager.setup.SetupService +import org.openremote.model.Container +import org.openremote.model.ContainerService +import org.openremote.model.asset.agent.ConnectionStatus +import org.openremote.model.asset.impl.ThingAsset +import org.openremote.model.attribute.Attribute +import org.openremote.model.attribute.AttributeEvent +import org.openremote.model.attribute.MetaItem +import org.openremote.model.auth.UsernamePassword +import org.openremote.model.datapoint.ValueDatapoint +import org.openremote.model.util.UniqueIdentifierGenerator +import org.openremote.setup.integration.KeycloakTestSetup +import org.openremote.test.ManagerContainerTrait +import spock.lang.Shared +import spock.lang.Specification +import spock.util.concurrent.PollingConditions + +import java.util.concurrent.CountDownLatch + +import static java.util.concurrent.TimeUnit.MILLISECONDS +import static org.openremote.container.Container.OR_EXECUTOR_THREADS_MAX +import static org.openremote.container.Container.OR_EXECUTOR_THREADS_MIN +import static org.openremote.container.web.WebService.OR_WEBSERVER_LISTEN_HOST +import static org.openremote.container.web.WebService.OR_WEBSERVER_LISTEN_PORT +import static org.openremote.manager.asset.AssetProcessingService.ATTRIBUTE_EVENT_PROCESSOR +import static org.openremote.manager.mqtt.MQTTBrokerService.MQTT_SERVER_LISTEN_HOST +import static org.openremote.manager.mqtt.MQTTBrokerService.MQTT_SERVER_LISTEN_PORT +import static org.openremote.model.security.User.SERVICE_ACCOUNT_PREFIX +import static org.openremote.model.value.MetaItemType.APPLY_PREDICTED_DATA_POINTS +import static org.openremote.model.value.MetaItemType.HAS_PREDICTED_DATA_POINTS +import static org.openremote.model.value.MetaItemType.USER_CONNECTED +import static org.openremote.model.value.ValueType.BOOLEAN + +class ManagerStartupTest extends Specification implements ManagerContainerTrait { + + @Shared + private int mqttPort + @Shared + private StartupMqttBroker mqttBroker + @Shared + private StartupMessageBroker messageBroker + @Shared + private AssetStorageService storage + @Shared + private LateStartupProbe lateStartupProbe + + def setupSpec() { + mqttPort = findEphemeralPort() + messageBroker = new StartupMessageBroker(mqttPort) + mqttBroker = new StartupMqttBroker(messageBroker.direct) + + def config = defaultConfig() << [ + (MQTT_SERVER_LISTEN_HOST): "127.0.0.1", + (MQTT_SERVER_LISTEN_PORT): mqttPort.toString(), + (OR_WEBSERVER_LISTEN_HOST): "127.0.0.1", + (OR_EXECUTOR_THREADS_MIN): "1", + (OR_EXECUTOR_THREADS_MAX): "1" + ] + lateStartupProbe = new LateStartupProbe(config[OR_WEBSERVER_LISTEN_PORT].toInteger(), mqttPort) + List services = defaultServices(lateStartupProbe).collect { + if (it instanceof MessageBrokerService) { + return messageBroker + } + if (it instanceof MQTTBrokerService) { + return mqttBroker + } + return it + } + // Exercise a custom service order in which Camel is already running when rules are prepared. + services.remove(messageBroker) + services.add(services.findIndexOf { it instanceof RulesService }, messageBroker) + + def manager = startContainer(config, services) + storage = manager.getService(AssetStorageService.class) + } + + def cleanupSpec() { + stopContainer() + } + + def "internal producers start after consumers and before public listeners"() { + expect: "rules wait for the producer phase even when Camel starts first" + verifyAll { + !lateStartupProbe.rulesStartedDuringStart + lateStartupProbe.rulesStartedDuringProducerPhase + } + + and: "both endpoints stay closed through the producer phase and accept connections after startup" + verifyAll { + !messageBroker.mqttPortOpenBeforeCamel + !lateStartupProbe.httpPortOpenDuringStart + !lateStartupProbe.mqttPortOpenDuringStart + !lateStartupProbe.httpPortOpenDuringProducerPhase + !lateStartupProbe.mqttPortOpenDuringProducerPhase + canConnectToPort(lateStartupProbe.httpPort) + canConnectToPort(mqttPort) + } + } + + def "MQTT connection monitoring reconciles state during startup"() { + given: "the startup probe ran under executor saturation with stale online values in storage" + def conditions = statusConditions() + MQTT_IOClient client + + expect: "startup completed without consumer waits and both stored statuses are offline" + verifyAll { + mqttBroker.startupConsumerTimeouts == 0 + connectionStatuses(storage, mqttBroker.monitoredAssetId) == [false, false] + } + + when: "the monitored client connects after startup" + client = new MQTT_IOClient(UniqueIdentifierGenerator.generateId(), "127.0.0.1", mqttPort, + false, true, mqttBroker.credentials, null, null) + client.connect() + + then: "both monitored attributes report the client online" + conditions.eventually { + assert client.connectionStatus == ConnectionStatus.CONNECTED + assert connectionStatuses(storage, mqttBroker.monitoredAssetId) == [true, true] + } + + when: "that client disconnects" + client.disconnect() + + then: "both monitored attributes report the client offline" + conditions.eventually { + assert connectionStatuses(storage, mqttBroker.monitoredAssetId) == [false, false] + } + + cleanup: + client?.disconnect() + } + + def "a Java attribute submission fails fast before Camel starts and can be retried"() { + given: + def assetId = messageBroker.submittedAssetId + def conditions = statusConditions() + + expect: "the early submission was rejected without waiting for a consumer" + verifyAll { + messageBroker.submissionConsumerTimeouts == 0 + messageBroker.submissionFailure?.message == "Attribute event processor is not ready" + attributeValue(storage, assetId, "submitted") == false + } + + when: "the caller retries after Camel starts" + container.getService(AssetProcessingService.class) + .sendAttributeEvent(new AttributeEvent(assetId, "submitted", true)) + + then: "the retried event is applied" + conditions.eventually { + assert attributeValue(storage, assetId, "submitted") == true + } + } + + def "a Java attribute submission fails fast while only its route is stopped"() { + given: + def routeController = messageBroker.context.routeController + routeController.stopRoute("AttributeEvent-Processor") + + when: + container.getService(AssetProcessingService.class) + .sendAttributeEvent(new AttributeEvent(messageBroker.submittedAssetId, "submitted", true)) + + then: + IllegalStateException failure = thrown() + failure.message == "Attribute event processor is not ready" + messageBroker.isStarted() + + cleanup: + routeController?.startRoute("AttributeEvent-Processor") + } + + def "a prediction already due during startup is applied once Camel is ready"() { + expect: + def conditions = statusConditions() + conditions.eventually { + assert attributeValue(storage, mqttBroker.monitoredAssetId, "startupPrediction") == true + } + } + + private static KeycloakTestSetup keycloakSetup(Container container) { + container.getService(SetupService.class).getTaskOfType(KeycloakTestSetup.class) + } + + private static Boolean attributeValue(AssetStorageService storage, String assetId, String name) { + storage.find(assetId).getAttribute(name).get().value.orElse(null) + } + + private static List connectionStatuses(AssetStorageService storage, String assetId) { + def asset = storage.find(assetId) + [ + asset.getAttribute("firstConnected").get().value.orElse(null), + asset.getAttribute("secondConnected").get().value.orElse(null) + ] + } + + private static PollingConditions statusConditions() { + def conditions = new PollingConditions(timeout: 10) + conditions.onTimeout { failure -> failure.message } + return conditions + } + + private static boolean canConnectToPort(int port) { + try { + new Socket().withCloseable { + it.connect(new InetSocketAddress("127.0.0.1", port), 1000) + } + return true + } catch (ConnectException refused) { + return false + } + } + + private static void withSaturatedExecutor(Container container, Closure action) { + def workerOccupied = new CountDownLatch(1) + def releaseWorker = new CountDownLatch(1) + def startupThread = Thread.currentThread() + + try { + new PollingConditions(timeout: 10).eventually { + if (workerOccupied.count> 0) { + container.executor.execute { + // Retry if another startup task occupies the worker and this submission runs inline. + if (Thread.currentThread() != startupThread) { + workerOccupied.countDown() + releaseWorker.await() + } + } + } + assert workerOccupied.await(100, MILLISECONDS) + } + action.call() + } catch (AssertionError failure) { + // Let the container's startup error handling stop partially started services. + throw new IllegalStateException("Could not arrange the startup probe", failure) + } finally { + releaseWorker.countDown() + } + } + + private static class StartupMqttBroker extends MQTTBrokerService { + private final StartupDirectComponent direct + String monitoredAssetId + UsernamePassword credentials + int startupConsumerTimeouts + + StartupMqttBroker(StartupDirectComponent direct) { + this.direct = direct + } + + @Override + void start(Container container) { + def keycloak = keycloakSetup(container) + def realm = keycloak.realmBuilding.name + def user = keycloak.serviceUser + credentials = new UsernamePassword("${realm}:${user.username}".toString(), user.secret) + long now = container.getService(TimerService.class).getCurrentTimeMillis() + def asset = new ThingAsset("Startup attributes").setRealm(realm).addAttributes( + new Attribute<>("firstConnected", BOOLEAN, true).addMeta(new MetaItem<>(USER_CONNECTED, user.username)), + new Attribute<>("secondConnected", BOOLEAN, true) + .addMeta(new MetaItem<>(USER_CONNECTED, SERVICE_ACCOUNT_PREFIX + user.username)), + new Attribute<>("startupPrediction", BOOLEAN, false).setTimestamp(now - 2000) + .addOrReplaceMeta(new MetaItem<>(HAS_PREDICTED_DATA_POINTS, true), + new MetaItem<>(APPLY_PREDICTED_DATA_POINTS, true)) + ) + monitoredAssetId = container.getService(AssetStorageService.class).merge(asset).id + container.getService(AssetPredictedDatapointService.class) + .updateValues(monitoredAssetId, "startupPrediction", [new ValueDatapoint<>(now - 1000, true)]) + + int timeoutsBeforeMqtt = direct.startupConsumerTimeouts + withSaturatedExecutor(container) { + super.start(container) + } + startupConsumerTimeouts = direct.startupConsumerTimeouts - timeoutsBeforeMqtt + } + } + + private static class StartupMessageBroker extends MessageBrokerService { + final StartupDirectComponent direct = new StartupDirectComponent() + private final int mqttPort + boolean mqttPortOpenBeforeCamel + String submittedAssetId + IllegalStateException submissionFailure + int submissionConsumerTimeouts + + StartupMessageBroker(int mqttPort) { + this.mqttPort = mqttPort + context.addComponent("direct", direct) + } + + @Override + void init(Container container) { + direct.startupThread = Thread.currentThread() + super.init(container) + } + + @Override + void start(Container container) { + try { + def keycloak = keycloakSetup(container) + def storage = container.getService(AssetStorageService.class) + def asset = new ThingAsset("Startup attribute event").setRealm(keycloak.realmBuilding.name) + .addAttributes(new Attribute<>("submitted", BOOLEAN, false)) + submittedAssetId = storage.merge(asset).id + + withSaturatedExecutor(container) { + mqttPortOpenBeforeCamel = canConnectToPort(mqttPort) + int timeoutsBeforeSubmission = direct.startupConsumerTimeouts + try { + container.getService(AssetProcessingService.class) + .sendAttributeEvent(new AttributeEvent(submittedAssetId, "submitted", true)) + } catch (IllegalStateException failure) { + submissionFailure = failure + } + submissionConsumerTimeouts = direct.startupConsumerTimeouts - timeoutsBeforeSubmission + } + super.start(container) + } finally { + direct.startupThread = null + } + } + } + + private static class LateStartupProbe implements ContainerService { + final int httpPort + private final int mqttPort + boolean rulesStartedDuringStart + boolean rulesStartedDuringProducerPhase + boolean httpPortOpenDuringStart + boolean mqttPortOpenDuringStart + boolean httpPortOpenDuringProducerPhase + boolean mqttPortOpenDuringProducerPhase + + LateStartupProbe(int httpPort, int mqttPort) { + this.httpPort = httpPort + this.mqttPort = mqttPort + } + + @Override + int getPriority() { + Integer.MAX_VALUE + } + + @Override + void init(Container container) {} + + @Override + void start(Container container) { + rulesStartedDuringStart = container.getService(RulesService.class).startDone + httpPortOpenDuringStart = canConnectToPort(httpPort) + mqttPortOpenDuringStart = canConnectToPort(mqttPort) + } + + @Override + void startProducers(Container container) { + rulesStartedDuringProducerPhase = container.getService(RulesService.class).startDone + httpPortOpenDuringProducerPhase = canConnectToPort(httpPort) + mqttPortOpenDuringProducerPhase = canConnectToPort(mqttPort) + } + + @Override + void stop(Container container) {} + } + + private static class StartupDirectComponent extends DirectComponent { + volatile Thread startupThread + int startupConsumerTimeouts + + @Override + protected DirectConsumer getConsumer(String key, boolean block, long timeout) { + if (key != ATTRIBUTE_EVENT_PROCESSOR || !block || timeout <= 0) { + return super.getConsumer(key, block, timeout) + } + + // Keep the real consumer wait and its failure path, with a short timeout for this spec. + def consumer = super.getConsumer(key, true, Math.min(timeout, 100L)) + if (consumer == null && Thread.currentThread() == startupThread) { + startupConsumerTimeouts++ + } + return consumer + } + } +}