Skip to content

NIFI-16304 Retain connector verification and purge results for one minute #11634

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
markap14 wants to merge 1 commit into
base: main
Choose a base branch
from
+69 −2
Open

NIFI-16304 Retain connector verification and purge results for one minute #11634

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ...mework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectorResource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,11 +175,11 @@ public class ConnectorResource extends ApplicationResource {
private UploadRequestReplicator uploadRequestReplicator;

private final RequestManager<VerifyConnectorConfigStepRequestEntity, List<ConfigVerificationResultDTO>> configVerificationRequestManager =
new AsyncRequestManager<>(100, 1L, "Connector Configuration Step Verification");
new AsyncRequestManager<>(100, TimeUnit.MINUTES.toMillis(1L), "Connector Configuration Step Verification");
private final RequestManager<MigrationRequestEntity, ConnectorEntity> migrationRequestManager =
new AsyncRequestManager<>(100, MIGRATION_REQUEST_TTL_MILLIS, "Connector Migration");

private final RequestManager<ConnectorEntity, Void> purgeRequestManager = new AsyncRequestManager<>(100, 1L, "Connector FlowFile Purge");
private final RequestManager<ConnectorEntity, Void> purgeRequestManager = new AsyncRequestManager<>(100, TimeUnit.MINUTES.toMillis(1L), "Connector FlowFile Purge");

/**
* Uploaded migration payloads keyed by their server-assigned identifier. Each entry's age is tracked alongside
Expand Down
67 changes: 67 additions & 0 deletions ...tem-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorCrudIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,16 @@
import org.apache.nifi.components.connector.BundleCompatibility;
import org.apache.nifi.components.connector.ConnectorState;
import org.apache.nifi.tests.system.NiFiSystemIT;
import org.apache.nifi.toolkit.client.ConnectorClient;
import org.apache.nifi.toolkit.client.NiFiClientException;
import org.apache.nifi.web.api.dto.BundleDTO;
import org.apache.nifi.web.api.dto.ConfigVerificationResultDTO;
import org.apache.nifi.web.api.dto.ConfigurationStepConfigurationDTO;
import org.apache.nifi.web.api.dto.ConnectorConfigurationDTO;
import org.apache.nifi.web.api.dto.ConnectorValueReferenceDTO;
import org.apache.nifi.web.api.dto.ProcessorDTO;
import org.apache.nifi.web.api.dto.PropertyGroupConfigurationDTO;
import org.apache.nifi.web.api.dto.VerifyConnectorConfigStepRequestDTO;
import org.apache.nifi.web.api.dto.flow.FlowDTO;
import org.apache.nifi.web.api.dto.flow.ProcessGroupFlowDTO;
import org.apache.nifi.web.api.entity.ConnectorEntity;
Expand All @@ -37,7 +41,9 @@
import org.apache.nifi.web.api.entity.ProcessGroupEntity;
import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity;
import org.apache.nifi.web.api.entity.ProcessorEntity;
import org.apache.nifi.web.api.entity.VerifyConnectorConfigStepRequestEntity;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;

import java.io.IOException;
import java.util.Collection;
Expand All @@ -51,6 +57,7 @@
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;

Expand Down Expand Up @@ -116,6 +123,66 @@ public void testConfigVerification() throws NiFiClientException, IOException, In
assertTrue(resultDto.getExplanation().contains("Test Value"));
}

@Test
@Timeout(30)
public void testCompletedConfigVerificationRetainedWhenAnotherConnectorIsVerified() throws NiFiClientException, IOException, InterruptedException {
final ConnectorEntity firstConnector = getClientUtil().createConnector("NopConnector");
final ConnectorEntity secondConnector = getClientUtil().createConnector("NopConnector");
final ConnectorClient connectorClient = getNifiClient().getConnectorClient();
final String stepName = "Ignored Step";

final ConnectorValueReferenceDTO propertyValue = new ConnectorValueReferenceDTO();
propertyValue.setValueType("STRING_LITERAL");
propertyValue.setValue("First Verification");
final PropertyGroupConfigurationDTO propertyGroupConfiguration = new PropertyGroupConfigurationDTO();
propertyGroupConfiguration.setPropertyValues(Map.of("Ignored Property", propertyValue));
final ConfigurationStepConfigurationDTO stepConfiguration = new ConfigurationStepConfigurationDTO();
stepConfiguration.setConfigurationStepName(stepName);
stepConfiguration.setPropertyGroupConfigurations(List.of(propertyGroupConfiguration));
final VerifyConnectorConfigStepRequestDTO request = new VerifyConnectorConfigStepRequestDTO();
request.setConnectorId(firstConnector.getId());
request.setConfigurationStepName(stepName);
request.setConfigurationStep(stepConfiguration);
final VerifyConnectorConfigStepRequestEntity requestEntity = new VerifyConnectorConfigStepRequestEntity();
requestEntity.setRequest(request);

// The usual verification helper deletes completed requests. Keep this request so another submission can exercise expiration.
final String requestId = connectorClient.submitConfigStepVerificationRequest(requestEntity).getRequest().getRequestId();
try {
waitFor(() -> connectorClient.getConfigStepVerificationRequest(firstConnector.getId(), stepName, requestId).getRequest().isComplete());

// Completed requests remain available during the retention interval.
Thread.sleep(100L);
final VerifyConnectorConfigStepRequestDTO completedRequest = connectorClient.getConfigStepVerificationRequest(firstConnector.getId(), stepName, requestId).getRequest();
assertTrue(completedRequest.isComplete());
assertNull(completedRequest.getFailureReason());
assertEquals(2, completedRequest.getResults().size());
assertTrue(completedRequest.getResults().stream().allMatch(result -> Outcome.SUCCESSFUL.name().equals(result.getOutcome())));
assertTrue(completedRequest.getResults().get(1).getExplanation().contains("First Verification"));

// A new submission triggers cleanup in the request manager shared by all connectors.
final List<ConfigVerificationResultDTO> secondResults = getClientUtil().verifyConnectorStepConfig(secondConnector.getId(), stepName,
Map.of("Ignored Property", "Second Verification"));
assertEquals(2, secondResults.size());
assertTrue(secondResults.stream().allMatch(result -> Outcome.SUCCESSFUL.name().equals(result.getOutcome())));

final VerifyConnectorConfigStepRequestEntity retainedRequest =
connectorClient.getConfigStepVerificationRequest(firstConnector.getId(), stepName, requestId);
assertEquals(requestId, retainedRequest.getRequest().getRequestId());
assertTrue(retainedRequest.getRequest().isComplete());
assertEquals(completedRequest.getResults().get(1).getExplanation(), retainedRequest.getRequest().getResults().get(1).getExplanation());
} finally {
try {
connectorClient.deleteConfigStepVerificationRequest(firstConnector.getId(), stepName, requestId);
} catch (final NiFiClientException e) {
// Cleanup accepts a request that expired while the test was running.
if (!(e.getCause() instanceof WebApplicationException cause) || cause.getResponse().getStatus() != Response.Status.NOT_FOUND.getStatusCode()) {
throw e;
}
}
}
}

@Test
public void testSecretReferences() throws NiFiClientException, IOException, InterruptedException {
// Create and configure a Parameter Provider with two secrets
Expand Down
Loading