Skip to content

NIFI-16229 - Fixed inherited asset Parameter Context updates #11567

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
skeossei wants to merge 1 commit into from
Closed

NIFI-16229 - Fixed inherited asset Parameter Context updates #11567

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 12 additions & 4 deletions ...rk/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1810,14 +1810,22 @@ private void setEffectiveParameterUpdates(final ParameterContextDTO parameterCon
parameterEntity = dtoFactory.createParameterEntity(parameterContext, parameter, revisionManager, parameterContextDAO);
}

// Parameter is inherited if either this is the removal of a parameter not directly in this context, or it's parameter not specified directly in the DTO
final boolean isInherited = (parameter == null && !parameterContext.getParameters().containsKey(new ParameterDescriptor.Builder().name(parameterName).build()))
|| (parameter != null && !parameterEntities.containsKey(parameterName));
parameterEntity.getParameter().setInherited(isInherited);
parameterEntity.getParameter().setInherited(isInheritedParameterUpdate(parameter, parameterContext, parameterEntities, parameterName));
parameterContextDto.getParameters().add(parameterEntity);
}
}

static boolean isInheritedParameterUpdate(final Parameter parameter, final ParameterContext parameterContext,
final Map<String, ParameterEntity> originalParameterEntities, final String parameterName) {
if (parameter == null) {
return !parameterContext.getParameters().containsKey(new ParameterDescriptor.Builder().name(parameterName).build());
}

final ParameterEntity originalParameterEntity = originalParameterEntities.get(parameterName);
return originalParameterEntity == null || originalParameterEntity.getParameter() != null
&& Boolean.TRUE.equals(originalParameterEntity.getParameter().getInherited());
}

private void addReferencingComponents(final ControllerServiceNode service, final Set<ComponentNode> affectedComponents, final List<ParameterDTO> affectedParameterDtos,
final boolean includeInactive) {

Expand Down
67 changes: 67 additions & 0 deletions ...ifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/StandardNiFiServiceFacadeTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,7 @@
import org.apache.nifi.web.api.dto.DtoFactory;
import org.apache.nifi.web.api.dto.EntityFactory;
import org.apache.nifi.web.api.dto.ParameterContextDTO;
import org.apache.nifi.web.api.dto.ParameterDTO;
import org.apache.nifi.web.api.dto.ProcessGroupDTO;
import org.apache.nifi.web.api.dto.RemoteProcessGroupDTO;
import org.apache.nifi.web.api.dto.RevisionDTO;
Expand All @@ -148,6 +149,7 @@
import org.apache.nifi.web.api.entity.CopyRequestEntity;
import org.apache.nifi.web.api.entity.CopyResponseEntity;
import org.apache.nifi.web.api.entity.ParameterContextEntity;
import org.apache.nifi.web.api.entity.ParameterEntity;
import org.apache.nifi.web.api.entity.ProcessGroupEntity;
import org.apache.nifi.web.api.entity.SecretsEntity;
import org.apache.nifi.web.api.entity.StatusHistoryEntity;
Expand Down Expand Up @@ -2791,6 +2793,71 @@ public void testVerifyCanReportConnectorBacklogAcceptsBacklogReportingConnector(
serviceFacade.verifyCanReportConnectorBacklog(connectorId);
}

@Test
public void testGeneratedEffectiveParameterUpdateIsInherited() {
final ParameterContext parameterContext = mock(ParameterContext.class);
final Parameter inheritedParameter = new Parameter.Builder().name("fileToIngest").parameterContextId("parent-id").build();

assertTrue(StandardNiFiServiceFacade.isInheritedParameterUpdate(inheritedParameter, parameterContext, Map.of(), "fileToIngest"));
}

@Test
public void testRepeatedEffectiveParameterUpdatePreservesInheritedFlag() {
final ParameterContext parameterContext = mock(ParameterContext.class);
final Parameter parameter = new Parameter.Builder().name("fileToIngest").parameterContextId("parent-id").build();
final ParameterEntity originalEntity = createParameterEntity("fileToIngest", true);

assertTrue(StandardNiFiServiceFacade.isInheritedParameterUpdate(parameter, parameterContext,
Map.of("fileToIngest", originalEntity), "fileToIngest"));
}

@Test
public void testClientSubmittedParameterUpdateIsLocal() {
final ParameterContext parameterContext = mock(ParameterContext.class);
final Parameter parameter = new Parameter.Builder().name("fileToIngest").parameterContextId("child-id").build();
final ParameterEntity originalEntity = createParameterEntity("fileToIngest", false);

assertFalse(StandardNiFiServiceFacade.isInheritedParameterUpdate(parameter, parameterContext,
Map.of("fileToIngest", originalEntity), "fileToIngest"));
}

@Test
public void testClientSubmittedDeletionRemainsLocalWhenInheritedParameterBecomesEffective() {
final ParameterContext parameterContext = mock(ParameterContext.class);
final Parameter inheritedParameter = new Parameter.Builder().name("shared").parameterContextId("parent-id").build();
final ParameterEntity deletionEntity = createParameterEntity("shared", null);

assertFalse(StandardNiFiServiceFacade.isInheritedParameterUpdate(inheritedParameter, parameterContext,
Map.of("shared", deletionEntity), "shared"));
}

@Test
public void testInheritedParameterRemovalUsesLocalParameterDefinitions() {
final ParameterContext parameterContext = mock(ParameterContext.class);
when(parameterContext.getParameters()).thenReturn(Map.of());

assertTrue(StandardNiFiServiceFacade.isInheritedParameterUpdate(null, parameterContext, Map.of(), "fileToIngest"));
}

@Test
public void testLocalParameterRemovalUsesLocalParameterDefinitions() {
final ParameterContext parameterContext = mock(ParameterContext.class);
final ParameterDescriptor descriptor = new ParameterDescriptor.Builder().name("fileToIngest").build();
when(parameterContext.getParameters()).thenReturn(Map.of(descriptor, new Parameter.Builder().descriptor(descriptor).value("value").build()));

assertFalse(StandardNiFiServiceFacade.isInheritedParameterUpdate(null, parameterContext, Map.of(), "fileToIngest"));
}

private static ParameterEntity createParameterEntity(final String name, final Boolean inherited) {
final ParameterDTO parameterDto = new ParameterDTO();
parameterDto.setName(name);
parameterDto.setInherited(inherited);

final ParameterEntity parameterEntity = new ParameterEntity();
parameterEntity.setParameter(parameterDto);
return parameterEntity;
}

@Test
public void testVerifyDeleteAssetWithUnknownAssetIdThrowsResourceNotFound() {
final ParameterContext parameterContext = mock(ParameterContext.class);
Expand Down
117 changes: 117 additions & 0 deletions ...-test-suite/src/test/java/org/apache/nifi/tests/system/parameters/ParameterContextIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.apache.nifi.toolkit.client.NiFiClientException;
import org.apache.nifi.toolkit.client.ParamContextClient;
import org.apache.nifi.web.api.dto.ParameterContextDTO;
import org.apache.nifi.web.api.dto.ParameterContextReferenceDTO;
import org.apache.nifi.web.api.dto.ParameterDTO;
import org.apache.nifi.web.api.dto.ProcessorConfigDTO;
import org.apache.nifi.web.api.entity.AffectedComponentEntity;
Expand All @@ -32,6 +33,7 @@
import org.apache.nifi.web.api.entity.ConnectionEntity;
import org.apache.nifi.web.api.entity.ControllerServiceEntity;
import org.apache.nifi.web.api.entity.ParameterContextEntity;
import org.apache.nifi.web.api.entity.ParameterContextReferenceEntity;
import org.apache.nifi.web.api.entity.ParameterContextUpdateRequestEntity;
import org.apache.nifi.web.api.entity.ParameterEntity;
import org.apache.nifi.web.api.entity.ParameterGroupConfigurationEntity;
Expand All @@ -56,6 +58,7 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;

import static org.junit.jupiter.api.Assertions.assertEquals;
Expand Down Expand Up @@ -1110,6 +1113,120 @@ public void testRemoveInheritedContextWithAssetReference() throws NiFiClientExce
assertEquals(childContext2.getId(), updatedParent.getComponent().getInheritedParameterContexts().get(0).getId());
}

@Test
public void testAddInheritedContextWithAssetReference() throws NiFiClientException, IOException, InterruptedException {
final ParameterContextEntity parentContext = getClientUtil().createParameterContext("parentContext", Map.of("fileToIngest", ""));
final File assetFile = new File("src/test/resources/sample-assets/helloworld.txt");
final AssetEntity asset = createAsset(parentContext.getId(), assetFile);

final ParameterContextUpdateRequestEntity referenceAssetUpdateRequest = getClientUtil().updateParameterAssetReferences(
parentContext, Map.of("fileToIngest", List.of(asset.getAsset().getId())));
getClientUtil().waitForParameterContextRequestToComplete(parentContext.getId(), referenceAssetUpdateRequest.getRequest().getRequestId());

final ParameterContextEntity childContext = getClientUtil().createParameterContext("childContext", Map.of("otherParam", "otherValue"));
final ParameterContextEntity fetchedChild = getNifiClient().getParamContextClient().getParamContext(childContext.getId(), true);
fetchedChild.getComponent().setParameters(Set.of());

final ParameterContextReferenceEntity parentReference = new ParameterContextReferenceEntity();
parentReference.setId(parentContext.getId());
final ParameterContextReferenceDTO parentReferenceDto = new ParameterContextReferenceDTO();
parentReferenceDto.setId(parentContext.getId());
parentReferenceDto.setName(parentContext.getComponent().getName());
parentReference.setComponent(parentReferenceDto);
fetchedChild.getComponent().setInheritedParameterContexts(List.of(parentReference));

final ParameterContextUpdateRequestEntity addInheritanceRequest =
getNifiClient().getParamContextClient().updateParamContext(fetchedChild);
getClientUtil().waitForParameterContextRequestToComplete(childContext.getId(), addInheritanceRequest.getRequest().getRequestId());

final ParameterContextEntity updatedChildLocal = getNifiClient().getParamContextClient().getParamContext(childContext.getId(), false);
assertEquals(1, updatedChildLocal.getComponent().getInheritedParameterContexts().size());
assertEquals(parentContext.getId(), updatedChildLocal.getComponent().getInheritedParameterContexts().getFirst().getId());
assertTrue(updatedChildLocal.getComponent().getParameters().stream()
.noneMatch(parameter -> "fileToIngest".equals(parameter.getParameter().getName())));

final ParameterContextEntity updatedChildEffective = getNifiClient().getParamContextClient().getParamContext(childContext.getId(), true);
final ParameterEntity inheritedAssetParameter = updatedChildEffective.getComponent().getParameters().stream()
.filter(parameter -> "fileToIngest".equals(parameter.getParameter().getName()))
.findFirst()
.orElse(null);

assertNotNull(inheritedAssetParameter);
assertTrue(inheritedAssetParameter.getParameter().getInherited());
assertEquals(parentContext.getId(), inheritedAssetParameter.getParameter().getParameterContext().getId());
assertEquals(asset.getAsset().getId(), inheritedAssetParameter.getParameter().getReferencedAssets().getFirst().getId());

final ParameterContextUpdateRequestEntity resubmitRequest =
getNifiClient().getParamContextClient().updateParamContext(updatedChildEffective);
getClientUtil().waitForParameterContextRequestToComplete(childContext.getId(), resubmitRequest.getRequest().getRequestId());

final ParameterContextEntity resubmittedChild = getNifiClient().getParamContextClient().getParamContext(childContext.getId(), true);
final ParameterEntity resubmittedAssetParameter = resubmittedChild.getComponent().getParameters().stream()
.filter(parameter -> "fileToIngest".equals(parameter.getParameter().getName()))
.findFirst()
.orElse(null);

assertNotNull(resubmittedAssetParameter);
assertTrue(resubmittedAssetParameter.getParameter().getInherited());
assertEquals(parentContext.getId(), resubmittedAssetParameter.getParameter().getParameterContext().getId());
}

@Test
public void testDeleteLocalParameterOverrideRevealsInheritedParameter() throws NiFiClientException, IOException, InterruptedException {
final ParameterContextEntity parentContext = getClientUtil().createParameterContext("overrideParent", Map.of("shared", "parent-value"));
final ParameterContextEntity childContext = getClientUtil().createParameterContext("overrideChild",
Map.of("shared", "child-value"), List.of(parentContext.getId()), null);

final ParameterContextEntity fetchedChild = getNifiClient().getParamContextClient().getParamContext(childContext.getId(), true);
final ParameterEntity localOverride = fetchedChild.getComponent().getParameters().stream()
.filter(parameter -> "shared".equals(parameter.getParameter().getName()))
.findFirst()
.orElse(null);
assertNotNull(localOverride);
assertFalse(Boolean.TRUE.equals(localOverride.getParameter().getInherited()));
assertEquals("child-value", localOverride.getParameter().getValue());

final ParameterDTO deletionDto = new ParameterDTO();
deletionDto.setName("shared");
final ParameterEntity deletionEntity = new ParameterEntity();
deletionEntity.setParameter(deletionDto);
fetchedChild.getComponent().setParameters(Set.of(deletionEntity));

final ParameterContextUpdateRequestEntity deleteRequest = getNifiClient().getParamContextClient().updateParamContext(fetchedChild);
getClientUtil().waitForParameterContextRequestToComplete(childContext.getId(), deleteRequest.getRequest().getRequestId());

final ParameterContextEntity updatedChildLocal = getNifiClient().getParamContextClient().getParamContext(childContext.getId(), false);
assertTrue(updatedChildLocal.getComponent().getParameters().stream()
.noneMatch(parameter -> "shared".equals(parameter.getParameter().getName())));

final ParameterContextEntity updatedChildEffective = getNifiClient().getParamContextClient().getParamContext(childContext.getId(), true);
final ParameterEntity inheritedParameter = updatedChildEffective.getComponent().getParameters().stream()
.filter(parameter -> "shared".equals(parameter.getParameter().getName()))
.findFirst()
.orElse(null);
assertNotNull(inheritedParameter);
assertTrue(inheritedParameter.getParameter().getInherited());
assertEquals("parent-value", inheritedParameter.getParameter().getValue());
assertEquals(parentContext.getId(), inheritedParameter.getParameter().getParameterContext().getId());
}

@Test
public void testUpdateReferencedLocalParameterPreservesRawReference() throws NiFiClientException, IOException, InterruptedException {
final ParameterContextEntity context = getClientUtil().createParameterContext("localReferenceContext",
Map.of("X", "original", "Y", "#{X}"));

final ParameterContextUpdateRequestEntity updateRequest = updateParameterContext(context, "X", "updated");
getClientUtil().waitForParameterContextRequestToComplete(context.getId(), updateRequest.getRequest().getRequestId());

final ParameterContextEntity updatedContext = getNifiClient().getParamContextClient().getParamContext(context.getId(), false);
final Map<String, ParameterDTO> parameters = updatedContext.getComponent().getParameters().stream()
.map(ParameterEntity::getParameter)
.collect(Collectors.toMap(ParameterDTO::getName, Function.identity()));

assertEquals("updated", parameters.get("X").getValue());
assertEquals("#{X}", parameters.get("Y").getValue());
}

@Test
public void testAssetReferenceAfterRestart() throws NiFiClientException, IOException, InterruptedException {
// Create Parameter Context
Expand Down
Loading