diff --git a/fcli-core/fcli-app/src/main/java/com/fortify/cli/app/FortifyCLI.java b/fcli-core/fcli-app/src/main/java/com/fortify/cli/app/FortifyCLI.java index f2ab435cb7..7d25180d7f 100644 --- a/fcli-core/fcli-app/src/main/java/com/fortify/cli/app/FortifyCLI.java +++ b/fcli-core/fcli-app/src/main/java/com/fortify/cli/app/FortifyCLI.java @@ -17,9 +17,9 @@ /** *

This class provides the {@link #main(String[])} entrypoint into the application, - * and also registers some GraalVM features, allowing the application to run properly + * and also registers some GraalVM features, allowing the application to run properly * as GraalVM native images.

- * + * * @author Ruud Senden */ public class FortifyCLI { @@ -33,6 +33,7 @@ public static final void main(String[] args) { private static final int execute(String[] args) { try { + System.out.println("FCLI9 started"); ConsoleHelper.installJAnsiConsole(); return DefaultFortifyCLIRunner.run(args); } finally { diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java index 2f717dd66b..c5ec58d096 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java @@ -30,6 +30,8 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.zip.ZipFile; @@ -53,8 +55,7 @@ import com.fortify.cli.aviator.fpr.utils.SourceDecoders; import com.fortify.cli.aviator.fpr.utils.SourceEncoder; import com.fortify.cli.aviator.fpr.utils.SourceEncoder.SourceEncodeException; -import com.fortify.cli.aviator.util.FprHandle; -import com.fortify.cli.aviator.util.FuzzyContextSearcher; +import com.fortify.cli.aviator.util.*; public class RemediationProcessor { private static final Logger LOG = LoggerFactory.getLogger(RemediationProcessor.class); @@ -64,10 +65,10 @@ public class RemediationProcessor { private final String sourceCodeDirectory; private final ISourceDecoder sourceDecoder; - public record RemediationMetric(int totalRemediations, int appliedRemediations, int skippedRemediations, Set modifiedFiles, + public record RemediationMetric(int totalRemediations, int appliedRemediations, int identicalRemediations,int skippedRemediations, Set modifiedFiles, Map skippedByReason) { - public RemediationMetric(int totalRemediations, int appliedRemediations, int skippedRemediations, Set modifiedFiles) { - this(totalRemediations, appliedRemediations, skippedRemediations, modifiedFiles, Map.of()); + public RemediationMetric(int totalRemediations, int appliedRemediations,int identicalRemediations,int skippedRemediations, Set modifiedFiles) { + this(totalRemediations, appliedRemediations,identicalRemediations, skippedRemediations, modifiedFiles, Map.of()); } } @@ -75,9 +76,28 @@ private record SourceFileContent(String content, Charset charset, String encodin private record PendingFileWrite(String filename, Path filePath, String content, Charset charset, String encodingSource, byte[] updatedBytes) {} + private record AppliedChange(Path filePath, int originalStart, int originalEnd, int resultingStart, int resultingEnd, String remediationId, String originalCode, String newCode) { + private int lineDelta() { return (resultingEnd - resultingStart + 1) - (originalEnd - originalStart + 1); } + } + private record ChangeApplication(String content, AppliedChange appliedChange) {} + + private record RemediationTrace( + String instanceId, + String filename, + int changeIndex, + int declaredLineFrom, + int declaredLineTo, + boolean fileHashMatches, + int previousChanges, + int projectedLineFrom, + int projectedLineTo, + String outcome, + String reason) {} private record RollbackFileWrite(String filename, Path filePath, byte[] originalBytes) {} + private record RemediationKey(String fileName, Path filePath,int lineFrom,int lineTo,String comparisonCode){} + private enum SkipReason { SOURCE_FILE_MISSING("Source file missing"), SOURCE_FILE_OUTSIDE_SOURCE_DIR("Source file outside source directory"), @@ -88,11 +108,14 @@ private enum SkipReason { SOURCE_CONTEXT_NOT_FOUND("Source context not found"), SOURCE_CONTEXT_AMBIGUOUS("Source context matched multiple locations"), ORIGINAL_CODE_NOT_FOUND("Original code not found"), + ANCHOR_MISMATCH("Anchor does not match"), + CONFLICT("Conflicts with another fix"), REMEDIATION_ENCODE_FAILED("Remediation encode failed"), SOURCE_WRITE_FAILED("Source file write failed"), NO_CHANGES("No file changes found"), UNEXPECTED_ERROR("Unexpected remediation processing error"); + private final String displayName; SkipReason(String displayName) { @@ -154,9 +177,12 @@ public RemediationMetric processRemediationXML() { Document remediationDoc; int totalRemediations; int appliedRemediations; + int identicalRemediations = 0; Set modifiedFiles = new LinkedHashSet<>(); Map skippedByReason = new LinkedHashMap<>(); - + Map remediationLookup = new LinkedHashMap<>(); + Map> appliedChangesByFile = new LinkedHashMap<>(); + LOG.debug("in the processRemediationXML method"); // Sanitize and normalize the base source directory path once. String trimmedSourceDir = sourceCodeDirectory.trim(); if (trimmedSourceDir.length() > 1 && @@ -183,10 +209,76 @@ public RemediationMetric processRemediationXML() { totalRemediations = remediationNodes.getLength(); LOG.debug("Loaded {} remediation entries from {}", totalRemediations, remediationPath); appliedRemediations = 0; + for (int i = 0; i < remediationNodes.getLength(); i++) { - Element remediation = (Element) remediationNodes.item(i); - if (processRemediation(remediation, sourceBasePath, fvdlMetadata, modifiedFiles, skippedByReason)) { + LOG.debug("........................"); + Element remediation = + (Element) remediationNodes.item(i); + + String instanceId = + remediation.getAttribute("instanceId"); + LOG.debug("remediation{}",instanceId); + + List remediationKeys = + createRemediationKeys( + remediation, + sourceBasePath); + + LOG.debug( + "Remediation {} generated {} lookup key(s): {}", + instanceId, + remediationKeys.size(), + remediationKeys); + + String identicalInstanceId = null; + + /* + * A remediation is identical only when all of its changes + * match an existing remediation. + */ + if (!remediationKeys.isEmpty()) { + for (String existingInstanceId : + new LinkedHashSet<>(remediationLookup.values())) { + + List existingKeys = + remediationLookup.entrySet().stream() + .filter(entry -> + existingInstanceId.equals(entry.getValue())) + .map(Map.Entry::getKey) + .toList(); + + if (existingKeys.size() == remediationKeys.size() + && existingKeys.containsAll(remediationKeys)) { + identicalInstanceId = existingInstanceId; + break; + } + } + } + + if (identicalInstanceId != null) { + identicalRemediations++; appliedRemediations++; + + LOG.info( + "Identical found: {}", + identicalInstanceId); + + LOG.info( + "Identical Remediation Applied: {} is identical to {}", + instanceId, + identicalInstanceId); + + continue; + } + + if (processRemediation(remediation, sourceBasePath, fvdlMetadata, modifiedFiles, skippedByReason, appliedChangesByFile)) { + + appliedRemediations++; + + for (RemediationKey key : remediationKeys) { + LOG.debug("putting {}",instanceId); + remediationLookup.put(key, instanceId); + } } } @@ -195,29 +287,68 @@ public RemediationMetric processRemediationXML() { throw new AviatorTechnicalException("Error processing remediation.xml file.", e); } catch (AviatorTechnicalException e) { throw e; + } catch (Exception e) { LOG.error("Unexpected error processing remediation.xml: {}", remediationPath, e); throw new AviatorTechnicalException("Unexpected error processing remediations.xml.", e); } + int skippedRemediations = totalRemediations - appliedRemediations; - LOG.info("Auto-remediation summary: total={}, applied={}, skipped={}", totalRemediations, appliedRemediations, skippedRemediations); + LOG.info("Auto-remediation summary: total={}, applied={},indentical={},skipped={}", totalRemediations, appliedRemediations, identicalRemediations,skippedRemediations); + if (!skippedByReason.isEmpty()) { - LOG.info("Skipped remediations by reason: {}", formatSkippedReasons(skippedByReason)); + LOG.info("Skipped remediations by reason: {}",formatSkippedReasons(skippedByReason)); } - return new RemediationMetric(totalRemediations, appliedRemediations, skippedRemediations, modifiedFiles, skippedByReason); + return new RemediationMetric(totalRemediations, appliedRemediations, identicalRemediations, skippedRemediations, modifiedFiles, skippedByReason); } - private boolean processRemediation(Element remediation, Path sourceBasePath, FVDLMetadata fvdlMetadata, - Set modifiedFiles, Map skippedByReason) { + + private boolean processRemediation(Element remediation, Path sourceBasePath, FVDLMetadata fvdlMetadata, Set modifiedFiles, + Map skippedByReason, Map> appliedChangesByFile) { String instanceId = remediation.getAttribute("instanceId"); + List traces = new ArrayList<>(); try { - Map pendingWrites = prepareFileChanges(remediation, sourceBasePath, fvdlMetadata); + List stagedChanges = new ArrayList<>(); + + Map pendingWrites = prepareFileChanges(remediation, sourceBasePath, fvdlMetadata, appliedChangesByFile, stagedChanges, traces); if (pendingWrites.isEmpty()) { recordSkipped(skippedByReason, SkipReason.NO_CHANGES.displayName); return false; } try { commitRemediationWrites(instanceId, pendingWrites, modifiedFiles); + for (AppliedChange change : stagedChanges) { + appliedChangesByFile.computeIfAbsent(change.filePath(), key -> new ArrayList<>()).add(change); + } + + LOG.info( + "========== REMEDIATION TRACE {} ==========", + instanceId); + + for (RemediationTrace trace : traces) { + LOG.info( + "TRACE instance={} file={} change={} declared={}-{} hashMatches={} previousChanges={} resolved={}-{} outcome={} reason={}", + trace.instanceId(), + trace.filename(), + trace.changeIndex(), + trace.declaredLineFrom(), + trace.declaredLineTo(), + trace.fileHashMatches(), + trace.previousChanges(), + trace.projectedLineFrom(), + trace.projectedLineTo(), + trace.outcome(), + trace.reason()); + } + + LOG.info( + "TRACE FINAL instance={} outcome=APPLIED", + instanceId); + + LOG.info( + "========== END REMEDIATION TRACE {} ==========", + instanceId); + return true; } catch (RemediationCommitException e) { rollbackRemediationWrites(instanceId, e.getRollbacks()); @@ -225,8 +356,53 @@ private boolean processRemediation(Element remediation, Path sourceBasePath, FVD } } catch (SkipRemediationException e) { recordSkipped(skippedByReason, skipReasonLabel(e)); + LOG.warn("Skipping remediation {}: {}", instanceId, e.getMessage()); - LOG.debug("Skip reason for remediation {}: {}", instanceId, e.reason.displayName, e); + + LOG.info( + "========== REMEDIATION TRACE {} ==========", + instanceId); + + if (traces.isEmpty()) { + LOG.info( + "TRACE instance={} outcome=SKIPPED reason={} message={}", + instanceId, + e.reason.displayName, + e.getMessage()); + } else { + for (RemediationTrace trace : traces) { + LOG.info( + "TRACE instance={} file={} change={} declared={}-{} hashMatches={} previousChanges={} resolved={}-{} outcome={} reason={}", + trace.instanceId(), + trace.filename(), + trace.changeIndex(), + trace.declaredLineFrom(), + trace.declaredLineTo(), + trace.fileHashMatches(), + trace.previousChanges(), + trace.projectedLineFrom(), + trace.projectedLineTo(), + trace.outcome(), + trace.reason()); + } + + LOG.info( + "TRACE FINAL instance={} outcome=SKIPPED reason={} message={}", + instanceId, + e.reason.displayName, + e.getMessage()); + } + + LOG.info( + "========== END REMEDIATION TRACE {} ==========", + instanceId); + + LOG.debug( + "Skip reason for remediation {}: {}", + instanceId, + e.reason.displayName, + e); + return false; } catch (RollbackRemediationException e) { throw e; @@ -238,8 +414,7 @@ private boolean processRemediation(Element remediation, Path sourceBasePath, FVD } } - private Map prepareFileChanges(Element remediation, Path sourceBasePath, - FVDLMetadata fvdlMetadata) { + private Map prepareFileChanges(Element remediation, Path sourceBasePath, FVDLMetadata fvdlMetadata, Map> appliedChangesByFile, List stagedChanges, List traces) { NodeList fileChangesNodes = remediation.getElementsByTagNameNS(NAMESPACE_URI, "FileChanges"); if (fileChangesNodes.getLength() == 0) { throw new SkipRemediationException(SkipReason.NO_CHANGES, "No file changes found"); @@ -247,14 +422,13 @@ private Map prepareFileChanges(Element remediation, Path Map pendingWrites = new LinkedHashMap<>(); for (int j = 0; j < fileChangesNodes.getLength(); j++) { - processFileChanges(remediation, (Element) fileChangesNodes.item(j), sourceBasePath, fvdlMetadata, pendingWrites); + processFileChanges(remediation, (Element) fileChangesNodes.item(j), sourceBasePath, fvdlMetadata, pendingWrites, appliedChangesByFile, stagedChanges, traces); } return pendingWrites; } - private boolean processFileChanges(Element remediation, Element fileChanges, Path sourceBasePath, FVDLMetadata fvdlMetadata, - Map pendingWrites) { - String instanceId = remediation.getAttribute("instanceId"); + private boolean processFileChanges(Element remediation, Element fileChanges, Path sourceBasePath, FVDLMetadata fvdlMetadata, Map pendingWrites, Map> appliedChangesByFile, List stagedChanges, List traces) {String instanceId = remediation.getAttribute("instanceId"); String filename = getRequiredElementText(fileChanges, "Filename"); Path filePath = sourceBasePath.resolve(filename).normalize(); LOG.debug("Processing remediation {} file change for '{}' resolved to '{}'", instanceId, filename, filePath); @@ -280,8 +454,9 @@ private boolean processFileChanges(Element remediation, Element fileChanges, Pat String updatedContent = sourceFileContent.content(); for (int k = 0; k < changesNodes.getLength(); k++) { - updatedContent = applyChange(instanceId, filename, fileHash, sourceEncoding, updatedContent, - (Element) changesNodes.item(k), k + 1); + ChangeApplication application = applyChange(instanceId, filename, filePath, fileHash, sourceEncoding, updatedContent, (Element) changesNodes.item(k), k + 1, appliedChangesByFile, stagedChanges, traces); + updatedContent = application.content(); + stagedChanges.add(application.appliedChange()); } byte[] updatedBytes = encodeSourceFile(updatedContent, sourceEncoding, filename); pendingWrites.put(filePath, new PendingFileWrite(filename, filePath, updatedContent, sourceEncoding, @@ -291,62 +466,425 @@ private boolean processFileChanges(Element remediation, Element fileChanges, Pat return true; } - private String applyChange(String instanceId, String filename, String fileHash, Charset sourceEncoding, String originalContent, - Element change, int changeIndex) { + private ChangeApplication applyChange(String instanceId, String filename, Path filePath, String fileHash, Charset sourceEncoding, String originalContent, Element change, int changeIndex, Map> appliedChangesByFile, List stagedChanges, List traces) { String lineSeparator = detectLineSeparator(originalContent); String content = normalizeLineEndings(originalContent); - List originalLines = Arrays.asList(content.split("\n", -1)); - LOG.debug("Decoded '{}' using {}; lineSeparator={}, normalizedLines={}", filename, sourceEncoding.name(), - describeLineSeparator(lineSeparator), originalLines.size()); - - int lineFrom = parseRequiredInt(change, "LineFrom"); - int lineTo = parseRequiredInt(change, "LineTo"); - LOG.debug("Remediation {} change {} for '{}' targets lines {}-{}", instanceId, changeIndex, filename, lineFrom, lineTo); - + int declaredLineFrom = parseRequiredInt(change, "LineFrom"); + int declaredLineTo = parseRequiredInt(change, "LineTo"); String calculatedHash = calculateHashBase64(content, "SHA-256"); boolean fileHashMatches = calculatedHash.equals(fileHash); - LOG.debug("Remediation {} hash check for '{}': {}", instanceId, filename, fileHashMatches ? "matched" : "mismatched"); - if (!fileHashMatches) { - LOG.debug("File hash mismatch for remediation {} in {}; searching changed source content", instanceId, filename); + int lineFrom = declaredLineFrom; + int lineTo = declaredLineTo; + List previousChanges = new ArrayList<>(); + List committedChanges = appliedChangesByFile.get(filePath); + if (committedChanges != null) { previousChanges.addAll(committedChanges); } + for (AppliedChange stagedChange : stagedChanges) { if (filePath.equals(stagedChange.filePath())) { previousChanges.add(stagedChange); } } + + + LOG.debug( + "ANCHOR DEBUG: instance={}, file={}, declaredRange={}-{}, fileHashMatches={}, previousChanges={}", + instanceId, + filename, + declaredLineFrom, + declaredLineTo, + fileHashMatches, + previousChanges.size()); + + + + + if (!previousChanges.isEmpty() && !fileHashMatches ) { Element contextElement = getRequiredElement(change, "Context"); - String contextText = contextElement.getTextContent(); - List contextLine = Arrays.asList(contextText.split("\\r?\\n")); - int contextLineFrom = fuzzySearchContext(instanceId, filename, originalLines, contextLine); - if (contextLineFrom == -1) { - LOG.debug("Context search failed for remediation {} in {}; context lines={}, source lines={}", instanceId, filename, - contextLine.size(), originalLines.size()); - throw new SkipRemediationException(SkipReason.SOURCE_CONTEXT_NOT_FOUND, "Source context not found for file '" + filename + - "'; file may have changed or remediation may overlap a previous change"); + + LOG.debug("ANCHOR DEBUG: Context for instance={}: [{}]", + instanceId, + contextElement.getTextContent()); + + LOG.debug( + "ANCHOR DEBUG: Context before={}, after={}", + contextElement.getAttribute("before"), + contextElement.getAttribute("after")); + + LOG.debug("ANCHOR DEBUG: Previous changes for '{}':", filename); + + for (AppliedChange previousChange : previousChanges) { + LOG.debug( + "ANCHOR DEBUG: remediation={}, original={}-{}, resulting={}-{}, delta={}", + previousChange.remediationId(), + previousChange.originalStart(), + previousChange.originalEnd(), + previousChange.resultingStart(), + previousChange.resultingEnd(), + previousChange.lineDelta()); } - LOG.debug("Context for remediation {} in {} matched at line {}", instanceId, filename, contextLineFrom + 1); + int[] projectedRange = projectLineRange( + + declaredLineFrom, + declaredLineTo, + previousChanges, + instanceId, + filename); + + lineFrom = projectedRange[0]; + lineTo = projectedRange[1]; + + validateLineRange( + lineFrom, + lineTo, + originalLines.size(), + filename); + + try { + verifyOriginalCodeAtRange( + instanceId, + filename, + originalLines, + lineFrom, + lineTo, + change); + } catch (SkipRemediationException e) { + if (e.reason != SkipReason.ANCHOR_MISMATCH) { + throw e; + } + + LOG.debug("=== ANCHOR MISMATCH - TRYING REANCHOR ==="); + LOG.debug("InstanceId: {}", instanceId); + LOG.debug("=== ANCHOR MISMATCH - TRYING REANCHOR ==="); + LOG.debug("InstanceId: {}", instanceId); + LOG.debug("Filename: {}", filename); + LOG.debug("Projected range: {}-{}", lineFrom, lineTo); + int debugStart = Math.max(1, lineFrom - 10); + int debugEnd = Math.min(originalLines.size(), lineTo + 10); + + LOG.debug( + "ANCHOR DEBUG: Source window {}-{}", + debugStart, + debugEnd); + + for (int line = debugStart; line <= debugEnd; line++) { + LOG.debug( + "ANCHOR DEBUG: source[{}] = [{}]", + line, + originalLines.get(line - 1)); + } + + int[] recoveredRange = findOriginalCodeUsingContext( + instanceId, + filename, + originalLines, + change, + lineFrom, + lineTo); + + if (recoveredRange[0] == -1) { + LOG.debug("REANCHOR FAILED"); + throw e; + } + + lineFrom = recoveredRange[0] + 1; + lineTo = recoveredRange[1] + 1; + + LOG.debug( + "REANCHOR SUCCESS: using range {}-{}", + lineFrom, + lineTo); + + validateLineRange( + lineFrom, + lineTo, + originalLines.size(), + filename); + + verifyOriginalCodeAtRange( + instanceId, + filename, + originalLines, + lineFrom, + lineTo, + change); + } + } else if (!fileHashMatches) { + Element contextElement = getRequiredElement(change, "Context"); + List contextLine = Arrays.asList(contextElement.getTextContent().split("\\r?\\n")); + int contextLineFrom = fuzzySearchContext(instanceId, filename, originalLines, contextLine); + if (contextLineFrom == -1) { throw new SkipRemediationException(SkipReason.SOURCE_CONTEXT_NOT_FOUND, "Source context not found for file '" + filename + "'"); } String originalCodeText = getRequiredElementText(change, "OriginalCode"); List originalCodeLine = Arrays.asList(originalCodeText.split("\\r?\\n")); int contextBefore = parseRequiredContextAttribute(contextElement, "before"); int contextAfter = parseRequiredContextAttribute(contextElement, "after"); - int[] lineFromTo = fuzzySearchOriginalCode(instanceId, filename, originalLines, originalCodeLine, - contextLineFrom, contextLine.size(), contextBefore, contextAfter); + + LOG.debug("=== FUZZY ORIGINAL SEARCH ==="); + LOG.debug("InstanceId:{}",instanceId); + LOG.debug("Declared range: {} {}",declaredLineFrom, declaredLineTo); + LOG.debug("File hash: {}", fileHash); + LOG.debug("Calculated hash: {}", calculatedHash); + LOG.debug("OriginalCode:\n{}", originalCodeText); + LOG.debug("Context:\n{}", contextElement.getTextContent()); + LOG.debug("Context matches from: {}", contextLineFrom); + LOG.debug("=============================="); + int[] lineFromTo = fuzzySearchOriginalCode(instanceId, filename, originalLines, originalCodeLine, contextLineFrom, contextLine.size(), contextBefore, contextAfter); if (lineFromTo[0] == -1 || lineFromTo[1] == -1) { - LOG.debug("Original code search failed for remediation {} in {}; context line={}, original code lines={}, source lines={}", - instanceId, filename, contextLineFrom + 1, originalCodeLine.size(), originalLines.size()); - throw new SkipRemediationException(SkipReason.ORIGINAL_CODE_NOT_FOUND, "Original code not found for file '" + filename + - "'; file may have changed or remediation may overlap a previous change"); + LOG.debug("=== ORIGINAL CODE NOT FOUND ==="); + LOG.debug("InstanceId: {}", instanceId); + LOG.debug("Filename: {}", filename); + LOG.debug("FilePath: {}", filePath); + LOG.debug("OriginalCode:\n{}", originalCodeText); + LOG.debug("Declared lines: {}-{}", declaredLineFrom, declaredLineTo); + LOG.debug("Context line: {}", contextLineFrom); + LOG.debug("==============================="); + throw new SkipRemediationException(SkipReason.ORIGINAL_CODE_NOT_FOUND, "Original code not found for file '" + filename + "'"); } lineFrom = lineFromTo[0] + 1; lineTo = lineFromTo[1] + 1; - LOG.debug("Original code for remediation {} in {} matched at lines {}-{}", instanceId, filename, lineFrom, lineTo); } - validateLineRange(lineFrom, lineTo, originalLines.size(), filename); - List newCodeLines = Arrays.asList(getRequiredElementText(change, "NewCode").split("\n")); + List newCodeLines = getCodeLines(change, "NewCode"); List updatedLines = new ArrayList<>(); updatedLines.addAll(originalLines.subList(0, lineFrom - 1)); updatedLines.addAll(newCodeLines); updatedLines.addAll(originalLines.subList(lineTo, originalLines.size())); - LOG.debug("Staged remediation {} change {} for '{}' using FVDL encoding {}; updatedLines={}", instanceId, changeIndex, - filename, sourceEncoding.name(), updatedLines.size()); - return String.join(lineSeparator, updatedLines); + String updatedContent = String.join(lineSeparator, updatedLines); + AppliedChange appliedChange = new AppliedChange(filePath, declaredLineFrom, declaredLineTo, lineFrom, lineFrom + newCodeLines.size() - 1, instanceId, getRequiredElementText(change, "OriginalCode"), getRequiredElementText(change, "NewCode")); + + traces.add( + new RemediationTrace( + instanceId, + filename, + changeIndex, + declaredLineFrom, + declaredLineTo, + fileHashMatches, + previousChanges.size(), + lineFrom, + lineTo, + "APPLIED", + "OriginalCode matched")); + + return new ChangeApplication(updatedContent, appliedChange); + } + private int[] projectLineRange( + int originalStart, + int originalEnd, + List appliedChanges, + String instanceId, + String filename) { + + int projectedStart = originalStart; + int projectedEnd = originalEnd; + + for (int i = 0; i < appliedChanges.size(); i++) { + AppliedChange applied = appliedChanges.get(i); + + /* + * Project this previous change's physical range forward + * through all changes that were applied after it. + */ + int appliedStart = applied.resultingStart(); + int appliedEnd = applied.resultingEnd(); + + for (int j = i + 1; j < appliedChanges.size(); j++) { + AppliedChange later = appliedChanges.get(j); + + if (later.originalEnd() < applied.originalStart()) { + int delta = later.lineDelta(); + appliedStart += delta; + appliedEnd += delta; + } + } + + /* + * A previous change completely before this remediation + * shifts the target's physical location. + */ + if (originalStart > applied.originalEnd()) { + int delta = applied.lineDelta(); + + projectedStart += delta; + projectedEnd += delta; + + continue; + } + + /* + * Compare against the previous change's CURRENT physical + * location, not the stale location recorded when it ran. + */ + boolean physicalOverlap = + projectedStart <= appliedEnd + && projectedEnd >= appliedStart; + + if (physicalOverlap) { + int overlapStart = + Math.max(projectedStart, appliedStart); + + int overlapEnd = + Math.min(projectedEnd, appliedEnd); + + throw new SkipRemediationException( + SkipReason.CONFLICT, + "Remediation '" + instanceId + + "' conflicts with remediation '" + + applied.remediationId() + + "' in file '" + filename + + "'; overlapping physical lines " + + overlapStart + "-" + overlapEnd); + } + } + + return new int[] { + projectedStart, + projectedEnd + }; + } + /**private void verifyOriginalCodeAtRange( + String instanceId, + String filename, + List sourceLines, + int lineFrom, + int lineTo, + Element change) { + + List expectedLines = + getCodeLines(change, "OriginalCode"); + + List actualLines = + sourceLines.subList(lineFrom - 1, lineTo); + + List normalizedActual = + actualLines.stream() + .map(String::trim) + .toList(); + + List normalizedExpected = + expectedLines.stream() + .map(String::trim) + .toList(); + + if (!normalizedExpected.equals(normalizedActual)) { + throw new SkipRemediationException( + SkipReason.ANCHOR_MISMATCH, + "Anchor does not match for remediation '" + + instanceId + + "' in file '" + + filename + + "' at lines " + + lineFrom + "-" + lineTo); + } + }**/ + private void verifyOriginalCodeAtRange( + String instanceId, + String filename, + List sourceLines, + int lineFrom, + int lineTo, + Element change) { + + String originalCode = + normalizeLineEndings( + getRequiredElementText(change, "OriginalCode")); + + List expectedLines = + Arrays.asList(originalCode.split("\n", -1)); + + List actualLines = + sourceLines.subList(lineFrom - 1, lineTo); + + if (expectedLines.size() != actualLines.size()) { + + LOG.debug("========== ANCHOR SIZE MISMATCH =========="); + LOG.debug("Remediation instanceId: {}", instanceId); + LOG.debug("Filename: {}", filename); + LOG.debug("Expected range: {}-{}", lineFrom, lineTo); + LOG.debug("Expected line count: {}", expectedLines.size()); + LOG.debug("Actual line count: {}", actualLines.size()); + + LOG.debug("Expected OriginalCode:"); + for (int i = 0; i < expectedLines.size(); i++) { + LOG.debug( + " expected[{}] = [{}]", + i, + expectedLines.get(i)); + } + + LOG.debug("Actual source lines:"); + for (int i = 0; i < actualLines.size(); i++) { + LOG.debug( + " actual[{}] = [{}]", + i, + actualLines.get(i)); + } + + LOG.debug("=========================================="); + + throw new SkipRemediationException( + SkipReason.ANCHOR_MISMATCH, + "Anchor does not match for remediation '" + + instanceId + + "' in file '" + + filename + + "' at lines " + + lineFrom + + "-" + + lineTo); + } + for (int i = 0; i < expectedLines.size(); i++) { + String expected = expectedLines.get(i).strip(); + String actual = actualLines.get(i).strip(); + + if (!expected.equals(actual)) { + + LOG.debug("========== ANCHOR CONTENT MISMATCH =========="); + LOG.debug("Remediation instanceId: {}", instanceId); + LOG.debug("Filename: {}", filename); + LOG.debug("Range: {}-{}", lineFrom, lineTo); + LOG.debug("Mismatch at relative line: {}", i); + LOG.debug("Expected: [{}]", expected); + LOG.debug("Actual: [{}]", actual); + LOG.debug("=============================================="); + + throw new SkipRemediationException( + SkipReason.ANCHOR_MISMATCH, + "Anchor does not match for remediation '" + + instanceId + + "' in file '" + + filename + + "' at lines " + + lineFrom + + "-" + + lineTo); + } + } + } + + private List getCodeLines( + Element change, + String elementName) { + + String code = + normalizeLineEndings( + getRequiredElementText(change, elementName)); + + String[] lines = code.split("\n", -1); + + int start = 0; + int end = lines.length - 1; + + while (start <= end && lines[start].isBlank()) { + start++; + } + + while (end >= start && lines[end].isBlank()) { + end--; + } + + if (start > end) { + return List.of(); + } + + return Arrays.asList( + Arrays.copyOfRange(lines, start, end + 1)); } private SourceFileContent getPendingOrSourceContent(Path filePath, String filename, FVDLMetadata fvdlMetadata, @@ -398,6 +936,13 @@ private int fuzzySearchContext(String instanceId, String filename, List String candidateLines = matches.stream() .map(line -> String.valueOf(line + 1)) .collect(Collectors.joining(", ")); + + LOG.debug("=== SOURCE CONTEXT AMBIGUOUS ==="); + LOG.debug("InstanceId: {}", instanceId); + LOG.debug("Filename: {}", filename); + LOG.debug("SourceContext:\n{}", String.join("\n", contextLine)); + LOG.debug("Matching locations: {}", matches); + LOG.debug("==============================="); throw new SkipRemediationException(SkipReason.SOURCE_CONTEXT_AMBIGUOUS, "Source context matched multiple locations in file '" + filename + "'; candidate lines: " + candidateLines); } @@ -408,20 +953,57 @@ private int fuzzySearchContext(String instanceId, String filename, List } } - private int[] fuzzySearchOriginalCode(String instanceId, String filename, List originalLines, List originalCodeLine, - int contextLineFrom, int contextLineCount, int contextBefore, int contextAfter) { + private int[] fuzzySearchOriginalCode( + String instanceId, + String filename, + List originalLines, + List originalCodeLine, + int contextLineFrom, + int contextLineCount, + int contextBefore, + int contextAfter) { + + /* + * First try the location implied by the context. + * This keeps the context as the primary anchor. + */ int contextStart = contextLineFrom + contextBefore; int contextEnd = contextLineFrom + contextLineCount - contextAfter; - if (contextStart < 0 || contextStart >= contextEnd || contextEnd > originalLines.size()) { - return new int[] {-1, -1}; - } - int[] lineFromTo = FuzzyContextSearcher.fuzzySearchOriginalCode( - originalLines.subList(contextStart, contextEnd), originalCodeLine, 0, 0); - if (lineFromTo[0] == -1 || lineFromTo[1] == -1) { - return lineFromTo; + if (contextStart >= 0 + && contextStart < contextEnd + && contextEnd <= originalLines.size()) { + + int[] match = FuzzyContextSearcher.fuzzySearchOriginalCode( + originalLines.subList(contextStart, contextEnd), + originalCodeLine, + 0, + 0); + + if (match[0] != -1 && match[1] != -1) { + return new int[] { + match[0] + contextStart, + match[1] + contextStart + }; + } } - return new int[] {lineFromTo[0] + contextStart, lineFromTo[1] + contextStart}; + + /* + * Context boundaries can become unreliable when blank lines, + * formatting changes, or inserted lines are involved. + * + * Fall back to searching the complete source file. + */ + LOG.debug( + "Original code not found inside context window; searching entire source for remediation {} in '{}'", + instanceId, + filename); + + return FuzzyContextSearcher.fuzzySearchOriginalCode( + originalLines, + originalCodeLine, + 2, + 0); } private boolean isFilePresent(Path path) { @@ -577,4 +1159,416 @@ private String formatSkippedReasons(Map skippedByReason) { skippedByReason.forEach((reason, count) -> parts.add(reason + "=" + count)); return String.join(", ", parts); } + + private RemediationKey createRemediationKey( + Element fileChanges, + Element change, + Path sourceBasePath, + String comparisonCode) { + + String fileName = getRequiredElementText(fileChanges, "Filename"); + Path filePath = sourceBasePath.resolve(fileName).normalize(); + + int lineFrom = parseRequiredInt(change, "LineFrom"); + int lineTo = parseRequiredInt(change, "LineTo"); + + return new RemediationKey( + fileName, + filePath, + lineFrom, + lineTo, + comparisonCode + ); + } + + + private String trimBlankLines(String content) { + String[] lines = content.split("\\R", -1); + + int start = 0; + int end = lines.length - 1; + + while (start <= end && lines[start].isBlank()) { + start++; + } + + while (end >= start && lines[end].isBlank()) { + end--; + } + + if (start > end) { + return ""; + } + + return String.join( + System.lineSeparator(), + Arrays.copyOfRange(lines, start, end + 1)); + } + + private String normalizeProposedCode(String content, String fileName) { + if (content == null) { + return null; + } + + String language = FileTypeLanguageMapperUtil.getProgrammingLanguage( + FileUtil.getFileExtension(fileName)); + + String commentSymbol = + LanguageCommentMapperUtil.getProgrammingLanguageComment(language); + + if ("Unknown".equals(commentSymbol)) { + return trimBlankLines(content); + } + + String closingToken = commentSymbol.equals("" + : commentSymbol.equals("<%--") ? "--%>" + : null; + + Pattern markerPattern = Pattern.compile( + "[ \\t]*" + Pattern.quote(commentSymbol) + " L\\d+" + + (closingToken != null + ? "[ \\t]*" + Pattern.quote(closingToken) + : "") + + "[ \\t]*$"); + + String[] lines = content.split("\\R", -1); + StringBuilder result = new StringBuilder(); + + for (int i = 0; i < lines.length; i++) { + Matcher matcher = markerPattern.matcher(lines[i]); + + result.append( + matcher.find() + ? lines[i].substring(0, matcher.start()) + : lines[i]); + + if (i < lines.length - 1) { + result.append(System.lineSeparator()); + } + } + + return trimBlankLines(result.toString()); + } + + private String createComparisonCode(String normalizedCode, String fileName) { + if (normalizedCode == null) { + return null; + } + + String language = FileTypeLanguageMapperUtil.getProgrammingLanguage( + FileUtil.getFileExtension(fileName)); + + String commentSymbol = + LanguageCommentMapperUtil.getProgrammingLanguageComment(language); + + if ("Unknown".equals(commentSymbol)) { + return normalizedCode.replaceAll("\\s+", ""); + } + + String comparisonCode = normalizedCode; + + // Remove block comments + String closingToken = commentSymbol.equals("" + : commentSymbol.equals("<%--") ? "--%>" + : null; + + if (closingToken != null) { + comparisonCode = comparisonCode.replaceAll( + "(?s)" + Pattern.quote(commentSymbol) + + ".*?" + Pattern.quote(closingToken), + ""); + } else if ("//".equals(commentSymbol)) { + comparisonCode = comparisonCode.replaceAll( + "(?m)" + Pattern.quote(commentSymbol) + ".*$", + ""); + comparisonCode = comparisonCode.replaceAll( + "(?s)/\\*.*?\\*/", + ""); + } else if ("#".equals(commentSymbol)) { + comparisonCode = comparisonCode.replaceAll( + "(?m)" + Pattern.quote(commentSymbol) + ".*$", + ""); + } + + // Normalize whitespace + return comparisonCode.replaceAll("\\s+", ""); + } + + private List createRemediationKeys( + Element remediation, + Path sourceBasePath) { + + List keys = new ArrayList<>(); + + NodeList fileChangesNodes = + remediation.getElementsByTagNameNS(NAMESPACE_URI, "FileChanges"); + + for (int i = 0; i < fileChangesNodes.getLength(); i++) { + Element fileChanges = (Element) fileChangesNodes.item(i); + + NodeList changeNodes = + fileChanges.getElementsByTagNameNS(NAMESPACE_URI, "Change"); + + for (int j = 0; j < changeNodes.getLength(); j++) { + Element change = (Element) changeNodes.item(j); + + String fileName = + getRequiredElementText(fileChanges, "Filename"); + + String newCode = + getRequiredElementText(change, "NewCode"); + + String normalizedCode = + normalizeProposedCode(newCode, fileName); + + String comparisonCode = + createComparisonCode(normalizedCode, fileName); + + keys.add(createRemediationKey( + fileChanges, + change, + sourceBasePath, + comparisonCode)); + } + } + + return keys; + } + + private int[] findExactOriginalCodeNearRange( + List sourceLines, + int lineFrom, + int lineTo, + String originalCode, + int radius) { + + List expectedLines = + Arrays.asList( + normalizeLineEndings(originalCode) + .split("\n", -1)); + + int expectedCount = expectedLines.size(); + + int targetStart = lineFrom - 1; + + int searchStart = + Math.max(0, targetStart - radius); + + int searchEnd = + Math.min( + sourceLines.size() - expectedCount, + targetStart + radius); + + LOG.debug( + "REANCHOR SEARCH: projected={} - {}, search={} - {}", + lineFrom, + lineTo, + searchStart + 1, + searchEnd + expectedCount); + + int matchStart = -1; + + for (int i = searchStart; i <= searchEnd; i++) { + + boolean matches = true; + + for (int j = 0; j < expectedCount; j++) { + + String expected = + expectedLines.get(j) + .trim() + .replaceAll("\\s+", " "); + + String actual = + sourceLines.get(i + j) + .trim() + .replaceAll("\\s+", " "); + + if (!expected.equalsIgnoreCase(actual)) { + matches = false; + break; + } + } + + if (matches) { + + LOG.debug( + "REANCHOR CANDIDATE: {}-{}", + i + 1, + i + expectedCount); + + /* + * More than one normalized match means we cannot + * safely determine which occurrence is the remediation + * target. + */ + if (matchStart != -1) { + + LOG.debug( + "REANCHOR AMBIGUOUS: second match {}-{}", + i + 1, + i + expectedCount); + + return new int[] {-1, -1}; + } + + matchStart = i; + } + } + + if (matchStart == -1) { + LOG.debug("REANCHOR NOT FOUND"); + return new int[] {-1, -1}; + } + + LOG.debug( + "REANCHOR SUCCESS: {}-{}", + matchStart + 1, + matchStart + expectedCount); + + return new int[] { + matchStart, + matchStart + expectedCount - 1 + }; + } + + private int[] findOriginalCodeUsingContext( + String instanceId, + String filename, + List sourceLines, + Element change, + int projectedLineFrom, + int projectedLineTo) { + + Element contextElement = getRequiredElement(change, "Context"); + + List contextLines = + Arrays.asList( + normalizeLineEndings( + contextElement.getTextContent()) + .split("\n", -1)); + + int contextBefore = + parseRequiredContextAttribute( + contextElement, + "before"); + + int contextAfter = + parseRequiredContextAttribute( + contextElement, + "after"); + + int projectedContextStart = + Math.max( + 0, + projectedLineFrom - 1 - contextBefore); + + int projectedContextEnd = + Math.min( + sourceLines.size(), + projectedLineTo + contextAfter); + + LOG.debug( + "CONTEXT REANCHOR SEARCH: projected={} - {}, context={} - {}", + projectedLineFrom, + projectedLineTo, + projectedContextStart + 1, + projectedContextEnd); + + int[] contextRange = + findContextNearRange( + sourceLines, + contextLines, + projectedContextStart, + projectedContextEnd, + 100); + + if (contextRange[0] == -1) { + LOG.debug( + "CONTEXT REANCHOR FAILED for remediation {}", + instanceId); + + return new int[] {-1, -1}; + } + + int contextStart = contextRange[0]; + int contextEnd = contextRange[1]; + + List originalCodeLines = + Arrays.asList( + normalizeLineEndings( + getRequiredElementText(change, "OriginalCode")) + .split("\n", -1)); + + int[] originalRange = + FuzzyContextSearcher.fuzzySearchOriginalCode( + sourceLines.subList(contextStart, contextEnd), + originalCodeLines, + 0, + 0); + + if (originalRange[0] == -1) { + LOG.debug( + "ORIGINAL CODE NOT FOUND INSIDE CONTEXT REANCHOR for remediation {}", + instanceId); + + return new int[] {-1, -1}; + } + + return new int[] { + originalRange[0] + contextStart, + originalRange[1] + contextStart + }; + } + + private int[] findContextNearRange( + List sourceLines, + List contextLines, + int projectedStart, + int projectedEnd, + int radius) { + + int searchStart = + Math.max( + 0, + projectedStart - radius); + + int searchEnd = + Math.min( + sourceLines.size(), + projectedEnd + radius); + + List searchLines = + sourceLines.subList( + searchStart, + searchEnd); + + List matches; + + try { + matches = + FuzzyContextSearcher.fuzzySearchContextMatches( + searchLines, + contextLines, + 0); + } catch (IOException e) { + return new int[] {-1, -1}; + } + + if (matches.size() != 1) { + LOG.debug( + "CONTEXT REANCHOR MATCHES: {}", + matches); + + return new int[] {-1, -1}; + } + + int matchStart = + searchStart + matches.get(0); + + return new int[] { + matchStart, + matchStart + contextLines.size() + }; + } } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java1 b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java1 new file mode 100644 index 0000000000..fba8339812 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java1 @@ -0,0 +1,1469 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.processor; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.zip.ZipFile; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +import com.fortify.cli.aviator._common.exception.AviatorSimpleException; +import com.fortify.cli.aviator._common.exception.AviatorTechnicalException; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder.DecodeResult; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder.SourceDecodeException; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; +import com.fortify.cli.aviator.fpr.utils.SourceEncoder; +import com.fortify.cli.aviator.fpr.utils.SourceEncoder.SourceEncodeException; +import com.fortify.cli.aviator.util.*; + +public class RemediationProcessor { + private static final Logger LOG = LoggerFactory.getLogger(RemediationProcessor.class); + private static final String NAMESPACE_URI = "xmlns://www.fortify.com/schema/remediations"; + + private final FprHandle fprHandle; + private final String sourceCodeDirectory; + private final ISourceDecoder sourceDecoder; + + public record RemediationMetric(int totalRemediations, int appliedRemediations, int identicalRemediations,int skippedRemediations, Set modifiedFiles, + Map skippedByReason) { + public RemediationMetric(int totalRemediations, int appliedRemediations,int identicalRemediations,int skippedRemediations, Set modifiedFiles) { + this(totalRemediations, appliedRemediations,identicalRemediations, skippedRemediations, modifiedFiles, Map.of()); + } + } + + private record SourceFileContent(String content, Charset charset, String encodingSource) {} + + private record PendingFileWrite(String filename, Path filePath, String content, Charset charset, String encodingSource, + byte[] updatedBytes) {} + private record AppliedChange(Path filePath, int originalStart, int originalEnd, int resultingStart, int resultingEnd, String remediationId, String originalCode, String newCode) { + private int lineDelta() { return (resultingEnd - resultingStart + 1) - (originalEnd - originalStart + 1); } + } + private record ChangeApplication(String content, AppliedChange appliedChange) {} + + private record RollbackFileWrite(String filename, Path filePath, byte[] originalBytes) {} + + private record RemediationKey(String fileName, Path filePath,int lineFrom,int lineTo,String comparisonCode){} + + private enum SkipReason { + SOURCE_FILE_MISSING("Source file missing"), + SOURCE_FILE_OUTSIDE_SOURCE_DIR("Source file outside source directory"), + SOURCE_READ_FAILED("Source file read failed"), + SOURCE_DECODE_FAILED("Source file decode failed"), + REMEDIATION_DATA_INVALID("Remediation data invalid"), + REMEDIATION_LINE_RANGE_INVALID("Remediation line range invalid"), + SOURCE_CONTEXT_NOT_FOUND("Source context not found"), + SOURCE_CONTEXT_AMBIGUOUS("Source context matched multiple locations"), + ORIGINAL_CODE_NOT_FOUND("Original code not found"), + ANCHOR_MISMATCH("Anchor does not match"), + CONFLICT("Conflicts with another fix"), + REMEDIATION_ENCODE_FAILED("Remediation encode failed"), + SOURCE_WRITE_FAILED("Source file write failed"), + NO_CHANGES("No file changes found"), + UNEXPECTED_ERROR("Unexpected remediation processing error"); + + + private final String displayName; + + SkipReason(String displayName) { + this.displayName = displayName; + } + } + + private static class SkipRemediationException extends AviatorSimpleException { + private static final long serialVersionUID = 1L; + + private final SkipReason reason; + + SkipRemediationException(SkipReason reason, String message) { + super(message); + this.reason = reason; + } + + SkipRemediationException(SkipReason reason, String message, Throwable cause) { + super(message, cause); + this.reason = reason; + } + } + + private static class RemediationCommitException extends AviatorTechnicalException { + private static final long serialVersionUID = 1L; + + private final List rollbacks; + + RemediationCommitException(String message, Throwable cause, List rollbacks) { + super(message, cause); + this.rollbacks = rollbacks; + } + + List getRollbacks() { + return rollbacks; + } + } + + private static class RollbackRemediationException extends AviatorTechnicalException { + private static final long serialVersionUID = 1L; + + RollbackRemediationException(String message, Throwable cause) { + super(message, cause); + } + } + + public RemediationProcessor(FprHandle fprHandle, String sourceCodeDirectory) { + this(fprHandle, sourceCodeDirectory, SourceDecoders.defaults()); + } + + public RemediationProcessor(FprHandle fprHandle, String sourceCodeDirectory, ISourceDecoder sourceDecoder) { + this.fprHandle = fprHandle; + this.sourceCodeDirectory = sourceCodeDirectory; + this.sourceDecoder = Objects.requireNonNull(sourceDecoder, "sourceDecoder"); + } + + public RemediationMetric processRemediationXML() { + Path remediationPath = fprHandle.getPath("/remediations.xml"); + Document remediationDoc; + int totalRemediations; + int appliedRemediations; + int identicalRemediations = 0; + Set modifiedFiles = new LinkedHashSet<>(); + Map skippedByReason = new LinkedHashMap<>(); + Map remediationLookup = new LinkedHashMap<>(); + Map> appliedChangesByFile = new LinkedHashMap<>(); + LOG.debug("in the processRemediationXML method"); + // Sanitize and normalize the base source directory path once. + String trimmedSourceDir = sourceCodeDirectory.trim(); + if (trimmedSourceDir.length() > 1 && + ((trimmedSourceDir.startsWith("\"") && trimmedSourceDir.endsWith("\"")) || + (trimmedSourceDir.startsWith("'") && trimmedSourceDir.endsWith("'")))) { + trimmedSourceDir = trimmedSourceDir.substring(1, trimmedSourceDir.length() - 1); + } + final Path sourceBasePath = Paths.get(trimmedSourceDir).toAbsolutePath().normalize(); + LOG.debug("Applying remediations from {} to source directory {}", remediationPath, sourceBasePath); + final FVDLMetadata fvdlMetadata = loadFvdlMetadata(); + + try (InputStream remediationStream = Files.newInputStream(remediationPath)) { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setNamespaceAware(true); + factory.setFeature("http://xml.org/sax/features/external-general-entities", false); + factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false); + factory.setXIncludeAware(false); + factory.setExpandEntityReferences(false); + DocumentBuilder builder = factory.newDocumentBuilder(); + remediationDoc = builder.parse(remediationStream); + + NodeList remediationNodes = remediationDoc.getElementsByTagNameNS(NAMESPACE_URI, "Remediation"); + totalRemediations = remediationNodes.getLength(); + LOG.debug("Loaded {} remediation entries from {}", totalRemediations, remediationPath); + appliedRemediations = 0; + + for (int i = 0; i < remediationNodes.getLength(); i++) { + LOG.debug("........................"); + Element remediation = + (Element) remediationNodes.item(i); + + String instanceId = + remediation.getAttribute("instanceId"); + LOG.debug("remediation{}",instanceId); + + List remediationKeys = + createRemediationKeys( + remediation, + sourceBasePath); + + LOG.debug( + "Remediation {} generated {} lookup key(s): {}", + instanceId, + remediationKeys.size(), + remediationKeys); + + String identicalInstanceId = null; + + /* + * A remediation is identical only when all of its changes + * match an existing remediation. + */ + if (!remediationKeys.isEmpty()) { + for (String existingInstanceId : + new LinkedHashSet<>(remediationLookup.values())) { + + List existingKeys = + remediationLookup.entrySet().stream() + .filter(entry -> + existingInstanceId.equals(entry.getValue())) + .map(Map.Entry::getKey) + .toList(); + + if (existingKeys.size() == remediationKeys.size() + && existingKeys.containsAll(remediationKeys)) { + identicalInstanceId = existingInstanceId; + break; + } + } + } + + if (identicalInstanceId != null) { + identicalRemediations++; + appliedRemediations++; + + LOG.info( + "Identical found: {}", + identicalInstanceId); + + LOG.info( + "Identical Remediation Applied: {} is identical to {}", + instanceId, + identicalInstanceId); + + continue; + } + + if (processRemediation(remediation, sourceBasePath, fvdlMetadata, modifiedFiles, skippedByReason, appliedChangesByFile)) { + + appliedRemediations++; + + for (RemediationKey key : remediationKeys) { + LOG.debug("putting {}",instanceId); + remediationLookup.put(key, instanceId); + } + } + } + + } catch (ParserConfigurationException | SAXException | IOException e) { + LOG.error("Error parsing remediations.xml file: {}", remediationPath, e); + throw new AviatorTechnicalException("Error processing remediation.xml file.", e); + } catch (AviatorTechnicalException e) { + throw e; + + } catch (Exception e) { + LOG.error("Unexpected error processing remediation.xml: {}", remediationPath, e); + throw new AviatorTechnicalException("Unexpected error processing remediations.xml.", e); + } + + int skippedRemediations = totalRemediations - appliedRemediations; + LOG.info("Auto-remediation summary: total={}, applied={},indentical={},skipped={}", totalRemediations, appliedRemediations, identicalRemediations,skippedRemediations); + + if (!skippedByReason.isEmpty()) { + LOG.info("Skipped remediations by reason: {}",formatSkippedReasons(skippedByReason)); + } + return new RemediationMetric(totalRemediations, appliedRemediations, identicalRemediations, skippedRemediations, modifiedFiles, skippedByReason); + } + + + private boolean processRemediation(Element remediation, Path sourceBasePath, FVDLMetadata fvdlMetadata, Set modifiedFiles, + Map skippedByReason, Map> appliedChangesByFile) { + String instanceId = remediation.getAttribute("instanceId"); + try { + List stagedChanges = new ArrayList<>(); + Map pendingWrites = prepareFileChanges(remediation, sourceBasePath, fvdlMetadata, appliedChangesByFile, stagedChanges); + if (pendingWrites.isEmpty()) { + recordSkipped(skippedByReason, SkipReason.NO_CHANGES.displayName); + return false; + } + try { + commitRemediationWrites(instanceId, pendingWrites, modifiedFiles); + for (AppliedChange change : stagedChanges) { + appliedChangesByFile.computeIfAbsent(change.filePath(), key -> new ArrayList<>()).add(change); + } + return true; + } catch (RemediationCommitException e) { + rollbackRemediationWrites(instanceId, e.getRollbacks()); + throw new SkipRemediationException(SkipReason.SOURCE_WRITE_FAILED, e.getMessage(), e); + } + } catch (SkipRemediationException e) { + recordSkipped(skippedByReason, skipReasonLabel(e)); + LOG.warn("Skipping remediation {}: {}", instanceId, e.getMessage()); + LOG.debug("Skip reason for remediation {}: {}", instanceId, e.reason.displayName, e); + return false; + } catch (RollbackRemediationException e) { + throw e; + } catch (Exception e) { + recordSkipped(skippedByReason, SkipReason.UNEXPECTED_ERROR.displayName); + LOG.warn("Skipping remediation {} due to an unexpected processing error", instanceId); + LOG.debug("Unexpected error while processing remediation {}", instanceId, e); + return false; + } + } + + private Map prepareFileChanges(Element remediation, Path sourceBasePath, FVDLMetadata fvdlMetadata, Map> appliedChangesByFile, List stagedChanges) { + NodeList fileChangesNodes = remediation.getElementsByTagNameNS(NAMESPACE_URI, "FileChanges"); + if (fileChangesNodes.getLength() == 0) { + throw new SkipRemediationException(SkipReason.NO_CHANGES, "No file changes found"); + } + + Map pendingWrites = new LinkedHashMap<>(); + for (int j = 0; j < fileChangesNodes.getLength(); j++) { + processFileChanges(remediation, (Element) fileChangesNodes.item(j), sourceBasePath, fvdlMetadata, pendingWrites, appliedChangesByFile, stagedChanges); + } + return pendingWrites; + } + + private boolean processFileChanges(Element remediation, Element fileChanges, Path sourceBasePath, FVDLMetadata fvdlMetadata, Map pendingWrites, Map> appliedChangesByFile, List stagedChanges) {String instanceId = remediation.getAttribute("instanceId"); + String filename = getRequiredElementText(fileChanges, "Filename"); + Path filePath = sourceBasePath.resolve(filename).normalize(); + LOG.debug("Processing remediation {} file change for '{}' resolved to '{}'", instanceId, filename, filePath); + + if (!filePath.startsWith(sourceBasePath)) { + throw new SkipRemediationException(SkipReason.SOURCE_FILE_OUTSIDE_SOURCE_DIR, + "Source file resolves outside source directory: " + filename); + } + + if (!isFilePresent(filePath)) { + throw new SkipRemediationException(SkipReason.SOURCE_FILE_MISSING, "Source code file not present at: " + filePath); + } + + String fileHash = getRequiredElementText(fileChanges, "Hash"); + NodeList changesNodes = fileChanges.getElementsByTagNameNS(NAMESPACE_URI, "Change"); + if (changesNodes.getLength() == 0) { + throw new SkipRemediationException(SkipReason.NO_CHANGES, "No changes found for file: " + filename); + } + SourceFileContent sourceFileContent = getPendingOrSourceContent(filePath, filename, fvdlMetadata, pendingWrites); + Charset sourceEncoding = sourceFileContent.charset(); + LOG.debug("Remediation {} has {} change(s) for '{}' using source encoding {}", instanceId, changesNodes.getLength(), filename, + sourceFileContent.encodingSource()); + + String updatedContent = sourceFileContent.content(); + for (int k = 0; k < changesNodes.getLength(); k++) { + ChangeApplication application = applyChange(instanceId, filename, filePath, fileHash, sourceEncoding, updatedContent, (Element) changesNodes.item(k), k + 1, appliedChangesByFile, stagedChanges); + updatedContent = application.content(); + stagedChanges.add(application.appliedChange()); + } + byte[] updatedBytes = encodeSourceFile(updatedContent, sourceEncoding, filename); + pendingWrites.put(filePath, new PendingFileWrite(filename, filePath, updatedContent, sourceEncoding, + sourceFileContent.encodingSource(), updatedBytes)); + LOG.debug("Staged remediation {} for '{}' using source encoding {}; changes={}, encodedBytes={}", instanceId, filename, + sourceFileContent.encodingSource(), changesNodes.getLength(), updatedBytes.length); + return true; + } + + private ChangeApplication applyChange(String instanceId, String filename, Path filePath, String fileHash, Charset sourceEncoding, String originalContent, Element change, int changeIndex, Map> appliedChangesByFile, List stagedChanges) { + String lineSeparator = detectLineSeparator(originalContent); + String content = normalizeLineEndings(originalContent); + List originalLines = Arrays.asList(content.split("\n", -1)); + int declaredLineFrom = parseRequiredInt(change, "LineFrom"); + int declaredLineTo = parseRequiredInt(change, "LineTo"); + String calculatedHash = calculateHashBase64(content, "SHA-256"); + boolean fileHashMatches = calculatedHash.equals(fileHash); + int lineFrom = declaredLineFrom; + int lineTo = declaredLineTo; + List previousChanges = new ArrayList<>(); + List committedChanges = appliedChangesByFile.get(filePath); + if (committedChanges != null) { previousChanges.addAll(committedChanges); } + for (AppliedChange stagedChange : stagedChanges) { if (filePath.equals(stagedChange.filePath())) { previousChanges.add(stagedChange); } } + + + LOG.debug( + "ANCHOR DEBUG: instance={}, file={}, declaredRange={}-{}, fileHashMatches={}, previousChanges={}", + instanceId, + filename, + declaredLineFrom, + declaredLineTo, + fileHashMatches, + previousChanges.size()); + + + + + if (!previousChanges.isEmpty() && !fileHashMatches ) { + Element contextElement = getRequiredElement(change, "Context"); + + LOG.debug("ANCHOR DEBUG: Context for instance={}: [{}]", + instanceId, + contextElement.getTextContent()); + + LOG.debug( + "ANCHOR DEBUG: Context before={}, after={}", + contextElement.getAttribute("before"), + contextElement.getAttribute("after")); + + LOG.debug("ANCHOR DEBUG: Previous changes for '{}':", filename); + + for (AppliedChange previousChange : previousChanges) { + LOG.debug( + "ANCHOR DEBUG: remediation={}, original={}-{}, resulting={}-{}, delta={}", + previousChange.remediationId(), + previousChange.originalStart(), + previousChange.originalEnd(), + previousChange.resultingStart(), + previousChange.resultingEnd(), + previousChange.lineDelta()); + } + + int[] projectedRange = projectLineRange( + + declaredLineFrom, + declaredLineTo, + previousChanges, + instanceId, + filename); + + lineFrom = projectedRange[0]; + lineTo = projectedRange[1]; + + validateLineRange( + lineFrom, + lineTo, + originalLines.size(), + filename); + + try { + verifyOriginalCodeAtRange( + instanceId, + filename, + originalLines, + lineFrom, + lineTo, + change); + } catch (SkipRemediationException e) { + if (e.reason != SkipReason.ANCHOR_MISMATCH) { + throw e; + } + + LOG.debug("=== ANCHOR MISMATCH - TRYING REANCHOR ==="); + LOG.debug("InstanceId: {}", instanceId); + LOG.debug("Filename: {}", filename); + LOG.debug("Projected range: {}-{}", lineFrom, lineTo); + int debugStart = Math.max(1, lineFrom - 10); + int debugEnd = Math.min(originalLines.size(), lineTo + 10); + + LOG.debug( + "ANCHOR DEBUG: Source window {}-{}", + debugStart, + debugEnd); + + for (int line = debugStart; line <= debugEnd; line++) { + LOG.debug( + "ANCHOR DEBUG: source[{}] = [{}]", + line, + originalLines.get(line - 1)); + } + + int[] recoveredRange = findOriginalCodeUsingContext( + instanceId, + filename, + originalLines, + change, + lineFrom, + lineTo); + + if (recoveredRange[0] == -1) { + LOG.debug("REANCHOR FAILED"); + throw e; + } + + lineFrom = recoveredRange[0] + 1; + lineTo = recoveredRange[1] + 1; + + LOG.debug( + "REANCHOR SUCCESS: using range {}-{}", + lineFrom, + lineTo); + + validateLineRange( + lineFrom, + lineTo, + originalLines.size(), + filename); + + verifyOriginalCodeAtRange( + instanceId, + filename, + originalLines, + lineFrom, + lineTo, + change); + } + + } else if (!fileHashMatches) { + Element contextElement = getRequiredElement(change, "Context"); + List contextLine = Arrays.asList(contextElement.getTextContent().split("\\r?\\n")); + int contextLineFrom = fuzzySearchContext(instanceId, filename, originalLines, contextLine); + if (contextLineFrom == -1) { throw new SkipRemediationException(SkipReason.SOURCE_CONTEXT_NOT_FOUND, "Source context not found for file '" + filename + "'"); } + String originalCodeText = getRequiredElementText(change, "OriginalCode"); + List originalCodeLine = Arrays.asList(originalCodeText.split("\\r?\\n")); + int contextBefore = parseRequiredContextAttribute(contextElement, "before"); + int contextAfter = parseRequiredContextAttribute(contextElement, "after"); + + LOG.debug("=== FUZZY ORIGINAL SEARCH ==="); + LOG.debug("InstanceId:{}",instanceId); + LOG.debug("Declared range: {} {}",declaredLineFrom, declaredLineTo); + LOG.debug("File hash: {}", fileHash); + LOG.debug("Calculated hash: {}", calculatedHash); + LOG.debug("OriginalCode:\n{}", originalCodeText); + LOG.debug("Context:\n{}", contextElement.getTextContent()); + LOG.debug("Context matches from: {}", contextLineFrom); + LOG.debug("=============================="); + int[] lineFromTo = fuzzySearchOriginalCode(instanceId, filename, originalLines, originalCodeLine, contextLineFrom, contextLine.size(), contextBefore, contextAfter); + if (lineFromTo[0] == -1 || lineFromTo[1] == -1) { + LOG.debug("=== ORIGINAL CODE NOT FOUND ==="); + LOG.debug("InstanceId: {}", instanceId); + LOG.debug("Filename: {}", filename); + LOG.debug("FilePath: {}", filePath); + LOG.debug("OriginalCode:\n{}", originalCodeText); + LOG.debug("Declared lines: {}-{}", declaredLineFrom, declaredLineTo); + LOG.debug("Context line: {}", contextLineFrom); + LOG.debug("==============================="); + throw new SkipRemediationException(SkipReason.ORIGINAL_CODE_NOT_FOUND, "Original code not found for file '" + filename + "'"); + } + lineFrom = lineFromTo[0] + 1; + lineTo = lineFromTo[1] + 1; + } + validateLineRange(lineFrom, lineTo, originalLines.size(), filename); + List newCodeLines = getCodeLines(change, "NewCode"); + List updatedLines = new ArrayList<>(); + updatedLines.addAll(originalLines.subList(0, lineFrom - 1)); + updatedLines.addAll(newCodeLines); + updatedLines.addAll(originalLines.subList(lineTo, originalLines.size())); + String updatedContent = String.join(lineSeparator, updatedLines); + AppliedChange appliedChange = new AppliedChange(filePath, declaredLineFrom, declaredLineTo, lineFrom, lineFrom + newCodeLines.size() - 1, instanceId, getRequiredElementText(change, "OriginalCode"), getRequiredElementText(change, "NewCode")); + return new ChangeApplication(updatedContent, appliedChange); + } + private int[] projectLineRange( + int originalStart, + int originalEnd, + List appliedChanges, + String instanceId, + String filename) { + + int projectedStart = originalStart; + int projectedEnd = originalEnd; + + for (int i = 0; i < appliedChanges.size(); i++) { + AppliedChange applied = appliedChanges.get(i); + + /* + * Project this previous change's physical range forward + * through all changes that were applied after it. + */ + int appliedStart = applied.resultingStart(); + int appliedEnd = applied.resultingEnd(); + + for (int j = i + 1; j < appliedChanges.size(); j++) { + AppliedChange later = appliedChanges.get(j); + + if (later.originalEnd() < applied.originalStart()) { + int delta = later.lineDelta(); + appliedStart += delta; + appliedEnd += delta; + } + } + + /* + * A previous change completely before this remediation + * shifts the target's physical location. + */ + if (originalStart > applied.originalEnd()) { + int delta = applied.lineDelta(); + + projectedStart += delta; + projectedEnd += delta; + + continue; + } + + /* + * Compare against the previous change's CURRENT physical + * location, not the stale location recorded when it ran. + */ + boolean physicalOverlap = + projectedStart <= appliedEnd + && projectedEnd >= appliedStart; + + if (physicalOverlap) { + int overlapStart = + Math.max(projectedStart, appliedStart); + + int overlapEnd = + Math.min(projectedEnd, appliedEnd); + + throw new SkipRemediationException( + SkipReason.CONFLICT, + "Remediation '" + instanceId + + "' conflicts with remediation '" + + applied.remediationId() + + "' in file '" + filename + + "'; overlapping physical lines " + + overlapStart + "-" + overlapEnd); + } + } + + return new int[] { + projectedStart, + projectedEnd + }; + } + /**private void verifyOriginalCodeAtRange( + String instanceId, + String filename, + List sourceLines, + int lineFrom, + int lineTo, + Element change) { + + List expectedLines = + getCodeLines(change, "OriginalCode"); + + List actualLines = + sourceLines.subList(lineFrom - 1, lineTo); + + List normalizedActual = + actualLines.stream() + .map(String::trim) + .toList(); + + List normalizedExpected = + expectedLines.stream() + .map(String::trim) + .toList(); + + if (!normalizedExpected.equals(normalizedActual)) { + throw new SkipRemediationException( + SkipReason.ANCHOR_MISMATCH, + "Anchor does not match for remediation '" + + instanceId + + "' in file '" + + filename + + "' at lines " + + lineFrom + "-" + lineTo); + } + }**/ + private void verifyOriginalCodeAtRange( + String instanceId, + String filename, + List sourceLines, + int lineFrom, + int lineTo, + Element change) { + + String originalCode = + normalizeLineEndings( + getRequiredElementText(change, "OriginalCode")); + + List expectedLines = + Arrays.asList(originalCode.split("\n", -1)); + + List actualLines = + sourceLines.subList(lineFrom - 1, lineTo); + + if (expectedLines.size() != actualLines.size()) { + + LOG.debug("========== ANCHOR SIZE MISMATCH =========="); + LOG.debug("Remediation instanceId: {}", instanceId); + LOG.debug("Filename: {}", filename); + LOG.debug("Expected range: {}-{}", lineFrom, lineTo); + LOG.debug("Expected line count: {}", expectedLines.size()); + LOG.debug("Actual line count: {}", actualLines.size()); + + LOG.debug("Expected OriginalCode:"); + for (int i = 0; i < expectedLines.size(); i++) { + LOG.debug( + " expected[{}] = [{}]", + i, + expectedLines.get(i)); + } + + LOG.debug("Actual source lines:"); + for (int i = 0; i < actualLines.size(); i++) { + LOG.debug( + " actual[{}] = [{}]", + i, + actualLines.get(i)); + } + + LOG.debug("=========================================="); + + throw new SkipRemediationException( + SkipReason.ANCHOR_MISMATCH, + "Anchor does not match for remediation '" + + instanceId + + "' in file '" + + filename + + "' at lines " + + lineFrom + + "-" + + lineTo); + } + for (int i = 0; i < expectedLines.size(); i++) { + String expected = expectedLines.get(i).strip(); + String actual = actualLines.get(i).strip(); + + if (!expected.equals(actual)) { + + LOG.debug("========== ANCHOR CONTENT MISMATCH =========="); + LOG.debug("Remediation instanceId: {}", instanceId); + LOG.debug("Filename: {}", filename); + LOG.debug("Range: {}-{}", lineFrom, lineTo); + LOG.debug("Mismatch at relative line: {}", i); + LOG.debug("Expected: [{}]", expected); + LOG.debug("Actual: [{}]", actual); + LOG.debug("=============================================="); + + throw new SkipRemediationException( + SkipReason.ANCHOR_MISMATCH, + "Anchor does not match for remediation '" + + instanceId + + "' in file '" + + filename + + "' at lines " + + lineFrom + + "-" + + lineTo); + } + } + } + + private List getCodeLines( + Element change, + String elementName) { + + String code = + normalizeLineEndings( + getRequiredElementText(change, elementName)); + + String[] lines = code.split("\n", -1); + + int start = 0; + int end = lines.length - 1; + + while (start <= end && lines[start].isBlank()) { + start++; + } + + while (end >= start && lines[end].isBlank()) { + end--; + } + + if (start > end) { + return List.of(); + } + + return Arrays.asList( + Arrays.copyOfRange(lines, start, end + 1)); + } + + private SourceFileContent getPendingOrSourceContent(Path filePath, String filename, FVDLMetadata fvdlMetadata, + Map pendingWrites) { + PendingFileWrite pendingWrite = pendingWrites.get(filePath); + return pendingWrite == null + ? readSourceFile(filePath, filename, fvdlMetadata) + : new SourceFileContent(pendingWrite.content(), pendingWrite.charset(), pendingWrite.encodingSource()); + } + + private void commitRemediationWrites(String instanceId, Map pendingWrites, Set modifiedFiles) + throws RemediationCommitException { + List rollbacks = new ArrayList<>(); + for (PendingFileWrite pendingWrite : pendingWrites.values()) { + try { + byte[] originalBytes = Files.readAllBytes(pendingWrite.filePath()); + rollbacks.add(new RollbackFileWrite(pendingWrite.filename(), pendingWrite.filePath(), originalBytes)); + LOG.debug("Writing remediation {} to '{}' using staged bytes; encodedBytes={}", instanceId, pendingWrite.filename(), + pendingWrite.updatedBytes().length); + Files.write(pendingWrite.filePath(), pendingWrite.updatedBytes()); + } catch (Exception e) { + throw new RemediationCommitException("Error writing source code file '" + pendingWrite.filename() + "'", e, rollbacks); + } + } + + for (PendingFileWrite pendingWrite : pendingWrites.values()) { + modifiedFiles.add(pendingWrite.filename()); + LOG.info("Remediation applied for {} in file {}", instanceId, pendingWrite.filename()); + } + } + + private void rollbackRemediationWrites(String instanceId, List rollbacks) { + for (RollbackFileWrite rollback : rollbacks) { + try { + Files.write(rollback.filePath(), rollback.originalBytes()); + LOG.warn("Rolled back remediation {} changes for '{}' after write failure", instanceId, rollback.filename()); + } catch (IOException rollbackException) { + LOG.error("Failed to roll back remediation {} changes for '{}'", instanceId, rollback.filename(), rollbackException); + throw new RollbackRemediationException("Failed to roll back remediation changes for '" + rollback.filename() + + "'. Source files may be partially modified; inspect the source tree before retrying", rollbackException); + } + } + } + + private int fuzzySearchContext(String instanceId, String filename, List originalLines, List contextLine) { + try { + List matches = FuzzyContextSearcher.fuzzySearchContextMatches(originalLines, contextLine, 0); + if (matches.size() > 1) { + String candidateLines = matches.stream() + .map(line -> String.valueOf(line + 1)) + .collect(Collectors.joining(", ")); + + LOG.debug("=== SOURCE CONTEXT AMBIGUOUS ==="); + LOG.debug("InstanceId: {}", instanceId); + LOG.debug("Filename: {}", filename); + LOG.debug("SourceContext:\n{}", String.join("\n", contextLine)); + LOG.debug("Matching locations: {}", matches); + LOG.debug("==============================="); + throw new SkipRemediationException(SkipReason.SOURCE_CONTEXT_AMBIGUOUS, + "Source context matched multiple locations in file '" + filename + "'; candidate lines: " + candidateLines); + } + return matches.isEmpty() ? -1 : matches.get(0); + } catch (IOException e) { + throw new SkipRemediationException(SkipReason.SOURCE_CONTEXT_NOT_FOUND, + "Error searching source context for remediation '" + instanceId + "' in file '" + filename + "'", e); + } + } + + private int[] fuzzySearchOriginalCode( + String instanceId, + String filename, + List originalLines, + List originalCodeLine, + int contextLineFrom, + int contextLineCount, + int contextBefore, + int contextAfter) { + + /* + * First try the location implied by the context. + * This keeps the context as the primary anchor. + */ + int contextStart = contextLineFrom + contextBefore; + int contextEnd = contextLineFrom + contextLineCount - contextAfter; + + if (contextStart >= 0 + && contextStart < contextEnd + && contextEnd <= originalLines.size()) { + + int[] match = FuzzyContextSearcher.fuzzySearchOriginalCode( + originalLines.subList(contextStart, contextEnd), + originalCodeLine, + 0, + 0); + + if (match[0] != -1 && match[1] != -1) { + return new int[] { + match[0] + contextStart, + match[1] + contextStart + }; + } + } + + /* + * Context boundaries can become unreliable when blank lines, + * formatting changes, or inserted lines are involved. + * + * Fall back to searching the complete source file. + */ + LOG.debug( + "Original code not found inside context window; searching entire source for remediation {} in '{}'", + instanceId, + filename); + + return FuzzyContextSearcher.fuzzySearchOriginalCode( + originalLines, + originalCodeLine, + 2, + 0); + } + + private boolean isFilePresent(Path path) { + return Files.exists(path) && Files.isRegularFile(path); + } + + /** Nullable: missing/unreadable FVDL means FPR encoding candidate is skipped. */ + private FVDLMetadata loadFvdlMetadata() { + if (!Files.exists(fprHandle.getPath("/audit.fvdl"))) { + LOG.warn("FVDL file '/audit.fvdl' is missing; FPR encoding candidate will be skipped"); + return null; + } + + try (ZipFile zipFile = new ZipFile(fprHandle.getFprPath().toFile())) { + LOG.debug("Loading FVDL build metadata from '{}' to resolve source encodings", fprHandle.getFprPath()); + // Decoder unused for metadata-only parse; ctor requires one for FileUtils wiring. + StreamingFVDLProcessor processor = new StreamingFVDLProcessor(fprHandle, sourceDecoder); + processor.parseBuildMetadata(zipFile, "audit.fvdl"); + LOG.debug("Loaded FVDL build metadata from '{}'", fprHandle.getFprPath()); + return processor.getFvdlMetadata(); + } catch (Exception e) { + LOG.warn("Error reading source file encodings from audit.fvdl; FPR encoding candidate will be skipped", e); + return null; + } + } + + private SourceFileContent readSourceFile(Path filePath, String filename, FVDLMetadata fvdlMetadata) { + try { + byte[] sourceBytes = Files.readAllBytes(filePath); + // Metadata may be null (FVDL missing); FPR candidate fails and other encodings are tried. + DecodeResult decodeResult = sourceDecoder.decode(sourceBytes, filename, fvdlMetadata); + LOG.debug("Strict decoded '{}' using {}; sourceBytes={}, decodedChars={}", filename, decodeResult.source(), sourceBytes.length, + decodeResult.content().length()); + return new SourceFileContent(decodeResult.content(), decodeResult.charset(), decodeResult.source()); + } catch (SourceDecodeException e) { + throw new SkipRemediationException(SkipReason.SOURCE_DECODE_FAILED, e.getMessage(), e); + } catch (IOException e) { + throw new SkipRemediationException(SkipReason.SOURCE_READ_FAILED, "Error reading source code file '" + filePath + "'", e); + } + } + + private byte[] encodeSourceFile(String content, Charset charset, String filename) { + try { + return SourceEncoder.encode(content, charset, filename); + } catch (SourceEncodeException e) { + throw new SkipRemediationException(SkipReason.REMEDIATION_ENCODE_FAILED, e.getMessage(), e); + } + } + + private String getRequiredElementText(Element parent, String elementName) { + return getRequiredElement(parent, elementName).getTextContent(); + } + + private Element getRequiredElement(Element parent, String elementName) { + NodeList nodes = parent.getElementsByTagNameNS(NAMESPACE_URI, elementName); + if (nodes.getLength() == 0 || nodes.item(0) == null) { + throw new SkipRemediationException(SkipReason.REMEDIATION_DATA_INVALID, + "Missing required remediation element '" + elementName + "'"); + } + return (Element) nodes.item(0); + } + + private int parseRequiredContextAttribute(Element context, String attributeName) { + String value = context.getAttribute(attributeName); + if (value == null || value.isBlank()) { + throw new SkipRemediationException(SkipReason.REMEDIATION_DATA_INVALID, + "Missing required remediation context attribute '" + attributeName + "'"); + } + try { + int parsedValue = Integer.parseInt(value); + if (parsedValue < 0) { + throw new NumberFormatException("negative value"); + } + return parsedValue; + } catch (NumberFormatException e) { + throw new SkipRemediationException(SkipReason.REMEDIATION_DATA_INVALID, + "Invalid remediation context attribute '" + attributeName + "': " + value, e); + } + } + + private int parseRequiredInt(Element parent, String elementName) { + String value = getRequiredElementText(parent, elementName); + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + throw new SkipRemediationException(SkipReason.REMEDIATION_DATA_INVALID, + "Invalid integer value for remediation element '" + elementName + "': " + value, e); + } + } + + private void validateLineRange(int lineFrom, int lineTo, int sourceLineCount, String filename) { + if (lineFrom < 1 || lineTo < lineFrom || lineTo > sourceLineCount) { + throw new SkipRemediationException(SkipReason.REMEDIATION_LINE_RANGE_INVALID, + "Invalid remediation line range " + lineFrom + "-" + lineTo + " for file '" + filename + "'"); + } + } + + private String detectLineSeparator(String content) { + int crlfIndex = content.indexOf("\r\n"); + int lfIndex = content.indexOf('\n'); + int crIndex = content.indexOf('\r'); + + if (crlfIndex >= 0 && (lfIndex == crlfIndex + 1 || lfIndex < 0) && (crIndex == crlfIndex || crIndex < 0)) { + return "\r\n"; + } + if (lfIndex >= 0 && (crIndex < 0 || lfIndex < crIndex)) { + return "\n"; + } + if (crIndex >= 0) { + return "\r"; + } + return System.lineSeparator(); + } + + private String normalizeLineEndings(String content) { + return content.replace("\r\n", "\n").replace('\r', '\n'); + } + + private String describeLineSeparator(String lineSeparator) { + return switch (lineSeparator) { + case "\r\n" -> "CRLF"; + case "\n" -> "LF"; + case "\r" -> "CR"; + default -> "system"; + }; + } + + private String calculateHashBase64(String content, String algorithm) { + String hash; + if (content == null) { + return ""; + } + try { + MessageDigest md = MessageDigest.getInstance(algorithm); + byte[] digest = md.digest(content.getBytes(StandardCharsets.UTF_8)); + hash = Base64.getEncoder().encodeToString(digest); + return hash; + } catch (NoSuchAlgorithmException e) { + throw new AviatorTechnicalException("Hashing algorithm not available: " + algorithm, e); + } + } + + private void recordSkipped(Map skippedByReason, String reason) { + skippedByReason.merge(reason, 1, Integer::sum); + } + + private String skipReasonLabel(SkipRemediationException exception) { + return exception.reason.displayName; + } + + private String formatSkippedReasons(Map skippedByReason) { + List parts = new ArrayList<>(); + skippedByReason.forEach((reason, count) -> parts.add(reason + "=" + count)); + return String.join(", ", parts); + } + + private RemediationKey createRemediationKey( + Element fileChanges, + Element change, + Path sourceBasePath, + String comparisonCode) { + + String fileName = getRequiredElementText(fileChanges, "Filename"); + Path filePath = sourceBasePath.resolve(fileName).normalize(); + + int lineFrom = parseRequiredInt(change, "LineFrom"); + int lineTo = parseRequiredInt(change, "LineTo"); + + return new RemediationKey( + fileName, + filePath, + lineFrom, + lineTo, + comparisonCode + ); + } + + + private String trimBlankLines(String content) { + String[] lines = content.split("\\R", -1); + + int start = 0; + int end = lines.length - 1; + + while (start <= end && lines[start].isBlank()) { + start++; + } + + while (end >= start && lines[end].isBlank()) { + end--; + } + + if (start > end) { + return ""; + } + + return String.join( + System.lineSeparator(), + Arrays.copyOfRange(lines, start, end + 1)); + } + + private String normalizeProposedCode(String content, String fileName) { + if (content == null) { + return null; + } + + String language = FileTypeLanguageMapperUtil.getProgrammingLanguage( + FileUtil.getFileExtension(fileName)); + + String commentSymbol = + LanguageCommentMapperUtil.getProgrammingLanguageComment(language); + + if ("Unknown".equals(commentSymbol)) { + return trimBlankLines(content); + } + + String closingToken = commentSymbol.equals("" + : commentSymbol.equals("<%--") ? "--%>" + : null; + + Pattern markerPattern = Pattern.compile( + "[ \\t]*" + Pattern.quote(commentSymbol) + " L\\d+" + + (closingToken != null + ? "[ \\t]*" + Pattern.quote(closingToken) + : "") + + "[ \\t]*$"); + + String[] lines = content.split("\\R", -1); + StringBuilder result = new StringBuilder(); + + for (int i = 0; i < lines.length; i++) { + Matcher matcher = markerPattern.matcher(lines[i]); + + result.append( + matcher.find() + ? lines[i].substring(0, matcher.start()) + : lines[i]); + + if (i < lines.length - 1) { + result.append(System.lineSeparator()); + } + } + + return trimBlankLines(result.toString()); + } + + private String createComparisonCode(String normalizedCode, String fileName) { + if (normalizedCode == null) { + return null; + } + + String language = FileTypeLanguageMapperUtil.getProgrammingLanguage( + FileUtil.getFileExtension(fileName)); + + String commentSymbol = + LanguageCommentMapperUtil.getProgrammingLanguageComment(language); + + if ("Unknown".equals(commentSymbol)) { + return normalizedCode.replaceAll("\\s+", ""); + } + + String comparisonCode = normalizedCode; + + // Remove block comments + String closingToken = commentSymbol.equals("" + : commentSymbol.equals("<%--") ? "--%>" + : null; + + if (closingToken != null) { + comparisonCode = comparisonCode.replaceAll( + "(?s)" + Pattern.quote(commentSymbol) + + ".*?" + Pattern.quote(closingToken), + ""); + } else if ("//".equals(commentSymbol)) { + comparisonCode = comparisonCode.replaceAll( + "(?m)" + Pattern.quote(commentSymbol) + ".*$", + ""); + comparisonCode = comparisonCode.replaceAll( + "(?s)/\\*.*?\\*/", + ""); + } else if ("#".equals(commentSymbol)) { + comparisonCode = comparisonCode.replaceAll( + "(?m)" + Pattern.quote(commentSymbol) + ".*$", + ""); + } + + // Normalize whitespace + return comparisonCode.replaceAll("\\s+", ""); + } + + private List createRemediationKeys( + Element remediation, + Path sourceBasePath) { + + List keys = new ArrayList<>(); + + NodeList fileChangesNodes = + remediation.getElementsByTagNameNS(NAMESPACE_URI, "FileChanges"); + + for (int i = 0; i < fileChangesNodes.getLength(); i++) { + Element fileChanges = (Element) fileChangesNodes.item(i); + + NodeList changeNodes = + fileChanges.getElementsByTagNameNS(NAMESPACE_URI, "Change"); + + for (int j = 0; j < changeNodes.getLength(); j++) { + Element change = (Element) changeNodes.item(j); + + String fileName = + getRequiredElementText(fileChanges, "Filename"); + + String newCode = + getRequiredElementText(change, "NewCode"); + + String normalizedCode = + normalizeProposedCode(newCode, fileName); + + String comparisonCode = + createComparisonCode(normalizedCode, fileName); + + keys.add(createRemediationKey( + fileChanges, + change, + sourceBasePath, + comparisonCode)); + } + } + + return keys; + } + + private int[] findExactOriginalCodeNearRange( + List sourceLines, + int lineFrom, + int lineTo, + String originalCode, + int radius) { + + List expectedLines = + Arrays.asList( + normalizeLineEndings(originalCode) + .split("\n", -1)); + + int expectedCount = expectedLines.size(); + + int targetStart = lineFrom - 1; + + int searchStart = + Math.max(0, targetStart - radius); + + int searchEnd = + Math.min( + sourceLines.size() - expectedCount, + targetStart + radius); + + LOG.debug( + "REANCHOR SEARCH: projected={} - {}, search={} - {}", + lineFrom, + lineTo, + searchStart + 1, + searchEnd + expectedCount); + + int matchStart = -1; + + for (int i = searchStart; i <= searchEnd; i++) { + + boolean matches = true; + + for (int j = 0; j < expectedCount; j++) { + + String expected = + expectedLines.get(j) + .trim() + .replaceAll("\\s+", " "); + + String actual = + sourceLines.get(i + j) + .trim() + .replaceAll("\\s+", " "); + + if (!expected.equalsIgnoreCase(actual)) { + matches = false; + break; + } + } + + if (matches) { + + LOG.debug( + "REANCHOR CANDIDATE: {}-{}", + i + 1, + i + expectedCount); + + /* + * More than one normalized match means we cannot + * safely determine which occurrence is the remediation + * target. + */ + if (matchStart != -1) { + + LOG.debug( + "REANCHOR AMBIGUOUS: second match {}-{}", + i + 1, + i + expectedCount); + + return new int[] {-1, -1}; + } + + matchStart = i; + } + } + + if (matchStart == -1) { + LOG.debug("REANCHOR NOT FOUND"); + return new int[] {-1, -1}; + } + + LOG.debug( + "REANCHOR SUCCESS: {}-{}", + matchStart + 1, + matchStart + expectedCount); + + return new int[] { + matchStart, + matchStart + expectedCount - 1 + }; + } + + private int[] findOriginalCodeUsingContext( + String instanceId, + String filename, + List sourceLines, + Element change, + int projectedLineFrom, + int projectedLineTo) { + + Element contextElement = getRequiredElement(change, "Context"); + + List contextLines = + Arrays.asList( + normalizeLineEndings( + contextElement.getTextContent()) + .split("\n", -1)); + + int contextBefore = + parseRequiredContextAttribute( + contextElement, + "before"); + + int contextAfter = + parseRequiredContextAttribute( + contextElement, + "after"); + + int projectedContextStart = + Math.max( + 0, + projectedLineFrom - 1 - contextBefore); + + int projectedContextEnd = + Math.min( + sourceLines.size(), + projectedLineTo + contextAfter); + + LOG.debug( + "CONTEXT REANCHOR SEARCH: projected={} - {}, context={} - {}", + projectedLineFrom, + projectedLineTo, + projectedContextStart + 1, + projectedContextEnd); + + int[] contextRange = + findContextNearRange( + sourceLines, + contextLines, + projectedContextStart, + projectedContextEnd, + 100); + + if (contextRange[0] == -1) { + LOG.debug( + "CONTEXT REANCHOR FAILED for remediation {}", + instanceId); + + return new int[] {-1, -1}; + } + + int contextStart = contextRange[0]; + int contextEnd = contextRange[1]; + + List originalCodeLines = + Arrays.asList( + normalizeLineEndings( + getRequiredElementText(change, "OriginalCode")) + .split("\n", -1)); + + int[] originalRange = + FuzzyContextSearcher.fuzzySearchOriginalCode( + sourceLines.subList(contextStart, contextEnd), + originalCodeLines, + 0, + 0); + + if (originalRange[0] == -1) { + LOG.debug( + "ORIGINAL CODE NOT FOUND INSIDE CONTEXT REANCHOR for remediation {}", + instanceId); + + return new int[] {-1, -1}; + } + + return new int[] { + originalRange[0] + contextStart, + originalRange[1] + contextStart + }; + } + + private int[] findContextNearRange( + List sourceLines, + List contextLines, + int projectedStart, + int projectedEnd, + int radius) { + + int searchStart = + Math.max( + 0, + projectedStart - radius); + + int searchEnd = + Math.min( + sourceLines.size(), + projectedEnd + radius); + + List searchLines = + sourceLines.subList( + searchStart, + searchEnd); + + List matches; + + try { + matches = + FuzzyContextSearcher.fuzzySearchContextMatches( + searchLines, + contextLines, + 0); + } catch (IOException e) { + return new int[] {-1, -1}; + } + + if (matches.size() != 1) { + LOG.debug( + "CONTEXT REANCHOR MATCHES: {}", + matches); + + return new int[] {-1, -1}; + } + + int matchStart = + searchStart + matches.get(0); + + return new int[] { + matchStart, + matchStart + contextLines.size() + }; + } +}