From c63f06ab862bbce5e25865cf0e162356a526da57 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Wed, 29 Jul 2026 14:17:32 -0400 Subject: [PATCH 01/39] Initial implementation of sync --- CHANGELOG.md | 1 + docker-compose.yml | 6 +- src/cls/IPM/Lifecycle/Base.cls | 71 +++ src/cls/IPM/Main.cls | 54 ++ src/cls/IPM/ResourceProcessor/Abstract.cls | 14 + .../ResourceProcessor/AbstractCompilable.cls | 8 + src/cls/IPM/ResourceProcessor/FileCopy.cls | 36 ++ src/cls/IPM/ResourceProcessor/Test.cls | 54 ++ src/cls/IPM/Storage/FileHash.cls | 244 +++++++++ src/cls/IPM/Storage/Module.cls | 502 ++++++++++++++++++ .../Test/PM/Integration/Sync.cls | 254 +++++++++ .../PM/Integration/_data/sync-test/module.xml | 19 + .../sync-test/src/cls/SyncTest/Consumer.cls | 11 + .../sync-test/src/cls/SyncTest/Deletable.cls | 9 + .../sync-test/src/cls/SyncTest/SubClass.cls | 9 + .../sync-test/src/cls/SyncTest/SuperClass.cls | 11 + .../_data/sync-test/src/inc/SyncTest.inc | 3 + .../_data/sync-test/static/config.txt | 1 + .../tests/unit/SyncTest/Tests/Trivial.cls | 9 + 19 files changed, 1313 insertions(+), 3 deletions(-) create mode 100644 src/cls/IPM/Storage/FileHash.cls create mode 100644 tests/integration_tests/Test/PM/Integration/Sync.cls create mode 100644 tests/integration_tests/Test/PM/Integration/_data/sync-test/module.xml create mode 100644 tests/integration_tests/Test/PM/Integration/_data/sync-test/src/cls/SyncTest/Consumer.cls create mode 100644 tests/integration_tests/Test/PM/Integration/_data/sync-test/src/cls/SyncTest/Deletable.cls create mode 100644 tests/integration_tests/Test/PM/Integration/_data/sync-test/src/cls/SyncTest/SubClass.cls create mode 100644 tests/integration_tests/Test/PM/Integration/_data/sync-test/src/cls/SyncTest/SuperClass.cls create mode 100644 tests/integration_tests/Test/PM/Integration/_data/sync-test/src/inc/SyncTest.inc create mode 100644 tests/integration_tests/Test/PM/Integration/_data/sync-test/static/config.txt create mode 100644 tests/integration_tests/Test/PM/Integration/_data/sync-test/tests/unit/SyncTest/Tests/Trivial.cls diff --git a/CHANGELOG.md b/CHANGELOG.md index c64689745..16f2799e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - #1178: Add `-password-env` and `-token-env` modifiers to the `repo` command to read the password/token from a named environment variable (secure alternatives to `-password` and `-token`). +- Add `sync` command for incremental loading of changed files in dev-mode modules. Detects modified files since last sync and recompiles only what is stale. Supports `-delete` for processing removed files and `-test` for running changed test-phase unit tests. ### Changed - #1186: Change %IPM.Main:ShellScript() to return a status. diff --git a/docker-compose.yml b/docker-compose.yml index 7f334806e..973e3285f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: - TEST_REGISTRY_USER=admin - TEST_REGISTRY_PASSWORD=SYS ports: - - 52774:52773 + - 52777:52773 volumes: - ~/iris.key:/usr/irissys/mgr/iris.key - ./:/home/irisowner/zpm/ @@ -28,7 +28,7 @@ services: build: ./tests/sandbox/ restart: always ports: - - 52776:52773 + - 52779:52773 environment: - IRISPASSWORD=SYS - IRISUSERNAME=admin @@ -43,4 +43,4 @@ services: image: ghcr.io/project-zot/zot-linux-amd64:latest restart: always ports: - - 5001:5000 + - 5002:5000 diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls index d6b9a03fe..85d759764 100644 --- a/src/cls/IPM/Lifecycle/Base.cls +++ b/src/cls/IPM/Lifecycle/Base.cls @@ -727,6 +727,59 @@ Method %Reload(ByRef pParams) As %Status quit tSC } +/// Build the set of tracked paths (resource-owned files + module.xml) for sync change detection. +/// Output: trackedPaths(normalizedRelPath)="" +Method GetTrackedPaths(Output trackedPaths) +{ + kill trackedPaths + set root = ##class(%File).NormalizeDirectory(..Module.Root) + + // Include module.xml + set trackedPaths(##class(%IPM.Storage.FileHash).NormalizePath("module.xml")) = "" + + // Walk resources and resolve children to get all source file paths + set orderedResourceList = ..Module.GetOrderedResourceList() + set key = "" + for { + set resource = orderedResourceList.GetNext(.key) + quit:key="" + + if '$isobject(resource.Processor) { + continue + } + + kill childArr + set sc = resource.ResolveChildren(.childArr) + if $$$ISERR(sc) { + continue + } + + set childName = "" + for { + set childName = $order(childArr(childName)) + quit:childName="" + + set relPath = $get(childArr(childName, "RelativePath")) + if relPath = "" { + set relPath = resource.Processor.OnItemRelativePath(childName) + } + if relPath = "" { + continue + } + + set trackedPaths(##class(%IPM.Storage.FileHash).NormalizePath(relPath)) = "" + } + } + set debugCount = 0 + set debugKey = "" + for { + set debugKey = $order(trackedPaths(debugKey)) + quit:debugKey="" + set debugCount = debugCount + 1 + } + write !, "[DEBUG GetTrackedPaths] module=", ..Module.Name, " trackedCount=", debugCount +} + Method InstallOrDownloadPythonRequirements( pRoot As %String = "", ByRef pParams, @@ -1203,6 +1256,24 @@ Method %Compile(ByRef pParams) As %Status $$$ThrowStatus(tSC) } } + + // Stamp file baselines for sync change detection (dev mode only, non-fatal). + // Done here after compile so test classes are in ^oddDEF and appear in trackedPaths. + if tDevMode { + try { + set trackedPaths = "" + do ..GetTrackedPaths(.trackedPaths) + write !, "[DEBUG %Compile] Stamping baselines for module: ", ..Module.Name + set stampSC = ##class(%IPM.Storage.FileHash).StampModule(..Module, .trackedPaths) + if $$$ISERR(stampSC) { + write !, "[DEBUG %Compile] StampModule error: ", $system.Status.GetErrorText(stampSC) + } else { + write !, "[DEBUG %Compile] StampModule OK" + } + } catch stampErr { + write !, "[DEBUG %Compile] StampModule exception: ", stampErr.DisplayString() + } + } } catch e { set tSC = e.AsStatus() } diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls index 21eef44d1..f0c5c7add 100644 --- a/src/cls/IPM/Main.cls +++ b/src/cls/IPM/Main.cls @@ -104,6 +104,20 @@ This command is an alias for `module-action module-name reload` + +Incrementally syncs changed files from disk into the namespace. + +Detects files changed on disk since the last load/sync for development-mode modules, +routes each to its resource processor, and recompiles only what is stale. +If no module is specified, syncs all modules in development mode. + + + + + + + + This command is an alias for `module-action module-name compile` @@ -1098,6 +1112,8 @@ ClassMethod ShellInternal( do ..Information(.tCommandInfo) } elseif (tCommandInfo = "history") { do ..History(.tCommandInfo) + } elseif (tCommandInfo = "sync") { + do ..Sync(.tCommandInfo) } } catch pException { if (pException.Code = $$$ERCTRLC) { @@ -2310,6 +2326,44 @@ ClassMethod LoadFromRepo( quit tDirectoryName } +ClassMethod Sync(ByRef commandInfo) [ Private ] +{ + set moduleName = $get(commandInfo("parameters", "module")) + merge params = commandInfo("data") + + if moduleName '= "" { + $$$ThrowOnError(##class(%IPM.Storage.Module).Sync(moduleName, .params)) + } else { + // Sync all dev-mode modules + set result = ##class(%SQL.Statement).%ExecDirect(, + "select Name from %IPM_Storage.ModuleItem where DeveloperMode = 1") + if (result.%SQLCODE < 0) { + throw ##class(%Exception.SQL).CreateFromSQLCODE(result.%SQLCODE, result.%Message) + } + set found = 0 + set failures = "" + for { + set hasData = result.%Next(.sc) + $$$ThrowOnError(sc) + if 'hasData { + quit + } + set found = found + 1 + set name = result.%Get("Name") + set syncSC = ##class(%IPM.Storage.Module).Sync(name, .params) + if $$$ISERR(syncSC) { + set failures = failures _ $listbuild(name) + do $system.OBJ.DisplayError(syncSC) + } + } + if 'found { + write !, "No modules in development mode." + } elseif failures '= "" { + write !, "Sync completed with errors in: ", $listtostring(failures, ", ") + } + } +} + ClassMethod Load( ByRef pCommandInfo, pLog As %IPM.General.AbstractHistory = "") [ Internal ] diff --git a/src/cls/IPM/ResourceProcessor/Abstract.cls b/src/cls/IPM/ResourceProcessor/Abstract.cls index 2bf0eeadd..cf9ebc2f8 100644 --- a/src/cls/IPM/ResourceProcessor/Abstract.cls +++ b/src/cls/IPM/ResourceProcessor/Abstract.cls @@ -204,6 +204,20 @@ Method OnAfterPhase( quit $$$OK } +/// Called during 'sync' for resources whose files changed on disk. +/// modifiedPaths(relPath)="" for changed files; deletedPaths(relPath)="" for removed files. +/// Set handled=1 to fully own sync for this resource (skip default load+compile handling). +Method OnSync(ByRef modifiedPaths, ByRef deletedPaths, ByRef params, Output handled As %Boolean = 0) As %Status +{ + quit $$$OK +} + +/// Returns 1 if this processor supports incremental sync. Base returns 0. +Method SupportsSync() As %Boolean +{ + quit 0 +} + /// Returns the path relative to the module root for item pItemName within this resource. Method OnItemRelativePath(pItemName As %String) As %String { diff --git a/src/cls/IPM/ResourceProcessor/AbstractCompilable.cls b/src/cls/IPM/ResourceProcessor/AbstractCompilable.cls index 25839ade2..05ba2389b 100644 --- a/src/cls/IPM/ResourceProcessor/AbstractCompilable.cls +++ b/src/cls/IPM/ResourceProcessor/AbstractCompilable.cls @@ -19,4 +19,12 @@ Property ExportFlags As %String(MAXLEN = ""); /// Certain Deployed Resources cannot be compiled directly, but must be compiled from a studio project. Property CompileFromProject As %Boolean [ InitialExpression = 0 ]; +/// Compilable resources (classes, routines, includes) can be individually reloaded and +/// recompiled by sync without a full lifecycle run, so they support sync by default. +/// Non-compilable processors (FileCopy, CSP applications, etc.) keep the default of 0. +Method SupportsSync() As %Boolean +{ + quit 1 +} + } diff --git a/src/cls/IPM/ResourceProcessor/FileCopy.cls b/src/cls/IPM/ResourceProcessor/FileCopy.cls index 22976ea2a..2cc0542f5 100644 --- a/src/cls/IPM/ResourceProcessor/FileCopy.cls +++ b/src/cls/IPM/ResourceProcessor/FileCopy.cls @@ -184,6 +184,42 @@ Method DoCopy( quit tSC } +/// Enumerate source files so sync can detect changes to FileCopy resources. +/// Populates pResourceArray with RelativePath for each file under the source directory. +Method OnResolveChildren(ByRef pResourceArray, pCheckModuleOwnership As %Boolean) As %Status +{ + set sc = $$$OK + try { + set sourceDir = ##class(%File).NormalizeDirectory(..GetSource()) + set moduleRoot = ##class(%File).NormalizeDirectory(..ResourceReference.Module.Root) + if '##class(%File).DirectoryExists(sourceDir) { + quit + } + + set rs = ##class(%SQL.Statement).%ExecDirect(, + "SELECT Name, Type FROM %Library.File_FileSet(?, ?, ?, ?)", + sourceDir, "*", "", 1) + if rs.%SQLCODE < 0 { + $$$ThrowStatus($$$ERROR($$$GeneralError, "FileSet query error: " _ rs.%Message)) + } + while rs.%Next() { + if rs.%Get("Type") = "D" { + continue + } + set fullPath = rs.%Get("Name") + // Compute path relative to module root + set relPath = $extract(fullPath, $length(moduleRoot) + 1, *) + if relPath '= "" { + set pResourceArray(fullPath) = "" + set pResourceArray(fullPath, "RelativePath") = relPath + } + } + } catch e { + set sc = e.AsStatus() + } + quit sc +} + Method OnExportItem( pFullExportPath As %String, pItemName As %String, diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls index 10bcaf99e..067b38a91 100644 --- a/src/cls/IPM/ResourceProcessor/Test.cls +++ b/src/cls/IPM/ResourceProcessor/Test.cls @@ -359,6 +359,60 @@ Method OnItemRelativePath(pItemName As %String) As %String quit ..EmbeddedProcessor.OnItemRelativePath(pItemName) } +Method SupportsSync() As %Boolean +{ + quit 1 +} + +Method OnSync(ByRef modifiedPaths, ByRef deletedPaths, ByRef params, Output handled As %Boolean = 0) As %Status +{ + set sc = $$$OK + set handled = 1 + try { + set verbose = $get(params("Verbose")) + set unitTestDir = ##class(%File).NormalizeDirectory(..ResourceReference.Module.Root _ ..ResourceReference.Name) + + // Reload changed test files, then compile so %Extends checks are valid below. + // Test resources are not AbstractCompilable, so SyncCompile never touches them — + // OnSync owns the full load+compile cycle for this resource type. + $$$ThrowOnError(##class(%IPM.Test.Manager).LoadTestDirectory(unitTestDir, verbose, .loadedList, ..Format)) + if ..Package '= "" { + $$$ThrowOnError($system.OBJ.CompilePackage(..Package, "ck"_$select(verbose:"d",1:"-d"))) + } elseif ..Class '= "" { + $$$ThrowOnError($system.OBJ.Compile(..Class, "ck"_$select(verbose:"d",1:"-d"))) + } + + // Record changed TestCase subclasses for SyncRunTests. + // relPath is relative to module root (e.g. "tests/unit/SyncTest/Tests/Trivial.cls"). + // Strip the resource directory prefix (e.g. "tests/unit/") to get the package-relative path. + set resourceDir = ##class(%IPM.Storage.FileHash).NormalizePath(..ResourceReference.Name) + set relPath = "" + for { + set relPath = $order(modifiedPaths(relPath)) + quit:relPath="" + + // Strip resource directory prefix to get path relative to the test dir + set packageRelPath = relPath + if $extract(packageRelPath, 1, $length(resourceDir)) = resourceDir { + set packageRelPath = $extract(packageRelPath, $length(resourceDir) + 1, *) + } + // Convert path to class name: "SyncTest/Tests/Trivial.cls" -> "SyncTest.Tests.Trivial" + set fileName = $piece(packageRelPath, "/", *) + set baseName = $piece(fileName, ".", 1, *-1) + set dirPart = $piece(packageRelPath, "/", 1, *-1) + set className = $select(dirPart '= "": $translate(dirPart, "/", ".") _ "." _ baseName, 1: baseName) + if $zconvert($piece(fileName, ".", *), "U") = "CLS" + && $$$comClassDefined(className) + && $classmethod(className, "%Extends", "%UnitTest.TestCase") { + set params("Sync", "ChangedTestCases", className) = "" + } + } + } catch e { + set sc = e.AsStatus() + } + quit sc +} + Method %OnValidateObject() As %Status [ Private, ServerOnly = 1 ] { if ((..Package = "") && (..Class = "")) || ((..Package '= "") && (..Class '= "")) { diff --git a/src/cls/IPM/Storage/FileHash.cls b/src/cls/IPM/Storage/FileHash.cls new file mode 100644 index 000000000..4e36dd97a --- /dev/null +++ b/src/cls/IPM/Storage/FileHash.cls @@ -0,0 +1,244 @@ +Include %IPM.Common + +Class %IPM.Storage.FileHash Extends %Persistent +{ + +Property ModuleName As %String(MAXLEN = 255) [ Required ]; + +/// Path relative to the module root, normalized: forward slashes, no leading slash, no double slashes. +/// Used as the lookup key in ModulePathIndex — callers must normalize via NormalizePath before querying. +Property RelativePath As %String(MAXLEN = 512) [ Required ]; + +/// SHA-1 content hash (hex). Empty string means the row was stamped with mtime/size only and +/// no hash was computed yet. On the next sync, any mtime/size mismatch will compute a fresh hash. +Property Hash As %String(MAXLEN = 64) [ InitialExpression = "" ]; + +/// Last-modified timestamp as returned by ##class(%File).GetFileDateModified — $H format. +/// Used as a fast-path: if mtime and FileSize both match, assume content is unchanged. +Property FileTimestamp As %String(MAXLEN = 64); + +/// File size in bytes. Combined with FileTimestamp forms the fast-path unchanged check. +/// Both fields are needed: size alone misses same-size edits; mtime alone is unreliable on copies. +Property FileSize As %Integer; + +Index ModulePathIndex On (ModuleName, RelativePath) [ Unique ]; + +Index ModuleNameIndex On ModuleName; + +ForeignKey ModuleNameFK(ModuleName) References %IPM.Storage.Module(Name) [ OnDelete = cascade ]; + +/// Record the current mtime+size for each tracked file without reading file content (no hash). +/// This "stamp" establishes a baseline so subsequent sync calls can detect changes via ComputeChanges. +/// Called after a successful dev-mode %Compile (after %Reload) so all resources, including +/// unit test classes compiled during %Compile, are present on disk and in ^oddDEF. +ClassMethod StampModule(module As %IPM.Storage.Module, ByRef trackedPaths) As %Status +{ + set sc = $$$OK + try { + set root = ##class(%File).NormalizeDirectory(module.Root) + write !, "[DEBUG StampModule] module=", module.Name, " root=", root + &sql(SELECT COUNT(*) INTO :debugCount FROM %IPM_Storage.FileHash WHERE ModuleName = :module.Name) + write !, "[DEBUG StampModule] existing rows=", debugCount, " SQLCODE=", SQLCODE + set debugCount = 0 + set debugKey = "" + for { + set debugKey = $order(trackedPaths(debugKey)) + quit:debugKey="" + set debugCount = debugCount + 1 + } + write !, "[DEBUG StampModule] tracked path count=", debugCount + set relPath = "" + for { + set relPath = $order(trackedPaths(relPath)) + quit:relPath="" + + set fullPath = ##class(%File).NormalizeFilename(relPath, root) + write !, "[DEBUG StampModule] relPath=", relPath, " fullPath=", fullPath, " exists=", ##class(%File).Exists(fullPath) + if '##class(%File).Exists(fullPath) { + continue + } + + set normalizedRelPath = ..NormalizePath(relPath) + set existing = ..ModulePathIndexOpen(module.Name, normalizedRelPath, , .openSC) + if $isobject(existing) { + set instance = existing + } else { + set instance = ..%New() + set instance.ModuleName = module.Name + set instance.RelativePath = normalizedRelPath + } + + set instance.FileSize = ##class(%File).GetFileSize(fullPath) + set instance.FileTimestamp = ..GetFileTimestamp(fullPath) + $$$ThrowOnError(instance.%Save()) + } + } catch e { + set sc = e.AsStatus() + } + quit sc +} + +/// Compute which tracked files changed on disk vs stored baseline. +/// Returns modified(relPath)=newHash and deleted(relPath)="" arrays. +ClassMethod ComputeChanges(module As %IPM.Storage.Module, ByRef trackedPaths, Output modified, Output deleted) As %Status +{ + set sc = $$$OK + kill modified, deleted + try { + set root = ##class(%File).NormalizeDirectory(module.Root) + + set relPath = "" + for { + set relPath = $order(trackedPaths(relPath)) + quit:relPath="" + + set normalizedRelPath = ..NormalizePath(relPath) + set fullPath = ##class(%File).NormalizeFilename(relPath, root) + + if '##class(%File).Exists(fullPath) { + if ..ModulePathIndexExists(module.Name, normalizedRelPath) { + set deleted(normalizedRelPath) = "" + } + continue + } + + set currentSize = ##class(%File).GetFileSize(fullPath) + set currentTimestamp = ..GetFileTimestamp(fullPath) + + set existing = ..ModulePathIndexOpen(module.Name, normalizedRelPath, , .openSC) + if '$isobject(existing) { + // No baseline row — file was never stamped; skip (not a tracked change) + continue + } + + // Fast path: size+mtime match means unchanged + if (existing.FileSize = currentSize) && (existing.FileTimestamp = currentTimestamp) { + continue + } + + // Size or mtime differ — read hash to confirm real change + set newHash = ##class(%File).SHA1Hash(fullPath, 1) + if (existing.Hash '= "") && (newHash = existing.Hash) { + // Hash matches stored — content unchanged despite mtime/size difference; update fast-path + set existing.FileTimestamp = currentTimestamp + set existing.FileSize = currentSize + $$$ThrowOnError(existing.%Save()) + } else { + // Content changed (or no prior hash to confirm otherwise) + set modified(normalizedRelPath) = newHash + } + } + } catch e { + set sc = e.AsStatus() + } + quit sc +} + +/// After a successful sync, commit new hashes for modified files and optionally remove deleted rows. +ClassMethod CommitChanges(module As %IPM.Storage.Module, ByRef modified, ByRef deleted, processDeletes As %Boolean = 0) As %Status +{ + set sc = $$$OK + try { + set root = ##class(%File).NormalizeDirectory(module.Root) + + set relPath = "" + for { + set relPath = $order(modified(relPath), 1, newHash) + quit:relPath="" + + set fullPath = ##class(%File).NormalizeFilename(relPath, root) + set existing = ..ModulePathIndexOpen(module.Name, relPath, , .openSC) + if $isobject(existing) { + set instance = existing + } else { + set instance = ..%New() + set instance.ModuleName = module.Name + set instance.RelativePath = relPath + } + + set instance.Hash = newHash + set instance.FileSize = ##class(%File).GetFileSize(fullPath) + set instance.FileTimestamp = ..GetFileTimestamp(fullPath) + $$$ThrowOnError(instance.%Save()) + } + + if processDeletes { + set relPath = "" + for { + set relPath = $order(deleted(relPath)) + quit:relPath="" + + set existing = ..ModulePathIndexOpen(module.Name, relPath, , .openSC) + if $isobject(existing) { + $$$ThrowOnError(existing.%DeleteId(existing.%Id())) + } + } + } + } catch e { + set sc = e.AsStatus() + } + quit sc +} + +/// Returns 1 if this module has any stored baseline rows. +ClassMethod HasBaseline(moduleName As %String) As %Boolean +{ + set result = ##class(%SQL.Statement).%ExecDirect(, + "SELECT TOP 1 1 FROM %IPM_Storage.FileHash WHERE ModuleName = ?", + moduleName) + write !, "[DEBUG HasBaseline] moduleName=", moduleName, " SQLCODE=", result.%SQLCODE + set found = result.%Next() + write !, "[DEBUG HasBaseline] found=", found + quit found +} + +/// Normalize a relative path: forward slashes, collapse //, strip leading slash. +ClassMethod NormalizePath(path As %String) As %String +{ + set path = $translate(path, "\", "/") + while path [ "//" { + set path = $replace(path, "//", "/") + } + while $extract(path) = "/" { + set path = $extract(path, 2, *) + } + quit path +} + +/// Get the last-modified timestamp of a file in $H format (as returned by GetFileDateModified). +ClassMethod GetFileTimestamp(fullPath As %String) As %String +{ + quit ##class(%File).GetFileDateModified(fullPath) +} + +Storage Default +{ + + +%%CLASSNAME + + +ModuleName + + +RelativePath + + +Hash + + +FileTimestamp + + +FileSize + + +^IPM.Storage.FileHashD +FileHashDefaultData +^IPM.Storage.FileHashD +^IPM.Storage.FileHashI +^IPM.Storage.FileHashS +%Storage.Persistent +} + +} diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls index 9f0ad18ac..e01acdc03 100644 --- a/src/cls/IPM/Storage/Module.cls +++ b/src/cls/IPM/Storage/Module.cls @@ -719,6 +719,508 @@ ClassMethod ExecutePhases( quit tSC } +/// Incrementally sync changed files from disk into the namespace for a dev-mode module. +/// Detects files changed since last load/sync, routes them to resource processors, and recompiles. +ClassMethod Sync(moduleName As %String, ByRef params) As %Status +{ + set sc = $$$OK + try { + set verbose = $get(params("Verbose"), 0) + set processDeletes = $get(params("ProcessDeletes"), 0) + set runTests = $get(params("RunTests"), 0) + + set module = ..NameOpen(moduleName, , .sc) + if '$isobject(module) { + $$$ThrowStatus($$$ERROR($$$GeneralError, $$$FormatText("Module '%1' not found.", moduleName))) + } + $$$ThrowOnError(sc) + + if 'module.DeveloperMode { + $$$ThrowStatus($$$ERROR($$$GeneralError, $$$FormatText("Module '%1' is not in development mode. Sync requires dev mode.", moduleName))) + } + + set root = ##class(%File).NormalizeDirectory(module.Root) + if root = "" { + $$$ThrowStatus($$$ERROR($$$GeneralError, $$$FormatText("Module '%1' has no root directory configured.", moduleName))) + } + + // Step 1: Check if module.xml changed; reload manifest if so + set moduleXmlRelPath = ##class(%IPM.Storage.FileHash).NormalizePath("module.xml") + set moduleXmlPath = root _ "module.xml" + set moduleXmlChanged = ..SyncCheckModuleXml(.module, moduleXmlPath, moduleXmlRelPath) + set lifecycle = module.Lifecycle + + // Step 2: Collect tracked paths and compute disk changes vs baseline + do lifecycle.GetTrackedPaths(.trackedPaths) + + if '##class(%IPM.Storage.FileHash).HasBaseline(moduleName) { + // Self-heal: establish baseline for modules loaded before this feature + do ##class(%IPM.Storage.FileHash).StampModule(module, .trackedPaths) + write !, "[", moduleName, "] Baseline established. Run sync again to detect changes." + quit + } + + $$$ThrowOnError(##class(%IPM.Storage.FileHash).ComputeChanges(module, .trackedPaths, .modified, .deleted)) + + // Exclude module.xml from file routing (handled by SyncCheckModuleXml above) + kill modified(moduleXmlRelPath) + kill deleted(moduleXmlRelPath) + + if '$data(modified) && ('$data(deleted) || 'processDeletes) { + if verbose { + write !, "[", moduleName, "] Nothing to sync." + } + if moduleXmlChanged { + do ..SyncCommitModuleXml(module, moduleXmlPath, moduleXmlRelPath) + do ..SyncPrintModuleXmlWarning() + } + quit + } + + // Step 3: Build reverse index (relPath -> owning resource + processor) + kill reverseIndex + do ..SyncBuildReverseIndex(module, .reverseIndex) + set orderedResourceList = module.GetOrderedResourceList() + + // Step 4: Partition changes by resource, separating unsupported processors + kill syncByResource, unsupportedWarnings + do ..SyncRouteChanges(.modified, .deleted, .reverseIndex, processDeletes, .syncByResource, .unsupportedWarnings) + + // Step 5: Dispatch OnSync to each processor; load unhandled compilable files + $$$ThrowOnError(..SyncDispatchProcessors(module, root, verbose, .syncByResource, .params, .loadItems)) + + // Step 6: Compile the full resource set with u-flag to pick up dependent recompiles + if loadItems > 0 || processDeletes { + $$$ThrowOnError(..SyncCompile(module, verbose, .params)) + } + + // Step 7: Delete server-side documents for removed files, then recompile + if processDeletes { + do ..SyncApplyDeletes(.deleted, .reverseIndex, .syncByResource, verbose) + $$$ThrowOnError(..SyncCompile(module, verbose, .params)) + } + + // Step 8: Run changed test-phase tests if -test flag is set + if runTests { + $$$ThrowOnError(..SyncRunTests(orderedResourceList, verbose, .params)) + } + + // Step 9: Commit new hashes on success (skipped on error so next sync re-detects) + $$$ThrowOnError(##class(%IPM.Storage.FileHash).CommitChanges(module, .modified, .deleted, processDeletes)) + if moduleXmlChanged { + do ..SyncCommitModuleXml(module, moduleXmlPath, moduleXmlRelPath) + } + + set modCount = 0 + set key = "" + for { + set key = $order(modified(key)) + quit:key="" + set modCount = modCount + 1 + } + set delCount = 0 + if processDeletes { + set key = "" + for { + set key = $order(deleted(key)) + quit:key="" + set delCount = delCount + 1 + } + } + write !, "[", moduleName, "] Sync complete: ", modCount, " file(s) updated" + if delCount > 0 { + write ", ", delCount, " deleted" + } + write "." + + if moduleXmlChanged { + do ..SyncPrintModuleXmlWarning() + } + do ..SyncPrintUnsupportedWarnings(.unsupportedWarnings) + + } catch e { + set sc = e.AsStatus() + do $system.OBJ.DisplayError(sc) + } + quit sc +} + +/// Check if module.xml changed and reload the manifest if so. +/// Returns 1 if module.xml changed and was reloaded, 0 otherwise. +/// On reload, replaces module with the freshly-opened instance. +ClassMethod SyncCheckModuleXml( + ByRef module As %IPM.Storage.Module, + moduleXmlPath As %String, + moduleXmlRelPath As %String) As %Boolean +{ + if '##class(%File).Exists(moduleXmlPath) { + quit 0 + } + set existing = ##class(%IPM.Storage.FileHash).ModulePathIndexOpen(module.Name, moduleXmlRelPath) + if '$isobject(existing) { + quit 0 + } + set currentSize = ##class(%File).GetFileSize(moduleXmlPath) + set currentTimestamp = ##class(%IPM.Storage.FileHash).GetFileTimestamp(moduleXmlPath) + if (existing.FileSize = currentSize) && (existing.FileTimestamp = currentTimestamp) { + quit 0 + } + set newHash = ##class(%File).SHA1Hash(moduleXmlPath, 1) + if (existing.Hash '= "") && (newHash = existing.Hash) { + quit 0 + } + $$$ThrowOnError($system.OBJ.Load(moduleXmlPath, "-d")) + $$$ThrowOnError(module.%Reload()) + set module = ..NameOpen(module.Name, , .openSC) + $$$ThrowOnError(openSC) + quit 1 +} + +/// Build a reverse index: normalizedRelPath -> resource name, Processor, Resource object. +/// Used by SyncRouteChanges to map changed files back to their owning resource processors. +ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, ByRef reverseIndex) +{ + set orderedResourceList = module.GetOrderedResourceList() + set key = "" + for { + set resource = orderedResourceList.GetNext(.key) + quit:key="" + + if '$isobject(resource.Processor) { + continue + } + + kill childArr + set childSC = resource.ResolveChildren(.childArr) + if $$$ISERR(childSC) { + continue + } + + set childName = "" + for { + set childName = $order(childArr(childName)) + quit:childName="" + + set relPath = $get(childArr(childName, "RelativePath")) + if relPath = "" { + set relPath = resource.Processor.OnItemRelativePath(childName) + } + if relPath = "" { + continue + } + + set normalizedRelPath = ##class(%IPM.Storage.FileHash).NormalizePath(relPath) + set reverseIndex(normalizedRelPath) = resource.Name + set reverseIndex(normalizedRelPath, "Processor") = resource.Processor + set reverseIndex(normalizedRelPath, "Resource") = resource + } + } +} + +/// Partition modified and deleted paths into syncByResource (keyed by resource name) and +/// unsupportedWarnings (for processors that don't support sync). +ClassMethod SyncRouteChanges( + ByRef modified, + ByRef deleted, + ByRef reverseIndex, + processDeletes As %Boolean, + ByRef syncByResource, + ByRef unsupportedWarnings) +{ + set relPath = "" + for { + set relPath = $order(modified(relPath)) + quit:relPath="" + + if '$data(reverseIndex(relPath)) { + continue + } + set resName = reverseIndex(relPath) + set processor = reverseIndex(relPath, "Processor") + if 'processor.SupportsSync() { + set unsupportedWarnings(resName, relPath) = "" + } else { + set syncByResource(resName, "modified", relPath) = "" + set syncByResource(resName, "Processor") = processor + set syncByResource(resName, "Resource") = reverseIndex(relPath, "Resource") + } + } + + if 'processDeletes { + quit + } + set relPath = "" + for { + set relPath = $order(deleted(relPath)) + quit:relPath="" + + if '$data(reverseIndex(relPath)) { + continue + } + set resName = reverseIndex(relPath) + set processor = reverseIndex(relPath, "Processor") + if 'processor.SupportsSync() { + set unsupportedWarnings(resName, relPath) = "" + } else { + set syncByResource(resName, "deleted", relPath) = "" + set syncByResource(resName, "Processor") = processor + set syncByResource(resName, "Resource") = reverseIndex(relPath, "Resource") + } + } +} + +/// Call OnSync on each processor, then load any unhandled compilable files. +/// Returns the number of files loaded (used to decide whether SyncCompile is needed). +ClassMethod SyncDispatchProcessors( + module As %IPM.Storage.Module, + root As %String, + verbose As %Boolean, + ByRef syncByResource, + ByRef params, + Output loadItems As %Integer = 0) As %Status +{ + set sc = $$$OK + try { + set resName = "" + for { + set resName = $order(syncByResource(resName)) + quit:resName="" + + if '$data(syncByResource(resName, "Processor")) { + continue + } + set processor = syncByResource(resName, "Processor") + + kill resModified, resDeleted + merge resModified = syncByResource(resName, "modified") + merge resDeleted = syncByResource(resName, "deleted") + + set handled = 0 + $$$ThrowOnError(processor.OnSync(.resModified, .resDeleted, .params, .handled)) + + if handled || 'processor.%IsA("%IPM.ResourceProcessor.AbstractCompilable") { + continue + } + + // Default: load each changed file; SyncCompile will do a full compile with u-flag + set loadRelPath = "" + for { + set loadRelPath = $order(resModified(loadRelPath)) + quit:loadRelPath="" + + set fullPath = ##class(%File).NormalizeFilename(loadRelPath, root) + if ##class(%File).Exists(fullPath) { + set loadFlags = $select(verbose:"d", 1:"-d") + $$$ThrowOnError($system.OBJ.Load(fullPath, loadFlags _ "c")) + set loadItems = loadItems + 1 + } + } + } + } catch e { + set sc = e.AsStatus() + } + quit sc +} + +/// Delete server-side documents for compilable deleted files. +ClassMethod SyncApplyDeletes( + ByRef deleted, + ByRef reverseIndex, + ByRef syncByResource, + verbose As %Boolean) +{ + set relPath = "" + for { + set relPath = $order(deleted(relPath)) + quit:relPath="" + + if '$data(reverseIndex(relPath)) { + continue + } + set resName = reverseIndex(relPath) + if '$data(syncByResource(resName, "Processor")) { + continue + } + set processor = syncByResource(resName, "Processor") + if 'processor.%IsA("%IPM.ResourceProcessor.AbstractCompilable") { + continue + } + set docName = ..RelPathToDocName(relPath) + if docName '= "" { + set delFlags = $select(verbose:"d", 1:"-d") + do $system.OBJ.Delete(docName, delFlags) + } + } +} + +/// Run test-phase tests for changed test case classes recorded in params("Sync","ChangedTestCases"). +ClassMethod SyncRunTests( + orderedResourceList As %ListOfObjects, + verbose As %Boolean, + ByRef params) As %Status +{ + set sc = $$$OK + try { + set className = "" + for { + set className = $order(params("Sync", "ChangedTestCases", className)) + quit:className="" + + + set testKey = "" + for { + set testResource = orderedResourceList.GetNext(.testKey) + quit:testKey="" + + if '$isobject(testResource.Processor) { + continue + } + if 'testResource.Processor.%IsA("%IPM.ResourceProcessor.Test") { + continue + } + if '$listfind(testResource.Processor.Phase, "test") { + write:verbose !, "Skipping verify-scoped test: ", className, " (use 'verify' to run)" + continue + } + kill testParams + merge testParams = params + set testParams("UnitTest", "Case") = className + set testParams("DeveloperMode") = 1 + set handled = 0 + $$$ThrowOnError(testResource.Processor.OnPhase("Test", .testParams, .handled)) + } + } + } catch e { + set sc = e.AsStatus() + } + quit sc +} + +/// Commit the current module.xml hash to the baseline. +ClassMethod SyncCommitModuleXml( + module As %IPM.Storage.Module, + moduleXmlPath As %String, + moduleXmlRelPath As %String) +{ + kill moduleXmlMod, emptyDel + set moduleXmlMod(moduleXmlRelPath) = ##class(%File).SHA1Hash(moduleXmlPath, 1) + do ##class(%IPM.Storage.FileHash).CommitChanges(module, .moduleXmlMod, .emptyDel, 0) +} + +ClassMethod SyncPrintModuleXmlWarning() +{ + write ! + write !, "Warning: module.xml changed and was reloaded." + write !, " Resources may have been added/removed. A full 'reload' may be needed" + write !, " to fully apply manifest-level changes (mappings, dependencies, defaults)." +} + +ClassMethod SyncPrintUnsupportedWarnings(ByRef unsupportedWarnings) +{ + set warnRes = "" + for { + set warnRes = $order(unsupportedWarnings(warnRes)) + quit:warnRes="" + + write !, "Warning: resource '", warnRes, "' does not support sync. Changed files:" + set warnPath = "" + for { + set warnPath = $order(unsupportedWarnings(warnRes, warnPath)) + quit:warnPath="" + write !, " ", warnPath + } + write !, " Run a full 'reload' to apply these changes." + } +} + +/// Compile all compilable resources in a module with the 'u' (skip-up-to-date) flag. +ClassMethod SyncCompile(module As %IPM.Storage.Module, verbose As %Boolean = 0, ByRef params) As %Status +{ + set sc = $$$OK + try { + set orderedResourceList = module.GetOrderedResourceList() + kill compileArray + + set key = "" + for { + set resource = orderedResourceList.GetNext(.key) + quit:key="" + + if '$isobject(resource.Processor) { + continue + } + if 'resource.Processor.%IsA("%IPM.ResourceProcessor.AbstractCompilable") { + continue + } + if 'resource.IsInScope("Compile") { + continue + } + if resource.Processor.CompileFromProject { + continue + } + + kill oneResourceList + set oneResourceList(resource.Name) = "" + set resSC = resource.ResolveChildren(.oneResourceList) + if $$$ISERR(resSC) { + continue + } + + set childKey = "" + for { + set childKey = $order(oneResourceList(childKey)) + quit:childKey="" + set ext = $zconvert($piece(childKey, ".", *), "U") + if ext = "CLS" { + set className = $piece(childKey, ".", 1, *-1) + if '$$$comClassDefined(className) { + continue + } + } + set compileArray(childKey) = "" + } + } + + if $data(compileArray) > 1 { + set flags = $select(verbose:"d", 1:"-d") _ "cku" + set sc = ##class(%IPM.Utils.LegacyCompat).UpdateSuperclassAndCompile(.compileArray, .flags) + } + } catch e { + set sc = e.AsStatus() + } + quit sc +} + +/// Convert a normalized relative path to an IRIS server document name. +ClassMethod RelPathToDocName(relPath As %String) As %String +{ + set fileName = $piece(relPath, "/", *) + set ext = $zconvert($piece(fileName, ".", *), "U") + set baseName = $piece(fileName, ".", 1, *-1) + + if ext = "CLS" { + // Convert path like src/cls/My/Package/Class.cls -> My.Package.Class.cls + set parts = $length(relPath, "/") + set className = "" + // Skip common source prefixes to get class package path + set startPiece = 1 + for i = 1:1:parts-1 { + set piece = $piece(relPath, "/", i) + if $listfind($listbuild("src", "cls"), $zconvert(piece, "L")) { + set startPiece = i + 1 + } + } + for i = startPiece:1:parts-1 { + set className = className _ $select(className="":"", 1:".") _ $piece(relPath, "/", i) + } + set className = className _ $select(className="":"", 1:".") _ baseName + quit className _ ".cls" + } elseif (ext = "MAC") || (ext = "INC") || (ext = "INT") { + quit baseName _ "." _ $zconvert(ext, "L") + } + quit "" +} + /// Uninstalls a named module (pModuleName). /// May optionally force installation (uninstalling even if required by other modules) if pForce is 1. /// May optionally recurse to also uninstall dependencies that are not required by other modules if pRecurse is 1. diff --git a/tests/integration_tests/Test/PM/Integration/Sync.cls b/tests/integration_tests/Test/PM/Integration/Sync.cls new file mode 100644 index 000000000..b64e27aa1 --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/Sync.cls @@ -0,0 +1,254 @@ +Class Test.PM.Integration.Sync Extends %UnitTest.TestCase +{ + +Property TempDir As %String; + +Method OnBeforeAllTests() As %Status +{ + set sourceDir = ..GetModuleDir("sync-test") + + set ..TempDir = ##class(%File).NormalizeDirectory(##class(%File).ManagerDirectory() _ "sync-test-" _ $job) + $$$ThrowOnError(##class(%IPM.Utils.File).CreateDirectoryChain(..TempDir)) + if '##class(%Library.File).CopyDir(sourceDir, ..TempDir, 1) { + quit $$$ERROR($$$GeneralError, "Failed to copy sync-test fixture to temp directory") + } + + // Load in dev mode — this stamps the baseline via %Compile hook. + quit ##class(%IPM.Main).Shell("load " _ ..TempDir _ " -dev") +} + +// Uninstall and restore the fixture after each test to prevent state leakage between tests. +// Uninstalling first handles version-change tests (e.g. TestModuleXmlChangedWarning bumps version). +Method OnAfterOneTest(testName As %String) As %Status +{ + try { + // Uninstall may fail if the test left the module in a broken state — that's expected. + do ##class(%IPM.Main).Shell("uninstall sync-test") + } catch {} + do ##class(%Library.File).RemoveDirectoryTree(..TempDir) + $$$ThrowOnError(##class(%IPM.Utils.File).CreateDirectoryChain(..TempDir)) + if '##class(%Library.File).CopyDir(..GetModuleDir("sync-test"), ..TempDir, 1) { + quit $$$ERROR($$$GeneralError, "Failed to restore sync-test fixture after test: " _ testName) + } + quit ##class(%IPM.Main).Shell("load " _ ..TempDir _ " -dev") +} + +Method OnAfterAllTests() As %Status +{ + // OnAfterOneTest already uninstalled and reloaded after the last test. + // This just removes the temp directory. + if ..TempDir '= "" { + do ##class(%Library.File).RemoveDirectoryTree(..TempDir) + } + quit $$$OK +} + +Method TestNoChangeIsNoOp() +{ + do $$$AssertTrue(##class(%IPM.Storage.FileHash).HasBaseline("sync-test"), "Baseline rows exist after load") + + kill params + set params("Verbose") = 1 + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + + do $$$AssertStatusOK(sc, "Sync with no changes succeeds") + do $$$AssertTrue(..FindInOutput(.output, "Nothing to sync"), "Reports nothing to sync") +} + +Method TestModifiedClassRecompiles() +{ + set filePath = ..TempDir _ "src/cls/SyncTest/SuperClass.cls" + do ..ReplaceInFile(filePath, """original""", """modified""") + + kill params + set params("Verbose") = 1 + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + + do $$$AssertStatusOK(sc, "Sync after class modification succeeds") + do $$$AssertTrue(..FindInOutput(.output, "Sync complete"), "Reports sync complete") +} + +Method TestSuperclassEditRecompilesSubclass() +{ + // Add property to SuperClass — SubClass recompiles via u-flag + set filePath = ..TempDir _ "src/cls/SyncTest/SuperClass.cls" + do ..ReplaceInFile(filePath, "Property BaseValue", "Property NewProp As %String;" _ $char(10) _ $char(10) _ "Property BaseValue") + + kill params + set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + do $$$AssertStatusOK(sc, "Sync after superclass edit succeeds (subclass recompiles via u-flag)") +} + +Method TestUntrackedFileIgnored() +{ + set junkDir = ..TempDir _ "node_modules/" + $$$ThrowOnError(##class(%IPM.Utils.File).CreateDirectoryChain(junkDir)) + set stream = ##class(%Stream.FileCharacter).%New() + $$$ThrowOnError(stream.LinkToFile(junkDir _ "x.js")) + $$$ThrowOnError(stream.Write("junk")) + $$$ThrowOnError(stream.%Save()) + set stream = "" + + kill params + set params("Verbose") = 1 + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + + do $$$AssertStatusOK(sc, "Sync ignores untracked files") + do $$$AssertNotTrue(..FindInOutput(.output, "x.js"), "Untracked file not mentioned in sync output") + do $$$AssertNotTrue(##class(%IPM.Storage.FileHash).ModulePathIndexExists("sync-test", "node_modules/x.js"), "No FileHash row for untracked file") +} + +Method TestDeleteSkippedByDefault() +{ + set filePath = ..TempDir _ "src/cls/SyncTest/Deletable.cls" + do ##class(%Library.File).Delete(filePath) + + // Without -delete: class still present + kill params + set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + do $$$AssertStatusOK(sc, "Sync without -delete succeeds") + do $$$AssertTrue($$$comClassDefined("SyncTest.Deletable"), "Deletable class still exists without -delete flag") + + // With -delete: class is removed from server + kill params + set params("ProcessDeletes") = 1 + set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + do $$$AssertStatusOK(sc, "Sync with -delete succeeds") + do $$$AssertNotTrue($$$comClassDefined("SyncTest.Deletable"), "Deletable class removed after -delete sync") +} + +Method TestModuleXmlChangedWarning() +{ + set filePath = ..TempDir _ "module.xml" + do ..ReplaceInFile(filePath, "1.0.0", "1.0.1") + + kill params + set params("Verbose") = 1 + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + + do $$$AssertStatusOK(sc, "Sync after module.xml change succeeds") + do $$$AssertTrue(..FindInOutput(.output, "module.xml changed"), "Warning about module.xml change is shown") +} + +Method TestNonSyncProcessorWarning() +{ + set filePath = ..TempDir _ "static/config.txt" + do ..ReplaceInFile(filePath, "static file content", "modified static content") + + kill params + set params("Verbose") = 1 + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + + do $$$AssertStatusOK(sc, "Sync with non-sync processor file change succeeds") + do $$$AssertTrue(..FindInOutput(.output, "does not support sync"), "Warning about non-sync resource shown") +} + +Method TestSyncTestFlag() +{ + set filePath = ..TempDir _ "tests/unit/SyncTest/Tests/Trivial.cls" + do ..ReplaceInFile(filePath, "This test always passes.", "This test always passes (modified).") + + // Without RunTests: loads the changed test but does not execute it + kill params + set params("Verbose") = 1 + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + do $$$AssertStatusOK(sc, "Sync without RunTests loads but does not run tests") + do $$$AssertNotTrue(..FindInOutput(.output, "Test Results"), "No test results without RunTests flag") + + // Modify again so the file shows as changed on the next sync + do ..ReplaceInFile(filePath, "(modified).", "(modified again).") + + // With RunTests: executes the changed test-phase test class + kill params, output + set params("RunTests") = 1 + set params("Verbose") = 1 + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + do $$$AssertStatusOK(sc, "Sync with RunTests runs changed tests") + do $$$AssertTrue(..FindInOutput(.output, "Test Results"), "Test results shown with RunTests flag") + do $$$AssertTrue(..FindInOutput(.output, "SyncTest.Tests.Trivial"), "Changed test class appears in output") +} + +Method TestFailedCompileRetries() +{ + // Introduce a syntax error + set filePath = ..TempDir _ "src/cls/SyncTest/SuperClass.cls" + set stream = ##class(%Stream.FileCharacter).%New() + $$$ThrowOnError(stream.LinkToFile(filePath)) + $$$ThrowOnError(stream.Write("Class SyncTest.SuperClass { INVALID SYNTAX HERE }")) + $$$ThrowOnError(stream.%Save()) + set stream = "" + + kill params + set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + do $$$AssertStatusNotOK(sc, "Sync fails with syntax error") + + // Fix and retry — CommitChanges was skipped on failure, so file is still detected as changed + set fixContent = "Class SyncTest.SuperClass" _ $char(10) _ "{" _ $char(10) _ $char(10) + _ "ClassMethod GetValue() As %String" _ $char(10) _ "{" _ $char(10) + _ " quit ""fixed""" _ $char(10) _ "}" _ $char(10) _ $char(10) _ "}" _ $char(10) + set stream = ##class(%Stream.FileCharacter).%New() + $$$ThrowOnError(stream.LinkToFile(filePath)) + $$$ThrowOnError(stream.Write(fixContent)) + $$$ThrowOnError(stream.%Save()) + set stream = "" + + kill params + set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + do $$$AssertStatusOK(sc, "Sync succeeds after fixing syntax error (retry works)") +} + +/// Returns the path to a fixture directory under _data/. +/// Matches the convention from Test.PM.Integration.Base.GetModuleDir. +ClassMethod GetModuleDir(subfolders... As %String) As %String +{ + set testRoot = ##class(%File).NormalizeDirectory($get(^UnitTestRoot)) + set testRoot = ##class(%File).GetDirectory(testRoot) + set moduleDir = ##class(%File).Construct(testRoot, "_data", subfolders...) + quit ##class(%File).NormalizeDirectory(moduleDir) +} + +/// Read a file, replace oldStr with newStr, write back. +ClassMethod ReplaceInFile(filePath As %String, oldStr As %String, newStr As %String) +{ + set stream = ##class(%Stream.FileCharacter).%New() + $$$ThrowOnError(stream.LinkToFile(filePath)) + set content = "" + while 'stream.AtEnd { + set content = content _ stream.ReadLine() _ $char(10) + } + set stream = "" + set content = $replace(content, oldStr, newStr) + set writeStream = ##class(%Stream.FileCharacter).%New() + $$$ThrowOnError(writeStream.LinkToFile(filePath)) + $$$ThrowOnError(writeStream.Write(content)) + $$$ThrowOnError(writeStream.%Save()) +} + +ClassMethod FindInOutput(ByRef output, searchString As %String) As %Boolean +{ + set sub = "" + for { + set sub = $order(output(sub), 1, line) + quit:sub="" + if line [ searchString { + return 1 + } + } + return 0 +} + +} diff --git a/tests/integration_tests/Test/PM/Integration/_data/sync-test/module.xml b/tests/integration_tests/Test/PM/Integration/_data/sync-test/module.xml new file mode 100644 index 000000000..31150b1c4 --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/sync-test/module.xml @@ -0,0 +1,19 @@ + + + + + + sync-test + 1.0.0 + module + src + + + + + + + + + + diff --git a/tests/integration_tests/Test/PM/Integration/_data/sync-test/src/cls/SyncTest/Consumer.cls b/tests/integration_tests/Test/PM/Integration/_data/sync-test/src/cls/SyncTest/Consumer.cls new file mode 100644 index 000000000..f4b32480b --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/sync-test/src/cls/SyncTest/Consumer.cls @@ -0,0 +1,11 @@ +Include SyncTest + +Class SyncTest.Consumer +{ + +ClassMethod GetMacroValue() As %String +{ + quit $$$SyncTestValue +} + +} diff --git a/tests/integration_tests/Test/PM/Integration/_data/sync-test/src/cls/SyncTest/Deletable.cls b/tests/integration_tests/Test/PM/Integration/_data/sync-test/src/cls/SyncTest/Deletable.cls new file mode 100644 index 000000000..f715e57ad --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/sync-test/src/cls/SyncTest/Deletable.cls @@ -0,0 +1,9 @@ +Class SyncTest.Deletable +{ + +ClassMethod Hello() As %String +{ + quit "I exist" +} + +} diff --git a/tests/integration_tests/Test/PM/Integration/_data/sync-test/src/cls/SyncTest/SubClass.cls b/tests/integration_tests/Test/PM/Integration/_data/sync-test/src/cls/SyncTest/SubClass.cls new file mode 100644 index 000000000..d08376198 --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/sync-test/src/cls/SyncTest/SubClass.cls @@ -0,0 +1,9 @@ +Class SyncTest.SubClass Extends SyncTest.SuperClass +{ + +ClassMethod GetInherited() As %String +{ + quit ..GetValue() +} + +} diff --git a/tests/integration_tests/Test/PM/Integration/_data/sync-test/src/cls/SyncTest/SuperClass.cls b/tests/integration_tests/Test/PM/Integration/_data/sync-test/src/cls/SyncTest/SuperClass.cls new file mode 100644 index 000000000..dbbd75533 --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/sync-test/src/cls/SyncTest/SuperClass.cls @@ -0,0 +1,11 @@ +Class SyncTest.SuperClass +{ + +Property BaseValue As %String [ InitialExpression = "original" ]; + +ClassMethod GetValue() As %String +{ + quit "original" +} + +} diff --git a/tests/integration_tests/Test/PM/Integration/_data/sync-test/src/inc/SyncTest.inc b/tests/integration_tests/Test/PM/Integration/_data/sync-test/src/inc/SyncTest.inc new file mode 100644 index 000000000..9b4a4b69d --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/sync-test/src/inc/SyncTest.inc @@ -0,0 +1,3 @@ +ROUTINE SyncTest [Type=INC] + +#define SyncTestValue "original-include" diff --git a/tests/integration_tests/Test/PM/Integration/_data/sync-test/static/config.txt b/tests/integration_tests/Test/PM/Integration/_data/sync-test/static/config.txt new file mode 100644 index 000000000..89abc6b8a --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/sync-test/static/config.txt @@ -0,0 +1 @@ +static file content diff --git a/tests/integration_tests/Test/PM/Integration/_data/sync-test/tests/unit/SyncTest/Tests/Trivial.cls b/tests/integration_tests/Test/PM/Integration/_data/sync-test/tests/unit/SyncTest/Tests/Trivial.cls new file mode 100644 index 000000000..81dde040c --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/sync-test/tests/unit/SyncTest/Tests/Trivial.cls @@ -0,0 +1,9 @@ +Class SyncTest.Tests.Trivial Extends %UnitTest.TestCase +{ + +Method TestAlwaysPasses() +{ + do $$$AssertTrue(1, "This test always passes.") +} + +} From ee7986cdee9003a0308adc1565d227139384b4a8 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Wed, 15 Jul 2026 11:37:13 -0400 Subject: [PATCH 02/39] Remove debug code and add better output --- src/cls/IPM/Lifecycle/Base.cls | 19 ++----------------- src/cls/IPM/Storage/FileHash.cls | 17 +---------------- src/cls/IPM/Storage/Module.cls | 6 +++--- 3 files changed, 6 insertions(+), 36 deletions(-) diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls index 85d759764..94b4586d7 100644 --- a/src/cls/IPM/Lifecycle/Base.cls +++ b/src/cls/IPM/Lifecycle/Base.cls @@ -770,14 +770,6 @@ Method GetTrackedPaths(Output trackedPaths) set trackedPaths(##class(%IPM.Storage.FileHash).NormalizePath(relPath)) = "" } } - set debugCount = 0 - set debugKey = "" - for { - set debugKey = $order(trackedPaths(debugKey)) - quit:debugKey="" - set debugCount = debugCount + 1 - } - write !, "[DEBUG GetTrackedPaths] module=", ..Module.Name, " trackedCount=", debugCount } Method InstallOrDownloadPythonRequirements( @@ -1263,15 +1255,8 @@ Method %Compile(ByRef pParams) As %Status try { set trackedPaths = "" do ..GetTrackedPaths(.trackedPaths) - write !, "[DEBUG %Compile] Stamping baselines for module: ", ..Module.Name - set stampSC = ##class(%IPM.Storage.FileHash).StampModule(..Module, .trackedPaths) - if $$$ISERR(stampSC) { - write !, "[DEBUG %Compile] StampModule error: ", $system.Status.GetErrorText(stampSC) - } else { - write !, "[DEBUG %Compile] StampModule OK" - } - } catch stampErr { - write !, "[DEBUG %Compile] StampModule exception: ", stampErr.DisplayString() + do ##class(%IPM.Storage.FileHash).StampModule(..Module, .trackedPaths) + } catch { } } } catch e { diff --git a/src/cls/IPM/Storage/FileHash.cls b/src/cls/IPM/Storage/FileHash.cls index 4e36dd97a..dbafb8b85 100644 --- a/src/cls/IPM/Storage/FileHash.cls +++ b/src/cls/IPM/Storage/FileHash.cls @@ -36,24 +36,12 @@ ClassMethod StampModule(module As %IPM.Storage.Module, ByRef trackedPaths) As %S set sc = $$$OK try { set root = ##class(%File).NormalizeDirectory(module.Root) - write !, "[DEBUG StampModule] module=", module.Name, " root=", root - &sql(SELECT COUNT(*) INTO :debugCount FROM %IPM_Storage.FileHash WHERE ModuleName = :module.Name) - write !, "[DEBUG StampModule] existing rows=", debugCount, " SQLCODE=", SQLCODE - set debugCount = 0 - set debugKey = "" - for { - set debugKey = $order(trackedPaths(debugKey)) - quit:debugKey="" - set debugCount = debugCount + 1 - } - write !, "[DEBUG StampModule] tracked path count=", debugCount set relPath = "" for { set relPath = $order(trackedPaths(relPath)) quit:relPath="" set fullPath = ##class(%File).NormalizeFilename(relPath, root) - write !, "[DEBUG StampModule] relPath=", relPath, " fullPath=", fullPath, " exists=", ##class(%File).Exists(fullPath) if '##class(%File).Exists(fullPath) { continue } @@ -186,10 +174,7 @@ ClassMethod HasBaseline(moduleName As %String) As %Boolean set result = ##class(%SQL.Statement).%ExecDirect(, "SELECT TOP 1 1 FROM %IPM_Storage.FileHash WHERE ModuleName = ?", moduleName) - write !, "[DEBUG HasBaseline] moduleName=", moduleName, " SQLCODE=", result.%SQLCODE - set found = result.%Next() - write !, "[DEBUG HasBaseline] found=", found - quit found + quit result.%Next() } /// Normalize a relative path: forward slashes, collapse //, strip leading slash. diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls index e01acdc03..f98d25740 100644 --- a/src/cls/IPM/Storage/Module.cls +++ b/src/cls/IPM/Storage/Module.cls @@ -767,9 +767,7 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status kill deleted(moduleXmlRelPath) if '$data(modified) && ('$data(deleted) || 'processDeletes) { - if verbose { - write !, "[", moduleName, "] Nothing to sync." - } + write !, "[", moduleName, "] Nothing to sync." if moduleXmlChanged { do ..SyncCommitModuleXml(module, moduleXmlPath, moduleXmlRelPath) do ..SyncPrintModuleXmlWarning() @@ -817,6 +815,7 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status set key = $order(modified(key)) quit:key="" set modCount = modCount + 1 + write !, " Updated: ", key } set delCount = 0 if processDeletes { @@ -825,6 +824,7 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status set key = $order(deleted(key)) quit:key="" set delCount = delCount + 1 + write !, " Deleted: ", key } } write !, "[", moduleName, "] Sync complete: ", modCount, " file(s) updated" From ac87895bc848f91d51948b982a675c105fb4472d Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Wed, 15 Jul 2026 13:13:40 -0400 Subject: [PATCH 03/39] Fix some small issues --- docker-compose.yml | 6 +-- src/cls/IPM/Lifecycle/Base.cls | 7 ++-- src/cls/IPM/Main.cls | 1 - src/cls/IPM/ResourceProcessor/Test.cls | 2 +- src/cls/IPM/Storage/Module.cls | 51 ++++++++++++-------------- 5 files changed, 32 insertions(+), 35 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 973e3285f..7f334806e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: - TEST_REGISTRY_USER=admin - TEST_REGISTRY_PASSWORD=SYS ports: - - 52777:52773 + - 52774:52773 volumes: - ~/iris.key:/usr/irissys/mgr/iris.key - ./:/home/irisowner/zpm/ @@ -28,7 +28,7 @@ services: build: ./tests/sandbox/ restart: always ports: - - 52779:52773 + - 52776:52773 environment: - IRISPASSWORD=SYS - IRISUSERNAME=admin @@ -43,4 +43,4 @@ services: image: ghcr.io/project-zot/zot-linux-amd64:latest restart: always ports: - - 5002:5000 + - 5001:5000 diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls index 94b4586d7..ea7072b97 100644 --- a/src/cls/IPM/Lifecycle/Base.cls +++ b/src/cls/IPM/Lifecycle/Base.cls @@ -732,7 +732,6 @@ Method %Reload(ByRef pParams) As %Status Method GetTrackedPaths(Output trackedPaths) { kill trackedPaths - set root = ##class(%File).NormalizeDirectory(..Module.Root) // Include module.xml set trackedPaths(##class(%IPM.Storage.FileHash).NormalizePath("module.xml")) = "" @@ -1249,14 +1248,16 @@ Method %Compile(ByRef pParams) As %Status } } - // Stamp file baselines for sync change detection (dev mode only, non-fatal). + // Stamp file baselines for sync change detection (dev mode only). + // Non-fatal: stamping failure must not break a normal compile cycle. // Done here after compile so test classes are in ^oddDEF and appear in trackedPaths. if tDevMode { try { set trackedPaths = "" do ..GetTrackedPaths(.trackedPaths) do ##class(%IPM.Storage.FileHash).StampModule(..Module, .trackedPaths) - } catch { + } catch stampEx { + write !, "Warning: sync baseline stamping failed: ", $system.Status.GetOneErrorText(stampEx.AsStatus()) } } } catch e { diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls index f0c5c7add..18d9cf03a 100644 --- a/src/cls/IPM/Main.cls +++ b/src/cls/IPM/Main.cls @@ -115,7 +115,6 @@ If no module is specified, syncs all modules in development mode. - diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls index 067b38a91..f9f8198ad 100644 --- a/src/cls/IPM/ResourceProcessor/Test.cls +++ b/src/cls/IPM/ResourceProcessor/Test.cls @@ -404,7 +404,7 @@ Method OnSync(ByRef modifiedPaths, ByRef deletedPaths, ByRef params, Output hand if $zconvert($piece(fileName, ".", *), "U") = "CLS" && $$$comClassDefined(className) && $classmethod(className, "%Extends", "%UnitTest.TestCase") { - set params("Sync", "ChangedTestCases", className) = "" + set params("Sync", "ChangedTestCases", className) = ..ResourceReference.Name } } } catch e { diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls index f98d25740..59a422c95 100644 --- a/src/cls/IPM/Storage/Module.cls +++ b/src/cls/IPM/Storage/Module.cls @@ -766,6 +766,7 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status kill modified(moduleXmlRelPath) kill deleted(moduleXmlRelPath) + // Falls through when only deletes exist and processDeletes=1 if '$data(modified) && ('$data(deleted) || 'processDeletes) { write !, "[", moduleName, "] Nothing to sync." if moduleXmlChanged { @@ -788,12 +789,12 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status $$$ThrowOnError(..SyncDispatchProcessors(module, root, verbose, .syncByResource, .params, .loadItems)) // Step 6: Compile the full resource set with u-flag to pick up dependent recompiles - if loadItems > 0 || processDeletes { + if loadItems > 0 { $$$ThrowOnError(..SyncCompile(module, verbose, .params)) } // Step 7: Delete server-side documents for removed files, then recompile - if processDeletes { + if processDeletes && ($data(deleted) > 1) { do ..SyncApplyDeletes(.deleted, .reverseIndex, .syncByResource, verbose) $$$ThrowOnError(..SyncCompile(module, verbose, .params)) } @@ -927,31 +928,23 @@ ClassMethod SyncRouteChanges( ByRef syncByResource, ByRef unsupportedWarnings) { - set relPath = "" - for { - set relPath = $order(modified(relPath)) - quit:relPath="" - - if '$data(reverseIndex(relPath)) { - continue - } - set resName = reverseIndex(relPath) - set processor = reverseIndex(relPath, "Processor") - if 'processor.SupportsSync() { - set unsupportedWarnings(resName, relPath) = "" - } else { - set syncByResource(resName, "modified", relPath) = "" - set syncByResource(resName, "Processor") = processor - set syncByResource(resName, "Resource") = reverseIndex(relPath, "Resource") - } + do ..SyncRoutePathSet(.modified, .reverseIndex, "modified", .syncByResource, .unsupportedWarnings) + if processDeletes { + do ..SyncRoutePathSet(.deleted, .reverseIndex, "deleted", .syncByResource, .unsupportedWarnings) } +} - if 'processDeletes { - quit - } +/// Route a set of changed paths to their owning resources, categorized by type (modified/deleted). +ClassMethod SyncRoutePathSet( + ByRef paths, + ByRef reverseIndex, + category As %String, + ByRef syncByResource, + ByRef unsupportedWarnings) [ Private ] +{ set relPath = "" for { - set relPath = $order(deleted(relPath)) + set relPath = $order(paths(relPath)) quit:relPath="" if '$data(reverseIndex(relPath)) { @@ -962,7 +955,7 @@ ClassMethod SyncRouteChanges( if 'processor.SupportsSync() { set unsupportedWarnings(resName, relPath) = "" } else { - set syncByResource(resName, "deleted", relPath) = "" + set syncByResource(resName, category, relPath) = "" set syncByResource(resName, "Processor") = processor set syncByResource(resName, "Resource") = reverseIndex(relPath, "Resource") } @@ -1063,10 +1056,9 @@ ClassMethod SyncRunTests( try { set className = "" for { - set className = $order(params("Sync", "ChangedTestCases", className)) + set className = $order(params("Sync", "ChangedTestCases", className), 1, owningResource) quit:className="" - set testKey = "" for { set testResource = orderedResourceList.GetNext(.testKey) @@ -1078,6 +1070,9 @@ ClassMethod SyncRunTests( if 'testResource.Processor.%IsA("%IPM.ResourceProcessor.Test") { continue } + if testResource.Name '= owningResource { + continue + } if '$listfind(testResource.Processor.Phase, "test") { write:verbose !, "Skipping verify-scoped test: ", className, " (use 'verify' to run)" continue @@ -1202,12 +1197,14 @@ ClassMethod RelPathToDocName(relPath As %String) As %String // Convert path like src/cls/My/Package/Class.cls -> My.Package.Class.cls set parts = $length(relPath, "/") set className = "" - // Skip common source prefixes to get class package path + // Skip leading source prefixes (e.g. "src/cls/") to get class package path set startPiece = 1 for i = 1:1:parts-1 { set piece = $piece(relPath, "/", i) if $listfind($listbuild("src", "cls"), $zconvert(piece, "L")) { set startPiece = i + 1 + } else { + quit } } for i = startPiece:1:parts-1 { From 670890dca86c3b55fce0ebe57be251d7984f8d29 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Wed, 15 Jul 2026 13:25:11 -0400 Subject: [PATCH 04/39] Reorder operations --- src/cls/IPM/Storage/Module.cls | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls index 59a422c95..c0a95b4cc 100644 --- a/src/cls/IPM/Storage/Module.cls +++ b/src/cls/IPM/Storage/Module.cls @@ -799,12 +799,9 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status $$$ThrowOnError(..SyncCompile(module, verbose, .params)) } - // Step 8: Run changed test-phase tests if -test flag is set - if runTests { - $$$ThrowOnError(..SyncRunTests(orderedResourceList, verbose, .params)) - } - - // Step 9: Commit new hashes on success (skipped on error so next sync re-detects) + // Step 8: Commit new hashes on success (skipped on error so next sync re-detects). + // Done before running tests below so a test failure doesn't prevent the file-sync + // outcome from being committed and reported — file sync and test results are independent. $$$ThrowOnError(##class(%IPM.Storage.FileHash).CommitChanges(module, .modified, .deleted, processDeletes)) if moduleXmlChanged { do ..SyncCommitModuleXml(module, moduleXmlPath, moduleXmlRelPath) @@ -839,6 +836,11 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status } do ..SyncPrintUnsupportedWarnings(.unsupportedWarnings) + // Step 9: Run changed test-phase tests if -test flag is set (after sync is reported) + if runTests { + $$$ThrowOnError(..SyncRunTests(orderedResourceList, verbose, .params)) + } + } catch e { set sc = e.AsStatus() do $system.OBJ.DisplayError(sc) From baa8b4f0c035ab181bbd9e6e34534223af6d130a Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Wed, 15 Jul 2026 14:27:03 -0400 Subject: [PATCH 05/39] Add more tests --- src/cls/IPM/Storage/FileHash.cls | 4 +- src/cls/IPM/Storage/Module.cls | 27 ++++ .../Test/PM/Integration/Sync.cls | 144 ++++++++++++++++++ .../PM/Integration/_data/sync-test/module.xml | 1 + .../tests/unit2/SyncTest2/Tests/Trivial2.cls | 9 ++ 5 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 tests/integration_tests/Test/PM/Integration/_data/sync-test/tests/unit2/SyncTest2/Tests/Trivial2.cls diff --git a/src/cls/IPM/Storage/FileHash.cls b/src/cls/IPM/Storage/FileHash.cls index dbafb8b85..77654cb56 100644 --- a/src/cls/IPM/Storage/FileHash.cls +++ b/src/cls/IPM/Storage/FileHash.cls @@ -95,7 +95,9 @@ ClassMethod ComputeChanges(module As %IPM.Storage.Module, ByRef trackedPaths, Ou set existing = ..ModulePathIndexOpen(module.Name, normalizedRelPath, , .openSC) if '$isobject(existing) { - // No baseline row — file was never stamped; skip (not a tracked change) + // No baseline row — file is newly tracked (added since the last sync/stamp). + // Report it as modified so it gets loaded; no prior hash to compare against. + set modified(normalizedRelPath) = ##class(%File).SHA1Hash(fullPath, 1) continue } diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls index c0a95b4cc..289a9b5b1 100644 --- a/src/cls/IPM/Storage/Module.cls +++ b/src/cls/IPM/Storage/Module.cls @@ -785,6 +785,11 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status kill syncByResource, unsupportedWarnings do ..SyncRouteChanges(.modified, .deleted, .reverseIndex, processDeletes, .syncByResource, .unsupportedWarnings) + // Changes routed to a processor that doesn't support sync were never applied. + // Drop them from modified/deleted so CommitChanges (Step 8) doesn't advance their + // baseline — otherwise the next sync would see "no change" despite the pending edit. + do ..SyncExcludeUnsupported(.unsupportedWarnings, .modified, .deleted) + // Step 5: Dispatch OnSync to each processor; load unhandled compilable files $$$ThrowOnError(..SyncDispatchProcessors(module, root, verbose, .syncByResource, .params, .loadItems)) @@ -936,6 +941,28 @@ ClassMethod SyncRouteChanges( } } +/// Remove paths routed to a non-sync-supporting processor from modified/deleted, since +/// those changes were never applied and must not advance the FileHash baseline. +ClassMethod SyncExcludeUnsupported( + ByRef unsupportedWarnings, + ByRef modified, + ByRef deleted) +{ + set resName = "" + for { + set resName = $order(unsupportedWarnings(resName)) + quit:resName="" + + set relPath = "" + for { + set relPath = $order(unsupportedWarnings(resName, relPath)) + quit:relPath="" + kill modified(relPath) + kill deleted(relPath) + } + } +} + /// Route a set of changed paths to their owning resources, categorized by type (modified/deleted). ClassMethod SyncRoutePathSet( ByRef paths, diff --git a/tests/integration_tests/Test/PM/Integration/Sync.cls b/tests/integration_tests/Test/PM/Integration/Sync.cls index b64e27aa1..6bdf72e21 100644 --- a/tests/integration_tests/Test/PM/Integration/Sync.cls +++ b/tests/integration_tests/Test/PM/Integration/Sync.cls @@ -43,6 +43,7 @@ Method OnAfterAllTests() As %Status quit $$$OK } +/// Sync with no files changed on disk since the last load/sync reports nothing to do. Method TestNoChangeIsNoOp() { do $$$AssertTrue(##class(%IPM.Storage.FileHash).HasBaseline("sync-test"), "Baseline rows exist after load") @@ -57,6 +58,7 @@ Method TestNoChangeIsNoOp() do $$$AssertTrue(..FindInOutput(.output, "Nothing to sync"), "Reports nothing to sync") } +/// A modified class file is detected and recompiled by sync. Method TestModifiedClassRecompiles() { set filePath = ..TempDir _ "src/cls/SyncTest/SuperClass.cls" @@ -72,6 +74,8 @@ Method TestModifiedClassRecompiles() do $$$AssertTrue(..FindInOutput(.output, "Sync complete"), "Reports sync complete") } +/// Editing a superclass recompiles its subclasses via the u-flag, even though the +/// subclass's own file on disk never changed. Method TestSuperclassEditRecompilesSubclass() { // Add property to SuperClass — SubClass recompiles via u-flag @@ -83,6 +87,8 @@ Method TestSuperclassEditRecompilesSubclass() do $$$AssertStatusOK(sc, "Sync after superclass edit succeeds (subclass recompiles via u-flag)") } +/// A file outside every tracked resource (e.g. under node_modules/) is ignored by sync +/// and never given a baseline row. Method TestUntrackedFileIgnored() { set junkDir = ..TempDir _ "node_modules/" @@ -104,6 +110,8 @@ Method TestUntrackedFileIgnored() do $$$AssertNotTrue(##class(%IPM.Storage.FileHash).ModulePathIndexExists("sync-test", "node_modules/x.js"), "No FileHash row for untracked file") } +/// A deleted file is left alone unless -delete is passed. With -delete, the corresponding +/// server-side class is removed. Method TestDeleteSkippedByDefault() { set filePath = ..TempDir _ "src/cls/SyncTest/Deletable.cls" @@ -123,6 +131,8 @@ Method TestDeleteSkippedByDefault() do $$$AssertNotTrue($$$comClassDefined("SyncTest.Deletable"), "Deletable class removed after -delete sync") } +/// A changed module.xml is detected and its manifest reloaded, with a warning that a full +/// reload may still be needed to apply manifest-level changes (mappings, dependencies, etc.). Method TestModuleXmlChangedWarning() { set filePath = ..TempDir _ "module.xml" @@ -138,6 +148,8 @@ Method TestModuleXmlChangedWarning() do $$$AssertTrue(..FindInOutput(.output, "module.xml changed"), "Warning about module.xml change is shown") } +/// A changed file owned by a resource processor that doesn't support sync produces a +/// warning directing the user to run a full reload. Method TestNonSyncProcessorWarning() { set filePath = ..TempDir _ "static/config.txt" @@ -153,6 +165,8 @@ Method TestNonSyncProcessorWarning() do $$$AssertTrue(..FindInOutput(.output, "does not support sync"), "Warning about non-sync resource shown") } +/// Without -test, a changed test-phase class is loaded but not executed. With -test, it is +/// executed and its results are shown. Method TestSyncTestFlag() { set filePath = ..TempDir _ "tests/unit/SyncTest/Tests/Trivial.cls" @@ -182,6 +196,136 @@ Method TestSyncTestFlag() do $$$AssertTrue(..FindInOutput(.output, "SyncTest.Tests.Trivial"), "Changed test class appears in output") } +/// A change to a file owned by a processor that doesn't support sync is never applied, so +/// its baseline must stay unchanged. The warning should reappear on every subsequent sync +/// until the change is actually applied (e.g. via a full reload). +Method TestUnsupportedProcessorChangeNotCommitted() +{ + set filePath = ..TempDir _ "static/config.txt" + do ..ReplaceInFile(filePath, "static file content", "modified static content") + + kill params + set params("Verbose") = 1 + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + do $$$AssertStatusOK(sc, "Sync with non-sync processor file change succeeds") + do $$$AssertTrue(..FindInOutput(.output, "does not support sync"), "Warning about non-sync resource shown") + + // Baseline must NOT reflect the on-disk change, since it was never applied. + // If it were committed, running sync again would report "Nothing to sync" and + // silently drop the warning even though config.txt server-side content is still stale. + kill params2 + set params2("Verbose") = 1 + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie2) + set sc2 = ##class(%IPM.Storage.Module).Sync("sync-test", .params2) + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie2, .output2) + do $$$AssertStatusOK(sc2, "Second sync succeeds") + do $$$AssertTrue(..FindInOutput(.output2, "does not support sync"), "Warning still shown on next sync (baseline was not falsely committed)") +} + +/// A brand-new file added under a directory-scanned FileCopy resource is recognized as a +/// change on the next sync. FileCopy resources are directory-scanned (OnResolveChildren +/// queries the source directory live), so a new file here is discoverable without any +/// module.xml change — unlike individually-declared classes, which require a +/// manifest entry regardless of sync (see TestModuleXmlAddsResourcePicksUpNewFile). +Method TestNewFileInFileCopyResourceDetected() +{ + set filePath = ..TempDir _ "static/new-file.txt" + set stream = ##class(%Stream.FileCharacter).%New() + $$$ThrowOnError(stream.LinkToFile(filePath)) + $$$ThrowOnError(stream.Write("new static file")) + $$$ThrowOnError(stream.%Save()) + set stream = "" + + kill params + set params("Verbose") = 1 + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + + do $$$AssertStatusOK(sc, "Sync with a new FileCopy file succeeds") + do $$$AssertTrue(..FindInOutput(.output, "does not support sync"), "New file under a non-sync-supporting resource is recognized as a change (warned, not silently ignored)") +} + +/// With two UnitTest resources present, a changed test class in one resource's directory +/// must only be dispatched through that owning resource — not through every Test processor +/// in the module (which would waste a reload+compile+RunTest cycle per extra resource). +Method TestSyncTestFlagOnlyRunsOwningResource() +{ + set filePath = ..TempDir _ "tests/unit/SyncTest/Tests/Trivial.cls" + do ..ReplaceInFile(filePath, "This test always passes.", "This test always passes (modified).") + + kill params + set params("RunTests") = 1 + set params("Verbose") = 1 + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + + do $$$AssertStatusOK(sc, "Sync with RunTests succeeds") + do $$$AssertTrue(..FindInOutput(.output, "SyncTest.Tests.Trivial"), "Changed test in tests/unit ran") + do $$$AssertNotTrue(..FindInOutput(.output, "SyncTest2.Tests.Trivial2"), "Unrelated test in tests/unit2 was not run") +} + +/// Running sync with no module name syncs every module currently in development mode. +Method TestSyncAllDevModeModules() +{ + kill params + set sc = ##class(%IPM.Main).Shell("sync") + do $$$AssertStatusOK(sc, "Sync with no module name (sync-all) succeeds") +} + +/// Sync fails with an error when given a module name that isn't installed. +Method TestSyncModuleNotFound() +{ + kill params + set sc = ##class(%IPM.Storage.Module).Sync("this-module-does-not-exist", .params) + do $$$AssertStatusNotOK(sc, "Sync fails for a module that doesn't exist") +} + +/// Sync fails with an error when the target module is not in development mode. +Method TestSyncNonDevModeModule() +{ + // Reload without -dev so the installed module is not in development mode. + // OnAfterOneTest unconditionally uninstalls and reloads -dev afterward, so no manual restore is needed here. + do ##class(%IPM.Main).Shell("uninstall sync-test") + $$$ThrowOnError(##class(%IPM.Main).Shell("load " _ ..TempDir)) + + kill params + set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + do $$$AssertStatusNotOK(sc, "Sync fails for a module not in development mode") +} + +/// module.xml adds a brand-new in the same edit that changes the version. +/// After SyncCheckModuleXml reloads the manifest, the newly-declared resource's files +/// must be tracked and picked up in that same sync call (not require a separate 'reload'). +Method TestModuleXmlAddsResourcePicksUpNewFile() +{ + set clsFilePath = ..TempDir _ "src/cls/SyncTest/AddedByManifest.cls" + set stream = ##class(%Stream.FileCharacter).%New() + $$$ThrowOnError(stream.LinkToFile(clsFilePath)) + $$$ThrowOnError(stream.Write("Class SyncTest.AddedByManifest" _ $char(10) _ "{" _ $char(10) _ "}" _ $char(10))) + $$$ThrowOnError(stream.%Save()) + set stream = "" + + set moduleXmlPath = ..TempDir _ "module.xml" + do ..ReplaceInFile(moduleXmlPath, "1.0.0", "1.0.1") + do ..ReplaceInFile(moduleXmlPath, "", "" _ $char(10) _ " ") + + kill params + set params("Verbose") = 1 + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + + do $$$AssertStatusOK(sc, "Sync after module.xml adds a resource succeeds") + do $$$AssertTrue(..FindInOutput(.output, "module.xml changed"), "Warning about module.xml change is shown") + do $$$AssertTrue($$$comClassDefined("SyncTest.AddedByManifest"), "Newly-declared resource's class was loaded and compiled in the same sync call") +} + +/// A sync that fails to compile leaves its baseline uncommitted, so the same file is +/// re-detected as changed and retried on the next sync once the error is fixed. Method TestFailedCompileRetries() { // Introduce a syntax error diff --git a/tests/integration_tests/Test/PM/Integration/_data/sync-test/module.xml b/tests/integration_tests/Test/PM/Integration/_data/sync-test/module.xml index 31150b1c4..a7671723e 100644 --- a/tests/integration_tests/Test/PM/Integration/_data/sync-test/module.xml +++ b/tests/integration_tests/Test/PM/Integration/_data/sync-test/module.xml @@ -13,6 +13,7 @@ + diff --git a/tests/integration_tests/Test/PM/Integration/_data/sync-test/tests/unit2/SyncTest2/Tests/Trivial2.cls b/tests/integration_tests/Test/PM/Integration/_data/sync-test/tests/unit2/SyncTest2/Tests/Trivial2.cls new file mode 100644 index 000000000..c9df607d4 --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/sync-test/tests/unit2/SyncTest2/Tests/Trivial2.cls @@ -0,0 +1,9 @@ +Class SyncTest2.Tests.Trivial2 Extends %UnitTest.TestCase +{ + +Method TestAlwaysPasses() +{ + do $$$AssertTrue(1, "This test always passes too.") +} + +} From 3c9fd2d6b2a49f29df72dc1f2aa0a97590270abd Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Wed, 15 Jul 2026 14:58:38 -0400 Subject: [PATCH 06/39] Cut test time by half through native copy of the test fixture --- .../Test/PM/Integration/Sync.cls | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/tests/integration_tests/Test/PM/Integration/Sync.cls b/tests/integration_tests/Test/PM/Integration/Sync.cls index 6bdf72e21..d066fa68b 100644 --- a/tests/integration_tests/Test/PM/Integration/Sync.cls +++ b/tests/integration_tests/Test/PM/Integration/Sync.cls @@ -3,13 +3,24 @@ Class Test.PM.Integration.Sync Extends %UnitTest.TestCase Property TempDir As %String; +// Native-filesystem copy of the fixture, made once. _data/sync-test/ lives on the repo's +// bind mount (slow for the many small-file copies OnAfterOneTest does between every test), +// so each restore copies from here instead of re-reading the mount every time. +Property PristineDir As %String; + Method OnBeforeAllTests() As %Status { set sourceDir = ..GetModuleDir("sync-test") + set ..PristineDir = ##class(%File).NormalizeDirectory(##class(%File).ManagerDirectory() _ "sync-test-pristine-" _ $job) + $$$ThrowOnError(##class(%IPM.Utils.File).CreateDirectoryChain(..PristineDir)) + if '##class(%Library.File).CopyDir(sourceDir, ..PristineDir, 1) { + quit $$$ERROR($$$GeneralError, "Failed to copy sync-test fixture to pristine directory") + } + set ..TempDir = ##class(%File).NormalizeDirectory(##class(%File).ManagerDirectory() _ "sync-test-" _ $job) $$$ThrowOnError(##class(%IPM.Utils.File).CreateDirectoryChain(..TempDir)) - if '##class(%Library.File).CopyDir(sourceDir, ..TempDir, 1) { + if '##class(%Library.File).CopyDir(..PristineDir, ..TempDir, 1) { quit $$$ERROR($$$GeneralError, "Failed to copy sync-test fixture to temp directory") } @@ -27,7 +38,7 @@ Method OnAfterOneTest(testName As %String) As %Status } catch {} do ##class(%Library.File).RemoveDirectoryTree(..TempDir) $$$ThrowOnError(##class(%IPM.Utils.File).CreateDirectoryChain(..TempDir)) - if '##class(%Library.File).CopyDir(..GetModuleDir("sync-test"), ..TempDir, 1) { + if '##class(%Library.File).CopyDir(..PristineDir, ..TempDir, 1) { quit $$$ERROR($$$GeneralError, "Failed to restore sync-test fixture after test: " _ testName) } quit ##class(%IPM.Main).Shell("load " _ ..TempDir _ " -dev") @@ -36,10 +47,13 @@ Method OnAfterOneTest(testName As %String) As %Status Method OnAfterAllTests() As %Status { // OnAfterOneTest already uninstalled and reloaded after the last test. - // This just removes the temp directory. + // This just removes the temp directories. if ..TempDir '= "" { do ##class(%Library.File).RemoveDirectoryTree(..TempDir) } + if ..PristineDir '= "" { + do ##class(%Library.File).RemoveDirectoryTree(..PristineDir) + } quit $$$OK } From 68ace5b32176316cbfded31fef3c710d7e938708 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Wed, 15 Jul 2026 15:27:41 -0400 Subject: [PATCH 07/39] Fix regression --- src/cls/IPM/Lifecycle/Base.cls | 14 ++++++++++++++ src/cls/IPM/ResourceProcessor/Abstract.cls | 12 ++++++++++++ src/cls/IPM/ResourceProcessor/FileCopy.cls | 7 +++---- src/cls/IPM/Storage/Module.cls | 18 ++++++++++++++++++ 4 files changed, 47 insertions(+), 4 deletions(-) diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls index ea7072b97..2814bb5c1 100644 --- a/src/cls/IPM/Lifecycle/Base.cls +++ b/src/cls/IPM/Lifecycle/Base.cls @@ -768,6 +768,20 @@ Method GetTrackedPaths(Output trackedPaths) set trackedPaths(##class(%IPM.Storage.FileHash).NormalizePath(relPath)) = "" } + + // Sync-only enumeration for resources whose files aren't captured by ResolveChildren + // (e.g. FileCopy's directory-scanned source). Does not affect packaging/export. + kill syncOnlyPaths + set sc = resource.Processor.OnSyncResolveFiles(.syncOnlyPaths) + if $$$ISERR(sc) { + continue + } + set relPath = "" + for { + set relPath = $order(syncOnlyPaths(relPath)) + quit:relPath="" + set trackedPaths(##class(%IPM.Storage.FileHash).NormalizePath(relPath)) = "" + } } } diff --git a/src/cls/IPM/ResourceProcessor/Abstract.cls b/src/cls/IPM/ResourceProcessor/Abstract.cls index cf9ebc2f8..9f2bdb420 100644 --- a/src/cls/IPM/ResourceProcessor/Abstract.cls +++ b/src/cls/IPM/ResourceProcessor/Abstract.cls @@ -218,6 +218,18 @@ Method SupportsSync() As %Boolean quit 0 } +/// Called by sync's tracked-path scan (only) to discover files owned by this resource that +/// aren't captured by OnResolveChildren/OnItemRelativePath — e.g. a directory-scanned resource +/// whose file set isn't declared as individual module.xml resources. Output relPaths(relPath)="" +/// relative to the module root. Base returns nothing. Do NOT populate ResolveChildren's shared +/// pResourceArray for this purpose — that array is also consumed by packaging/export +/// (GetResolvedReferences/ExportSingleModule), which expects entries keyed by resource name with +/// owning-module context, not by raw filesystem path. +Method OnSyncResolveFiles(Output relPaths) As %Status +{ + quit $$$OK +} + /// Returns the path relative to the module root for item pItemName within this resource. Method OnItemRelativePath(pItemName As %String) As %String { diff --git a/src/cls/IPM/ResourceProcessor/FileCopy.cls b/src/cls/IPM/ResourceProcessor/FileCopy.cls index 2cc0542f5..546dd89cf 100644 --- a/src/cls/IPM/ResourceProcessor/FileCopy.cls +++ b/src/cls/IPM/ResourceProcessor/FileCopy.cls @@ -185,8 +185,8 @@ Method DoCopy( } /// Enumerate source files so sync can detect changes to FileCopy resources. -/// Populates pResourceArray with RelativePath for each file under the source directory. -Method OnResolveChildren(ByRef pResourceArray, pCheckModuleOwnership As %Boolean) As %Status +/// Populates relPaths(relPath)="" for each file under the source directory, relative to module root. +Method OnSyncResolveFiles(Output relPaths) As %Status { set sc = $$$OK try { @@ -210,8 +210,7 @@ Method OnResolveChildren(ByRef pResourceArray, pCheckModuleOwnership As %Boolean // Compute path relative to module root set relPath = $extract(fullPath, $length(moduleRoot) + 1, *) if relPath '= "" { - set pResourceArray(fullPath) = "" - set pResourceArray(fullPath, "RelativePath") = relPath + set relPaths(relPath) = "" } } } catch e { diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls index 289a9b5b1..23e973084 100644 --- a/src/cls/IPM/Storage/Module.cls +++ b/src/cls/IPM/Storage/Module.cls @@ -922,6 +922,24 @@ ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, ByRef reverseIn set reverseIndex(normalizedRelPath, "Processor") = resource.Processor set reverseIndex(normalizedRelPath, "Resource") = resource } + + // Sync-only enumeration for resources whose files aren't captured by ResolveChildren + // (e.g. FileCopy's directory-scanned source). Does not affect packaging/export. + kill syncOnlyPaths + set childSC = resource.Processor.OnSyncResolveFiles(.syncOnlyPaths) + if $$$ISERR(childSC) { + continue + } + set relPath = "" + for { + set relPath = $order(syncOnlyPaths(relPath)) + quit:relPath="" + + set normalizedRelPath = ##class(%IPM.Storage.FileHash).NormalizePath(relPath) + set reverseIndex(normalizedRelPath) = resource.Name + set reverseIndex(normalizedRelPath, "Processor") = resource.Processor + set reverseIndex(normalizedRelPath, "Resource") = resource + } } } From 58978caf44a51ecc24b49ee9d0eaa7935a8d6fbc Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Wed, 15 Jul 2026 16:25:59 -0400 Subject: [PATCH 08/39] Improve testing and fix small bug --- src/cls/IPM/Storage/Module.cls | 21 ++++--- .../Test/PM/Integration/Sync.cls | 61 +++++++++++++++++++ 2 files changed, 74 insertions(+), 8 deletions(-) diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls index 23e973084..a381013b3 100644 --- a/src/cls/IPM/Storage/Module.cls +++ b/src/cls/IPM/Storage/Module.cls @@ -729,6 +729,11 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status set processDeletes = $get(params("ProcessDeletes"), 0) set runTests = $get(params("RunTests"), 0) + // params may be reused across modules by Main.Sync's sync-all-dev-mode-modules loop. + // Clear this module's own scratch subtree so a prior module's recorded test-case + // changes can't leak into this module's SyncRunTests dispatch. + kill params("Sync") + set module = ..NameOpen(moduleName, , .sc) if '$isobject(module) { $$$ThrowStatus($$$ERROR($$$GeneralError, $$$FormatText("Module '%1' not found.", moduleName))) @@ -771,7 +776,7 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status write !, "[", moduleName, "] Nothing to sync." if moduleXmlChanged { do ..SyncCommitModuleXml(module, moduleXmlPath, moduleXmlRelPath) - do ..SyncPrintModuleXmlWarning() + do ..SyncPrintModuleXmlWarning(moduleName) } quit } @@ -837,9 +842,9 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status write "." if moduleXmlChanged { - do ..SyncPrintModuleXmlWarning() + do ..SyncPrintModuleXmlWarning(moduleName) } - do ..SyncPrintUnsupportedWarnings(.unsupportedWarnings) + do ..SyncPrintUnsupportedWarnings(moduleName, .unsupportedWarnings) // Step 9: Run changed test-phase tests if -test flag is set (after sync is reported) if runTests { @@ -1149,15 +1154,15 @@ ClassMethod SyncCommitModuleXml( do ##class(%IPM.Storage.FileHash).CommitChanges(module, .moduleXmlMod, .emptyDel, 0) } -ClassMethod SyncPrintModuleXmlWarning() +ClassMethod SyncPrintModuleXmlWarning(moduleName As %String) { write ! write !, "Warning: module.xml changed and was reloaded." - write !, " Resources may have been added/removed. A full 'reload' may be needed" - write !, " to fully apply manifest-level changes (mappings, dependencies, defaults)." + write !, " Resources may have been added/removed. Run `reload ", moduleName, "` to fully apply" + write !, " manifest-level changes (mappings, dependencies, defaults)." } -ClassMethod SyncPrintUnsupportedWarnings(ByRef unsupportedWarnings) +ClassMethod SyncPrintUnsupportedWarnings(moduleName As %String, ByRef unsupportedWarnings) { set warnRes = "" for { @@ -1171,7 +1176,7 @@ ClassMethod SyncPrintUnsupportedWarnings(ByRef unsupportedWarnings) quit:warnPath="" write !, " ", warnPath } - write !, " Run a full 'reload' to apply these changes." + write !, " Run `reload ", moduleName, "` to apply these changes." } } diff --git a/tests/integration_tests/Test/PM/Integration/Sync.cls b/tests/integration_tests/Test/PM/Integration/Sync.cls index d066fa68b..4a234565a 100644 --- a/tests/integration_tests/Test/PM/Integration/Sync.cls +++ b/tests/integration_tests/Test/PM/Integration/Sync.cls @@ -72,6 +72,49 @@ Method TestNoChangeIsNoOp() do $$$AssertTrue(..FindInOutput(.output, "Nothing to sync"), "Reports nothing to sync") } +/// Simulates a module installed before this feature existed (zero FileHash baseline rows). +/// The first sync call must self-heal by stamping a baseline rather than failing or reporting +/// every tracked file as changed; only a second sync call (after a real edit) detects changes. +Method TestMigrationFromNoBaseline() +{ + set deleteRS = ##class(%SQL.Statement).%ExecDirect(, "DELETE FROM %IPM_Storage.FileHash WHERE ModuleName = ?", "sync-test") + do $$$AssertTrue(deleteRS.%SQLCODE >= 0, "Deleted existing baseline rows to simulate a pre-sync-feature module") + do $$$AssertNotTrue(##class(%IPM.Storage.FileHash).HasBaseline("sync-test"), "No baseline rows remain") + + set filePath = ..TempDir _ "src/cls/SyncTest/SuperClass.cls" + do ..ReplaceInFile(filePath, """original""", """modified""") + + kill params + set params("Verbose") = 1 + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + + do $$$AssertStatusOK(sc, "First sync after baseline loss succeeds") + do $$$AssertTrue(..FindInOutput(.output, "Baseline established"), "First sync self-heals by establishing a baseline") + do $$$AssertTrue(##class(%IPM.Storage.FileHash).HasBaseline("sync-test"), "Baseline rows now exist") + + // The edit made before the self-heal is captured in the new baseline (not retroactively + // detected), so a second sync with no further changes reports nothing to do. + kill params2, output2 + set params2("Verbose") = 1 + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie2) + set sc2 = ##class(%IPM.Storage.Module).Sync("sync-test", .params2) + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie2, .output2) + do $$$AssertStatusOK(sc2, "Second sync succeeds") + do $$$AssertTrue(..FindInOutput(.output2, "Nothing to sync"), "Second sync finds no changes (baseline already reflects the modified file)") + + // A genuinely new edit after the baseline is established is detected normally. + do ..ReplaceInFile(filePath, """modified""", """modified again""") + kill params3, output3 + set params3("Verbose") = 1 + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie3) + set sc3 = ##class(%IPM.Storage.Module).Sync("sync-test", .params3) + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie3, .output3) + do $$$AssertStatusOK(sc3, "Third sync succeeds") + do $$$AssertTrue(..FindInOutput(.output3, "Sync complete"), "Third sync detects the post-baseline edit normally") +} + /// A modified class file is detected and recompiled by sync. Method TestModifiedClassRecompiles() { @@ -145,6 +188,24 @@ Method TestDeleteSkippedByDefault() do $$$AssertNotTrue($$$comClassDefined("SyncTest.Deletable"), "Deletable class removed after -delete sync") } +/// Deleting a superclass (with -delete) leaves its subclass referencing a now-missing class. +/// The post-delete recompile pass (SyncApplyDeletes + SyncCompile) must still run and surface +/// this as a compile error, rather than silently leaving the subclass in a stale-but-compiled state. +Method TestDeleteRecompilesDependents() +{ + set filePath = ..TempDir _ "src/cls/SyncTest/SuperClass.cls" + do ##class(%Library.File).Delete(filePath) + + do $$$AssertTrue($$$comClassDefined("SyncTest.SubClass"), "SubClass still compiled before delete sync") + + kill params + set params("ProcessDeletes") = 1 + set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + + do $$$AssertStatusNotOK(sc, "Sync with -delete fails: SubClass now references a deleted superclass") + do $$$AssertNotTrue($$$comClassDefined("SyncTest.SuperClass"), "SuperClass was removed from the server") +} + /// A changed module.xml is detected and its manifest reloaded, with a warning that a full /// reload may still be needed to apply manifest-level changes (mappings, dependencies, etc.). Method TestModuleXmlChangedWarning() From 05bf4f9f2ce28ccc506a74a59d37b12091dacf03 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Thu, 16 Jul 2026 10:50:57 -0400 Subject: [PATCH 09/39] Correctly batch sync tests --- src/cls/IPM/ResourceProcessor/Test.cls | 177 +++++++++++++++++- src/cls/IPM/Storage/Module.cls | 60 +++--- .../Test/PM/Integration/Sync.cls | 49 ++++- 3 files changed, 260 insertions(+), 26 deletions(-) diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls index f9f8198ad..ed96311d3 100644 --- a/src/cls/IPM/ResourceProcessor/Test.cls +++ b/src/cls/IPM/ResourceProcessor/Test.cls @@ -364,6 +364,72 @@ Method SupportsSync() As %Boolean quit 1 } +/// Enumerate test files on disk so sync can detect brand-new test classes. +/// ResolveChildren (via GetChildren's StudioOpenDialog query) only sees already-compiled +/// classes, so a test file that was never compiled — e.g. one just added to the test +/// directory — is otherwise invisible to GetTrackedPaths/ComputeChanges until something +/// else compiles it first. This live directory scan closes that gap, mirroring FileCopy's +/// OnSyncResolveFiles for the same reason. +Method OnSyncResolveFiles(Output relPaths) As %Status +{ + set sc = $$$OK + try { + set unitTestDir = ##class(%File).NormalizeDirectory(..ResourceReference.Module.Root _ ..ResourceReference.Name) + set moduleRoot = ##class(%File).NormalizeDirectory(..ResourceReference.Module.Root) + if '##class(%File).DirectoryExists(unitTestDir) { + quit + } + $$$ThrowOnError(..WalkClsFilesRecursive(unitTestDir, moduleRoot, .relPaths)) + } catch e { + set sc = e.AsStatus() + } + quit sc +} + +/// Recursively collects .cls files under pDir into relPaths(relPath), relative to pModuleRoot. +/// %Library.File_FileSet's own recursive flag only returns immediate children per call +/// (confirmed experimentally — it does not descend through multiple directory levels in one +/// call), so each subdirectory must be visited with its own call, same as %UnitTest.Manager's +/// own GetSubDirectories does internally. +ClassMethod WalkClsFilesRecursive( + pDir As %String, + pModuleRoot As %String, + ByRef relPaths) As %Status +{ + set sc = $$$OK + try { + set rs = ##class(%SQL.Statement).%ExecDirect(, + "SELECT Name, Type FROM %Library.File_FileSet(?, ?, ?, ?)", + pDir, "*.cls", "", 0) + if rs.%SQLCODE < 0 { + $$$ThrowStatus($$$ERROR($$$GeneralError, "FileSet query error: " _ rs.%Message)) + } + while rs.%Next() { + continue:rs.%Get("Type")="D" + set fullPath = rs.%Get("Name") + set relPath = $extract(fullPath, $length(pModuleRoot) + 1, *) + if relPath '= "" { + set relPaths(relPath) = "" + } + } + + set dirRs = ##class(%SQL.Statement).%ExecDirect(, + "SELECT Name, Type FROM %Library.File_FileSet(?, ?, ?, ?)", + pDir, "*", "", 0) + if dirRs.%SQLCODE < 0 { + $$$ThrowStatus($$$ERROR($$$GeneralError, "FileSet query error: " _ dirRs.%Message)) + } + while dirRs.%Next() { + continue:dirRs.%Get("Type")'="D" + set subDir = ##class(%File).NormalizeDirectory(dirRs.%Get("Name")) + $$$ThrowOnError(..WalkClsFilesRecursive(subDir, pModuleRoot, .relPaths)) + } + } catch e { + set sc = e.AsStatus() + } + quit sc +} + Method OnSync(ByRef modifiedPaths, ByRef deletedPaths, ByRef params, Output handled As %Boolean = 0) As %Status { set sc = $$$OK @@ -404,12 +470,121 @@ Method OnSync(ByRef modifiedPaths, ByRef deletedPaths, ByRef params, Output hand if $zconvert($piece(fileName, ".", *), "U") = "CLS" && $$$comClassDefined(className) && $classmethod(className, "%Extends", "%UnitTest.TestCase") { - set params("Sync", "ChangedTestCases", className) = ..ResourceReference.Name + // dirPart (the class's real on-disk subdirectory, "/"-separated) travels with the + // resource name so SyncRunTests can build a batched, correctly-scoped RunTest + // testspec later without re-deriving the directory from the class name. + set params("Sync", "ChangedTestCases", className) = $listbuild(..ResourceReference.Name, dirPart) + } + } + } catch e { + set sc = e.AsStatus() + } + quit sc +} + +/// Called once per resource by SyncRunTests: runs every changed test-phase case belonging +/// to this resource in a single RunTest invocation instead of one call per class. +/// classInfo(className) = dirPart (path relative to this resource's test directory, "/"-separated). +/// Assumes OnSync already loaded and compiled these classes earlier in the same sync() call, +/// so no reload/recompile happens here — only testspec construction, RunTest, and reporting. +Method OnSyncRunTests(ByRef classInfo, ByRef params) As %Status +{ + set sc = $$$OK + try { + set verbose = $get(params("Verbose"), 0) + set unitTestDir = ##class(%File).NormalizeDirectory(..ResourceReference.Module.Root _ ..ResourceReference.Name) + + // Initialize test result accumulator for this phase if not already initialized (i.e., we're at top level, not nested) + // Mirrors OnPhase's accumulator so multiple resources synced in one -test run share one result set. + if '$data(^||%UnitTest.Manager.AllResultsCount) { + kill ^||%UnitTest.Manager.AllResults + set ^||%UnitTest.Manager.AllResultsCount = 0 + } + set phaseStartIndex = $get(^||%UnitTest.Manager.AllResultsCount, 0) + + // Build one testspec covering every changed class in this resource. Classes sharing + // a subdirectory must join with ";" within ONE "dir:Class1;Class2" entry — repeating + // the same dirPart across separate comma-joined entries is invalid grammar (RunTest + // treats each comma-separated entry as its own testsuite scope; a second entry naming + // the same dir errors instead of adding to the first). Distinct subdirectories become + // separate comma-joined entries so unrelated sibling test classes elsewhere in the + // resource are not swept in. + kill byDir + set className = "" + for { + set className = $order(classInfo(className), 1, dirPart) + quit:className="" + set byDir(dirPart, className) = "" + } + set testSpec = "" + set dirPart = "" + for { + set dirPart = $order(byDir(dirPart)) + quit:dirPart="" + set classList = "" + set className = "" + for { + set className = $order(byDir(dirPart, className)) + quit:className="" + set classList = classList _ $select(classList="":"", 1: ";") _ className + } + set testSpec = testSpec _ $select(testSpec="":"", 1: ",") _ dirPart _ ":" _ classList + } + + set explicitQuiet = ($data(params("Verbose")) && (params("Verbose") = 0)) + if explicitQuiet { + set suppressor = ##class(%IPM.Utils.OutputSuppressor).%New() + } + + set flags = $select(verbose:"/display=all",1:"/display=none")_"/nodelete/norecursive" + + if $data(^UnitTestRoot,oldUnitTestRoot) // Stash ^UnitTestRoot + set ^UnitTestRoot = unitTestDir + set managerClass = $get(params("UnitTest","ManagerClass"),..ManagerClass) + if (managerClass = "") { + set managerClass = "%IPM.Test.Manager" + } + merge userParams = params("UnitTest","UserParam") + set sc = $classmethod(managerClass,"RunTest",testSpec,flags,.userParams) + zkill ^UnitTestRoot + $$$ThrowOnError(sc) + + set testIndex = $get(^||%UnitTest.Manager.AllResults($get(^||%UnitTest.Manager.AllResultsCount))) + if testIndex = "" { + set testIndex = $order(^UnitTest.Result(""),-1) + } + set suppressor = "" + + set outputFormat = $get(params("outputformat")) + if outputFormat = "" { + set outputFormat = ##class(%IPM.Repo.UniversalSettings).GetTestReportFormat() + } + + write !!,"Test Results:" + if outputFormat '= "" { + set outputClass = "%IPM.Test."_$zconvert(outputFormat,"w")_"Output" + if '$$$defClassDefined(outputClass) { + $$$ThrowOnError($$$ERROR($$$GeneralError,"Unknown output format: "_outputFormat)) + } + set sc = $classmethod(outputClass,"OutputToDevice",testIndex,verbose,1) + $$$ThrowOnError(sc) + } else { + set sc = ##class(%IPM.Test.Abstract).OutputToDevice(testIndex,verbose,0) + $$$ThrowOnError(sc) + } + write ! + if $get(params("UnitTest","FailuresAreFatal"),1) { + if outputFormat = "" { + do ##class(%IPM.Test.Manager).OutputFailures(phaseStartIndex) } + set sc = ##class(%IPM.Test.Manager).GetAllTestsStatus(,phaseStartIndex) + $$$ThrowOnError(sc) } + write ! } catch e { set sc = e.AsStatus() } + if $data(oldUnitTestRoot,^UnitTestRoot) // Restore ^UnitTestRoot quit sc } diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls index a381013b3..4911c14e3 100644 --- a/src/cls/IPM/Storage/Module.cls +++ b/src/cls/IPM/Storage/Module.cls @@ -1099,6 +1099,9 @@ ClassMethod SyncApplyDeletes( } /// Run test-phase tests for changed test case classes recorded in params("Sync","ChangedTestCases"). +/// Groups changed classes by owning resource first, then dispatches one batched RunTest call +/// per resource (via OnSyncRunTests) instead of one call per class — avoids N separate +/// reload/compile/run cycles when several test classes in the same resource changed together. ClassMethod SyncRunTests( orderedResourceList As %ListOfObjects, verbose As %Boolean, @@ -1106,36 +1109,47 @@ ClassMethod SyncRunTests( { set sc = $$$OK try { + // Step 1: group changed classes by owning resource: byResource(resourceName, className) = dirPart + kill byResource set className = "" for { - set className = $order(params("Sync", "ChangedTestCases", className), 1, owningResource) + set className = $order(params("Sync", "ChangedTestCases", className), 1, resourceInfo) quit:className="" + set owningResource = $listget(resourceInfo, 1) + set dirPart = $listget(resourceInfo, 2) + set byResource(owningResource, className) = dirPart + } - set testKey = "" - for { - set testResource = orderedResourceList.GetNext(.testKey) - quit:testKey="" + // Step 2: one OnSyncRunTests call per resource + set testKey = "" + for { + set testResource = orderedResourceList.GetNext(.testKey) + quit:testKey="" - if '$isobject(testResource.Processor) { - continue - } - if 'testResource.Processor.%IsA("%IPM.ResourceProcessor.Test") { - continue - } - if testResource.Name '= owningResource { - continue - } - if '$listfind(testResource.Processor.Phase, "test") { - write:verbose !, "Skipping verify-scoped test: ", className, " (use 'verify' to run)" - continue + if '$isobject(testResource.Processor) { + continue + } + if 'testResource.Processor.%IsA("%IPM.ResourceProcessor.Test") { + continue + } + if '$data(byResource(testResource.Name)) { + continue + } + if '$listfind(testResource.Processor.Phase, "test") { + set skipClassName = "" + for { + set skipClassName = $order(byResource(testResource.Name, skipClassName)) + quit:skipClassName="" + write:verbose !, "Skipping verify-scoped test: ", skipClassName, " (use 'verify' to run)" } - kill testParams - merge testParams = params - set testParams("UnitTest", "Case") = className - set testParams("DeveloperMode") = 1 - set handled = 0 - $$$ThrowOnError(testResource.Processor.OnPhase("Test", .testParams, .handled)) + continue } + kill classInfo + merge classInfo = byResource(testResource.Name) + kill testParams + merge testParams = params + set testParams("DeveloperMode") = 1 + $$$ThrowOnError(testResource.Processor.OnSyncRunTests(.classInfo, .testParams)) } } catch e { set sc = e.AsStatus() diff --git a/tests/integration_tests/Test/PM/Integration/Sync.cls b/tests/integration_tests/Test/PM/Integration/Sync.cls index 4a234565a..b42f3b435 100644 --- a/tests/integration_tests/Test/PM/Integration/Sync.cls +++ b/tests/integration_tests/Test/PM/Integration/Sync.cls @@ -268,7 +268,52 @@ Method TestSyncTestFlag() do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) do $$$AssertStatusOK(sc, "Sync with RunTests runs changed tests") do $$$AssertTrue(..FindInOutput(.output, "Test Results"), "Test results shown with RunTests flag") - do $$$AssertTrue(..FindInOutput(.output, "SyncTest.Tests.Trivial"), "Changed test class appears in output") + do $$$AssertTrue(..FindInOutput(.output, "SyncTest.Tests.Trivial begins"), "Changed test class actually ran, not just compiled") +} + +/// Editing an existing test class and adding a brand-new test class in the same resource, +/// in the same sync call, batches both changed classes into a single RunTest invocation +/// (one "Test Results:" block) rather than one call per class — and both tests still run. +Method TestSyncTestFlagBatchesMultipleChangedClasses() +{ + set existingFilePath = ..TempDir _ "tests/unit/SyncTest/Tests/Trivial.cls" + do ..ReplaceInFile(existingFilePath, "This test always passes.", "This test always passes (modified).") + + set newFilePath = ..TempDir _ "tests/unit/SyncTest/Tests/BatchedNew.cls" + set newClassContent = "Class SyncTest.Tests.BatchedNew Extends %UnitTest.TestCase" _ $char(10) _ "{" _ $char(10) + set newClassContent = newClassContent _ "Method TestBatchedNewPasses()" _ $char(10) _ "{" _ $char(10) + set newClassContent = newClassContent _ " do $$$AssertTrue(1, ""BatchedNew always passes."")" _ $char(10) _ "}" _ $char(10) _ "}" _ $char(10) + set stream = ##class(%Stream.FileCharacter).%New() + $$$ThrowOnError(stream.LinkToFile(newFilePath)) + $$$ThrowOnError(stream.Write(newClassContent)) + $$$ThrowOnError(stream.%Save()) + set stream = "" + + kill params + set params("RunTests") = 1 + set params("Verbose") = 1 + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + + do $$$AssertStatusOK(sc, "Sync with two changed classes in one resource succeeds") + // "ClassName begins ..." / "ClassName passed" only appear when the test runner actually + // executes that class — unlike a bare class-name substring, which also matches the + // "Loading file .../Trivial.cls" and "Compiling class SyncTest.Tests.Trivial" lines + // that show up regardless of whether the test itself ran. + do $$$AssertTrue(..FindInOutput(.output, "SyncTest.Tests.Trivial begins"), "Modified existing class actually ran, not just compiled") + do $$$AssertTrue(..FindInOutput(.output, "SyncTest.Tests.BatchedNew begins"), "New class in the same resource actually ran, not just compiled") + + set testResultsCount = 0 + set lineKey = "" + for { + set lineKey = $order(output(lineKey), 1, line) + quit:lineKey="" + if line [ "Test Results:" { + set testResultsCount = testResultsCount + 1 + } + } + do $$$AssertEquals(testResultsCount, 1, "Both changed classes ran in a single batched Test Results block, not one per class") } /// A change to a file owned by a processor that doesn't support sync is never applied, so @@ -339,7 +384,7 @@ Method TestSyncTestFlagOnlyRunsOwningResource() do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) do $$$AssertStatusOK(sc, "Sync with RunTests succeeds") - do $$$AssertTrue(..FindInOutput(.output, "SyncTest.Tests.Trivial"), "Changed test in tests/unit ran") + do $$$AssertTrue(..FindInOutput(.output, "SyncTest.Tests.Trivial begins"), "Changed test in tests/unit actually ran, not just compiled") do $$$AssertNotTrue(..FindInOutput(.output, "SyncTest2.Tests.Trivial2"), "Unrelated test in tests/unit2 was not run") } From ebb50d6810147033f31c3843856fed6adbb05305 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Thu, 16 Jul 2026 11:09:10 -0400 Subject: [PATCH 10/39] Refactor --- src/cls/IPM/ResourceProcessor/Test.cls | 159 +++++++++++-------------- src/cls/IPM/Storage/Module.cls | 8 +- 2 files changed, 75 insertions(+), 92 deletions(-) diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls index ed96311d3..76d70deaf 100644 --- a/src/cls/IPM/ResourceProcessor/Test.cls +++ b/src/cls/IPM/ResourceProcessor/Test.cls @@ -214,64 +214,80 @@ Method OnPhase( zkill ^UnitTestRoot $$$ThrowOnError(tSC) - set testIndex = $get(^||%UnitTest.Manager.AllResults($get(^||%UnitTest.Manager.AllResultsCount))) - if testIndex = "" { - set testIndex = $order(^UnitTest.Result(""),-1) - } - if $data(pParams("outputfile"), outputFile) { - set fileExtension = $zconvert($piece(outputFile,".",*),"L") - set outputClass = $case(fileExtension, - "json":"%IPM.Test.JsonOutput", - "yaml":"%IPM.Test.YamlOutput", - "toon":"%IPM.Test.ToonOutput", - "xml":"%IPM.Test.JUnitOutput", - :"") - if outputClass = "" { - $$$ThrowOnError($$$ERROR($$$GeneralError,"Unsupported output-file extension '."_fileExtension_"'. Use .json, .yaml, .toon, or .xml.")) - } - set outputDir = ##class(%File).GetDirectory(outputFile) - if outputDir '= "" { - $$$ThrowOnError(##class(%IPM.Utils.File).CreateDirectoryChain(outputDir)) - } - set tSC = $classmethod(outputClass,"ToFile",outputFile) - $$$ThrowOnError(tSC) - } set suppressor = "" + $$$ThrowOnError(..ReportTestResults(phaseStartIndex, tVerbose, .pParams)) + } + } catch e { + set tSC = e.AsStatus() + } + if $data(tOldUnitTestRoot,^UnitTestRoot) // Restore ^UnitTestRoot + quit tSC +} - set outputFormat = $get(pParams("outputformat")) - if outputFormat = "" { - set outputFormat = ##class(%IPM.Repo.UniversalSettings).GetTestReportFormat() +/// Shared result-reporting tail for OnPhase and OnSyncRunTests: resolves the test-run's log +/// index, optionally exports full results to a file, then writes the "Test Results:" summary +/// (format-specific output class, or the legacy default) and throws if FailuresAreFatal and +/// any test in this phase failed. Callers must clear their own OutputSuppressor (if any) +/// before calling — the file-export step is unaffected by device redirection, but the summary +/// write below is not. +ClassMethod ReportTestResults( + phaseStartIndex As %Integer, + verbose As %Boolean, + ByRef params) As %Status [ Private ] +{ + set sc = $$$OK + try { + set testIndex = $get(^||%UnitTest.Manager.AllResults($get(^||%UnitTest.Manager.AllResultsCount))) + if testIndex = "" { + set testIndex = $order(^UnitTest.Result(""),-1) + } + if $data(params("outputfile"), outputFile) { + set fileExtension = $zconvert($piece(outputFile,".",*),"L") + set outputClass = $case(fileExtension, + "json":"%IPM.Test.JsonOutput", + "yaml":"%IPM.Test.YamlOutput", + "toon":"%IPM.Test.ToonOutput", + "xml":"%IPM.Test.JUnitOutput", + :"") + if outputClass = "" { + $$$ThrowOnError($$$ERROR($$$GeneralError,"Unsupported output-file extension '."_fileExtension_"'. Use .json, .yaml, .toon, or .xml.")) + } + set outputDir = ##class(%File).GetDirectory(outputFile) + if outputDir '= "" { + $$$ThrowOnError(##class(%IPM.Utils.File).CreateDirectoryChain(outputDir)) } + $$$ThrowOnError($classmethod(outputClass,"ToFile",outputFile)) + } - write !!,"Test Results:" - if outputFormat '= "" { - set outputClass = "%IPM.Test."_$zconvert(outputFormat,"w")_"Output" - if '$$$defClassDefined(outputClass) { - $$$ThrowOnError($$$ERROR($$$GeneralError,"Unknown output format: "_outputFormat)) - } - set tSC = $classmethod(outputClass,"OutputToDevice",testIndex,tVerbose,1) - $$$ThrowOnError(tSC) - } else { - set tSC = ##class(%IPM.Test.Abstract).OutputToDevice(testIndex,tVerbose,0) - $$$ThrowOnError(tSC) + set outputFormat = $get(params("outputformat")) + if outputFormat = "" { + set outputFormat = ##class(%IPM.Repo.UniversalSettings).GetTestReportFormat() + } + + write !!,"Test Results:" + if outputFormat '= "" { + set outputClass = "%IPM.Test."_$zconvert(outputFormat,"w")_"Output" + if '$$$defClassDefined(outputClass) { + $$$ThrowOnError($$$ERROR($$$GeneralError,"Unknown output format: "_outputFormat)) } - write ! - // Detect and report unit test failures as an error from this phase. - // OutputFailures shows legacy red FAILED lines only when no format is active. - if $get(pParams("UnitTest","FailuresAreFatal"),1) { - if outputFormat = "" { - do ##class(%IPM.Test.Manager).OutputFailures(phaseStartIndex) - } - set tSC = ##class(%IPM.Test.Manager).GetAllTestsStatus(,phaseStartIndex) - $$$ThrowOnError(tSC) + $$$ThrowOnError($classmethod(outputClass,"OutputToDevice",testIndex,verbose,1)) + } else { + $$$ThrowOnError(##class(%IPM.Test.Abstract).OutputToDevice(testIndex,verbose,0)) + } + write ! + // Detect and report unit test failures as an error from this phase. + // OutputFailures shows legacy red FAILED lines only when no format is active. + if $get(params("UnitTest","FailuresAreFatal"),1) { + if outputFormat = "" { + do ##class(%IPM.Test.Manager).OutputFailures(phaseStartIndex) } - write ! + $$$ThrowOnError(##class(%IPM.Test.Manager).GetAllTestsStatus(,phaseStartIndex)) } + write ! } catch e { - set tSC = e.AsStatus() + set sc = e.AsStatus() } - if $data(tOldUnitTestRoot,^UnitTestRoot) // Restore ^UnitTestRoot - quit tSC + quit sc } Method OnResolveChildren(ByRef pResourceArray) As %Status @@ -470,10 +486,7 @@ Method OnSync(ByRef modifiedPaths, ByRef deletedPaths, ByRef params, Output hand if $zconvert($piece(fileName, ".", *), "U") = "CLS" && $$$comClassDefined(className) && $classmethod(className, "%Extends", "%UnitTest.TestCase") { - // dirPart (the class's real on-disk subdirectory, "/"-separated) travels with the - // resource name so SyncRunTests can build a batched, correctly-scoped RunTest - // testspec later without re-deriving the directory from the class name. - set params("Sync", "ChangedTestCases", className) = $listbuild(..ResourceReference.Name, dirPart) + set params("Sync", "ChangedTestCases", className) = ..ResourceReference.Name } } } catch e { @@ -484,7 +497,7 @@ Method OnSync(ByRef modifiedPaths, ByRef deletedPaths, ByRef params, Output hand /// Called once per resource by SyncRunTests: runs every changed test-phase case belonging /// to this resource in a single RunTest invocation instead of one call per class. -/// classInfo(className) = dirPart (path relative to this resource's test directory, "/"-separated). +/// classInfo(className) = "" for each changed TestCase class in this resource. /// Assumes OnSync already loaded and compiled these classes earlier in the same sync() call, /// so no reload/recompile happens here — only testspec construction, RunTest, and reporting. Method OnSyncRunTests(ByRef classInfo, ByRef params) As %Status @@ -508,12 +521,14 @@ Method OnSyncRunTests(ByRef classInfo, ByRef params) As %Status // treats each comma-separated entry as its own testsuite scope; a second entry naming // the same dir errors instead of adding to the first). Distinct subdirectories become // separate comma-joined entries so unrelated sibling test classes elsewhere in the - // resource are not swept in. + // resource are not swept in. dirPart is derived from className (its inverse) rather + // than passed in, since a TestCase class name always mirrors its on-disk path. kill byDir set className = "" for { - set className = $order(classInfo(className), 1, dirPart) + set className = $order(classInfo(className)) quit:className="" + set dirPart = $translate($piece(className, ".", 1, *-1), ".", "/") set byDir(dirPart, className) = "" } set testSpec = "" @@ -549,38 +564,8 @@ Method OnSyncRunTests(ByRef classInfo, ByRef params) As %Status zkill ^UnitTestRoot $$$ThrowOnError(sc) - set testIndex = $get(^||%UnitTest.Manager.AllResults($get(^||%UnitTest.Manager.AllResultsCount))) - if testIndex = "" { - set testIndex = $order(^UnitTest.Result(""),-1) - } set suppressor = "" - - set outputFormat = $get(params("outputformat")) - if outputFormat = "" { - set outputFormat = ##class(%IPM.Repo.UniversalSettings).GetTestReportFormat() - } - - write !!,"Test Results:" - if outputFormat '= "" { - set outputClass = "%IPM.Test."_$zconvert(outputFormat,"w")_"Output" - if '$$$defClassDefined(outputClass) { - $$$ThrowOnError($$$ERROR($$$GeneralError,"Unknown output format: "_outputFormat)) - } - set sc = $classmethod(outputClass,"OutputToDevice",testIndex,verbose,1) - $$$ThrowOnError(sc) - } else { - set sc = ##class(%IPM.Test.Abstract).OutputToDevice(testIndex,verbose,0) - $$$ThrowOnError(sc) - } - write ! - if $get(params("UnitTest","FailuresAreFatal"),1) { - if outputFormat = "" { - do ##class(%IPM.Test.Manager).OutputFailures(phaseStartIndex) - } - set sc = ##class(%IPM.Test.Manager).GetAllTestsStatus(,phaseStartIndex) - $$$ThrowOnError(sc) - } - write ! + $$$ThrowOnError(..ReportTestResults(phaseStartIndex, verbose, .params)) } catch e { set sc = e.AsStatus() } diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls index 4911c14e3..2c2b6ba94 100644 --- a/src/cls/IPM/Storage/Module.cls +++ b/src/cls/IPM/Storage/Module.cls @@ -1109,15 +1109,13 @@ ClassMethod SyncRunTests( { set sc = $$$OK try { - // Step 1: group changed classes by owning resource: byResource(resourceName, className) = dirPart + // Step 1: group changed classes by owning resource: byResource(resourceName, className) = "" kill byResource set className = "" for { - set className = $order(params("Sync", "ChangedTestCases", className), 1, resourceInfo) + set className = $order(params("Sync", "ChangedTestCases", className), 1, owningResource) quit:className="" - set owningResource = $listget(resourceInfo, 1) - set dirPart = $listget(resourceInfo, 2) - set byResource(owningResource, className) = dirPart + set byResource(owningResource, className) = "" } // Step 2: one OnSyncRunTests call per resource From 99641337aac95158d9788ad402461465e18671cd Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Thu, 16 Jul 2026 12:14:33 -0400 Subject: [PATCH 11/39] Fix minor issues --- src/cls/IPM/Lifecycle/Base.cls | 1 - src/cls/IPM/ResourceProcessor/Test.cls | 3 +- src/cls/IPM/Storage/Module.cls | 16 +++++-- .../Test/PM/Integration/Sync.cls | 48 ++++++++++++------- 4 files changed, 46 insertions(+), 22 deletions(-) diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls index 2814bb5c1..70d8b2035 100644 --- a/src/cls/IPM/Lifecycle/Base.cls +++ b/src/cls/IPM/Lifecycle/Base.cls @@ -1267,7 +1267,6 @@ Method %Compile(ByRef pParams) As %Status // Done here after compile so test classes are in ^oddDEF and appear in trackedPaths. if tDevMode { try { - set trackedPaths = "" do ..GetTrackedPaths(.trackedPaths) do ##class(%IPM.Storage.FileHash).StampModule(..Module, .trackedPaths) } catch stampEx { diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls index 76d70deaf..5b998db1b 100644 --- a/src/cls/IPM/ResourceProcessor/Test.cls +++ b/src/cls/IPM/ResourceProcessor/Test.cls @@ -454,7 +454,8 @@ Method OnSync(ByRef modifiedPaths, ByRef deletedPaths, ByRef params, Output hand set verbose = $get(params("Verbose")) set unitTestDir = ##class(%File).NormalizeDirectory(..ResourceReference.Module.Root _ ..ResourceReference.Name) - // Reload changed test files, then compile so %Extends checks are valid below. + // Reload all test files in the directory, then compile so %Extends checks are valid below. + // New test classes may not yet be in ^oddDEF, so we can't limit to changed files here. // Test resources are not AbstractCompilable, so SyncCompile never touches them — // OnSync owns the full load+compile cycle for this resource type. $$$ThrowOnError(##class(%IPM.Test.Manager).LoadTestDirectory(unitTestDir, verbose, .loadedList, ..Format)) diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls index 2c2b6ba94..e554673fb 100644 --- a/src/cls/IPM/Storage/Module.cls +++ b/src/cls/IPM/Storage/Module.cls @@ -853,7 +853,6 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status } catch e { set sc = e.AsStatus() - do $system.OBJ.DisplayError(sc) } quit sc } @@ -880,6 +879,11 @@ ClassMethod SyncCheckModuleXml( } set newHash = ##class(%File).SHA1Hash(moduleXmlPath, 1) if (existing.Hash '= "") && (newHash = existing.Hash) { + // Content unchanged despite mtime/size change — update stored metadata to avoid + // re-hashing on every subsequent sync until the file is written again. + set existing.FileSize = currentSize + set existing.FileTimestamp = currentTimestamp + do existing.%Save() quit 0 } $$$ThrowOnError($system.OBJ.Load(moduleXmlPath, "-d")) @@ -1093,7 +1097,10 @@ ClassMethod SyncApplyDeletes( set docName = ..RelPathToDocName(relPath) if docName '= "" { set delFlags = $select(verbose:"d", 1:"-d") - do $system.OBJ.Delete(docName, delFlags) + set delSC = $system.OBJ.Delete(docName, delFlags) + if $$$ISERR(delSC) { + write !, "Warning: could not delete ", docName, ": ", $system.Status.GetOneErrorText(delSC) + } } } } @@ -1163,7 +1170,10 @@ ClassMethod SyncCommitModuleXml( { kill moduleXmlMod, emptyDel set moduleXmlMod(moduleXmlRelPath) = ##class(%File).SHA1Hash(moduleXmlPath, 1) - do ##class(%IPM.Storage.FileHash).CommitChanges(module, .moduleXmlMod, .emptyDel, 0) + set commitSC = ##class(%IPM.Storage.FileHash).CommitChanges(module, .moduleXmlMod, .emptyDel, 0) + if $$$ISERR(commitSC) { + write !, "Warning: failed to record module.xml hash: ", $system.Status.GetOneErrorText(commitSC) + } } ClassMethod SyncPrintModuleXmlWarning(moduleName As %String) diff --git a/tests/integration_tests/Test/PM/Integration/Sync.cls b/tests/integration_tests/Test/PM/Integration/Sync.cls index b42f3b435..9ecc11253 100644 --- a/tests/integration_tests/Test/PM/Integration/Sync.cls +++ b/tests/integration_tests/Test/PM/Integration/Sync.cls @@ -19,35 +19,32 @@ Method OnBeforeAllTests() As %Status } set ..TempDir = ##class(%File).NormalizeDirectory(##class(%File).ManagerDirectory() _ "sync-test-" _ $job) + quit $$$OK +} + +// Load a clean copy of the fixture in dev mode before each test to stamp a baseline. +// Uninstalling first handles version-change tests (e.g. TestModuleXmlChangedWarning bumps version). +Method OnBeforeOneTest(testName As %String) As %Status +{ $$$ThrowOnError(##class(%IPM.Utils.File).CreateDirectoryChain(..TempDir)) if '##class(%Library.File).CopyDir(..PristineDir, ..TempDir, 1) { - quit $$$ERROR($$$GeneralError, "Failed to copy sync-test fixture to temp directory") + quit $$$ERROR($$$GeneralError, "Failed to restore sync-test fixture before test: " _ testName) } - - // Load in dev mode — this stamps the baseline via %Compile hook. quit ##class(%IPM.Main).Shell("load " _ ..TempDir _ " -dev") } -// Uninstall and restore the fixture after each test to prevent state leakage between tests. -// Uninstalling first handles version-change tests (e.g. TestModuleXmlChangedWarning bumps version). Method OnAfterOneTest(testName As %String) As %Status { - try { - // Uninstall may fail if the test left the module in a broken state — that's expected. - do ##class(%IPM.Main).Shell("uninstall sync-test") - } catch {} + // Shell never throws; ignore the returned status since uninstall may fail if the test + // left the module in a broken state — that's expected. + do ##class(%IPM.Main).Shell("uninstall sync-test") do ##class(%Library.File).RemoveDirectoryTree(..TempDir) - $$$ThrowOnError(##class(%IPM.Utils.File).CreateDirectoryChain(..TempDir)) - if '##class(%Library.File).CopyDir(..PristineDir, ..TempDir, 1) { - quit $$$ERROR($$$GeneralError, "Failed to restore sync-test fixture after test: " _ testName) - } - quit ##class(%IPM.Main).Shell("load " _ ..TempDir _ " -dev") + quit $$$OK } Method OnAfterAllTests() As %Status { - // OnAfterOneTest already uninstalled and reloaded after the last test. - // This just removes the temp directories. + do ##class(%IPM.Main).Shell("uninstall sync-test") if ..TempDir '= "" { do ##class(%Library.File).RemoveDirectoryTree(..TempDir) } @@ -144,6 +141,19 @@ Method TestSuperclassEditRecompilesSubclass() do $$$AssertStatusOK(sc, "Sync after superclass edit succeeds (subclass recompiles via u-flag)") } +/// Editing an include file recompiles classes that include it, even though those class files +/// on disk are unchanged. SyncTest.Consumer includes SyncTest.INC and calls $$$SyncTestValue. +Method TestIncludeEditRecompilesConsumer() +{ + set incPath = ..TempDir _ "src/inc/SyncTest.inc" + do ..ReplaceInFile(incPath, """original-include""", """modified-include""") + + kill params + set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + do $$$AssertStatusOK(sc, "Sync after include edit succeeds") + do $$$AssertEquals(##class(SyncTest.Consumer).GetMacroValue(), "modified-include", "Consumer reflects updated macro value after include sync") +} + /// A file outside every tracked resource (e.g. under node_modules/) is ignored by sync /// and never given a baseline row. Method TestUntrackedFileIgnored() @@ -392,8 +402,12 @@ Method TestSyncTestFlagOnlyRunsOwningResource() Method TestSyncAllDevModeModules() { kill params + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) set sc = ##class(%IPM.Main).Shell("sync") + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + do $$$AssertStatusOK(sc, "Sync with no module name (sync-all) succeeds") + do $$$AssertTrue(..FindInOutput(.output, "Nothing to sync"), "sync-test reports nothing to sync (no changes since baseline)") } /// Sync fails with an error when given a module name that isn't installed. @@ -408,7 +422,7 @@ Method TestSyncModuleNotFound() Method TestSyncNonDevModeModule() { // Reload without -dev so the installed module is not in development mode. - // OnAfterOneTest unconditionally uninstalls and reloads -dev afterward, so no manual restore is needed here. + // OnAfterOneTest uninstalls, and OnBeforeOneTest will reinstall -dev for the next test. do ##class(%IPM.Main).Shell("uninstall sync-test") $$$ThrowOnError(##class(%IPM.Main).Shell("load " _ ..TempDir)) From 8a4d3a16a2f8406784b1368e91c35835431e4baa Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Fri, 17 Jul 2026 11:59:50 -0400 Subject: [PATCH 12/39] First part of rework to use BFS --- src/cls/IPM/Lifecycle/Base.cls | 15 +- src/cls/IPM/Storage/FileHash.cls | 166 ++++++++++++++---- src/cls/IPM/Storage/Module.cls | 43 ++++- .../Test/PM/Integration/Sync.cls | 77 ++++++++ .../_data/sync-flat-test/module.xml | 12 ++ .../sync-flat-test/src/SyncFlat/Flat.cls | 6 + 6 files changed, 277 insertions(+), 42 deletions(-) create mode 100644 tests/integration_tests/Test/PM/Integration/_data/sync-flat-test/module.xml create mode 100644 tests/integration_tests/Test/PM/Integration/_data/sync-flat-test/src/SyncFlat/Flat.cls diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls index 70d8b2035..1da642e69 100644 --- a/src/cls/IPM/Lifecycle/Base.cls +++ b/src/cls/IPM/Lifecycle/Base.cls @@ -733,10 +733,10 @@ Method GetTrackedPaths(Output trackedPaths) { kill trackedPaths - // Include module.xml + // Always derive paths from resource metadata — this discovers new files that aren't + // stamped yet (e.g. a newly-added resource or a new file in a directory-scanned resource). set trackedPaths(##class(%IPM.Storage.FileHash).NormalizePath("module.xml")) = "" - // Walk resources and resolve children to get all source file paths set orderedResourceList = ..Module.GetOrderedResourceList() set key = "" for { @@ -769,8 +769,6 @@ Method GetTrackedPaths(Output trackedPaths) set trackedPaths(##class(%IPM.Storage.FileHash).NormalizePath(relPath)) = "" } - // Sync-only enumeration for resources whose files aren't captured by ResolveChildren - // (e.g. FileCopy's directory-scanned source). Does not affect packaging/export. kill syncOnlyPaths set sc = resource.Processor.OnSyncResolveFiles(.syncOnlyPaths) if $$$ISERR(sc) { @@ -783,6 +781,15 @@ Method GetTrackedPaths(Output trackedPaths) set trackedPaths(##class(%IPM.Storage.FileHash).NormalizePath(relPath)) = "" } } + + // Also merge stored paths. These cover files at non-standard locations that derivation + // would place at a different path (e.g. no cls/ subdir). Without stored paths, a change + // to such a file is never presented to ComputeChanges and sync can't detect it. + if ##class(%IPM.Storage.FileHash).HasBaseline(..Module.Name) { + kill storedPaths + do ##class(%IPM.Storage.FileHash).GetStoredPaths(..Module.Name, .storedPaths) + merge trackedPaths = storedPaths + } } Method InstallOrDownloadPythonRequirements( diff --git a/src/cls/IPM/Storage/FileHash.cls b/src/cls/IPM/Storage/FileHash.cls index 77654cb56..285909c9e 100644 --- a/src/cls/IPM/Storage/FileHash.cls +++ b/src/cls/IPM/Storage/FileHash.cls @@ -1,4 +1,4 @@ -Include %IPM.Common +Include (%IPM.Common, %occReference) Class %IPM.Storage.FileHash Extends %Persistent { @@ -27,15 +27,16 @@ Index ModuleNameIndex On ModuleName; ForeignKey ModuleNameFK(ModuleName) References %IPM.Storage.Module(Name) [ OnDelete = cascade ]; -/// Record the current mtime+size for each tracked file without reading file content (no hash). -/// This "stamp" establishes a baseline so subsequent sync calls can detect changes via ComputeChanges. -/// Called after a successful dev-mode %Compile (after %Reload) so all resources, including -/// unit test classes compiled during %Compile, are present on disk and in ^oddDEF. +/// Stamp actual file paths for this module by combining two passes: +/// Pass 1 covers convention-derived paths that exist on disk (standard layouts). +/// Pass 2 walks the module root for compilable files not found at their derived location. ClassMethod StampModule(module As %IPM.Storage.Module, ByRef trackedPaths) As %Status { set sc = $$$OK try { set root = ##class(%File).NormalizeDirectory(module.Root) + + // Pass 1: stamp all convention-derived paths that exist on disk. set relPath = "" for { set relPath = $order(trackedPaths(relPath)) @@ -45,20 +46,57 @@ ClassMethod StampModule(module As %IPM.Storage.Module, ByRef trackedPaths) As %S if '##class(%File).Exists(fullPath) { continue } + $$$ThrowOnError(..StampOneFile(module.Name, root, fullPath)) + } - set normalizedRelPath = ..NormalizePath(relPath) - set existing = ..ModulePathIndexOpen(module.Name, normalizedRelPath, , .openSC) - if $isobject(existing) { - set instance = existing - } else { - set instance = ..%New() - set instance.ModuleName = module.Name - set instance.RelativePath = normalizedRelPath + // Pass 2: walk the module root for compilable files not already in trackedPaths. + // Catches non-standard layouts where OnItemRelativePath produced a path that doesn't + // match the actual file location on disk. + // Uses a manual BFS queue because File_FileSet's recursive flag only returns + // immediate children — it does not descend through multiple directory levels in one call. + kill walkQueue + set walkHead = 1, walkTail = 1 + set walkQueue(walkTail) = root + for { + quit:(walkHead > walkTail) + set walkDir = walkQueue(walkHead) + set walkHead = walkHead + 1 + + set walkResult = ##class(%SQL.Statement).%ExecDirect(, + "SELECT Name, Type FROM %Library.File_FileSet(?, ?, ?, ?)", + walkDir, "*", "", 0) + while walkResult.%Next() { + set entryType = walkResult.%Get("Type") + set entryPath = walkResult.%Get("Name") + if entryType = "D" { + set walkTail = walkTail + 1 + set walkQueue(walkTail) = ##class(%File).NormalizeDirectory(entryPath) + continue + } + set ext = $$$lcase($piece(entryPath, ".", *)) + if ",cls,inc,mac,int," '[ (","_ext_",") { + continue + } + set normalizedRelPath = ..NormalizePath($extract(entryPath, $length(root) + 1, *)) + if $data(trackedPaths(normalizedRelPath)) { + continue + } + set docName = ..RelPathToDocName(normalizedRelPath) + if docName = "" { + continue + } + set docExt = $$$lcase($piece(docName, ".", *)) + if docExt = "cls" { + if '$$$comClassDefined($piece(docName, ".", 1, *-1)) { + continue + } + } else { + if '##class(%RoutineMgr).Exists(docName) { + continue + } + } + $$$ThrowOnError(..StampOneFile(module.Name, root, entryPath)) } - - set instance.FileSize = ##class(%File).GetFileSize(fullPath) - set instance.FileTimestamp = ..GetFileTimestamp(fullPath) - $$$ThrowOnError(instance.%Save()) } } catch e { set sc = e.AsStatus() @@ -66,6 +104,62 @@ ClassMethod StampModule(module As %IPM.Storage.Module, ByRef trackedPaths) As %S quit sc } +/// Stamp (or update) the FileHash row for a single file. Computes hash at stamp time. +ClassMethod StampOneFile(moduleName As %String, root As %String, fullPath As %String) As %Status +{ + set normalizedRelPath = ..NormalizePath($extract(fullPath, $length(root) + 1, *)) + set existing = ..ModulePathIndexOpen(moduleName, normalizedRelPath, , .openSC) + if $isobject(existing) { + set instance = existing + } else { + set instance = ..%New() + set instance.ModuleName = moduleName + set instance.RelativePath = normalizedRelPath + } + set instance.Hash = ##class(%File).SHA1Hash(fullPath, 1) + set instance.FileSize = ##class(%File).GetFileSize(fullPath) + set instance.FileTimestamp = ..GetFileTimestamp(fullPath) + quit instance.%Save() +} + +/// Derive a server document name from a module-root-relative path. +/// Returns "" for paths that don't map to a compilable document. +/// Examples: +/// "src/cls/Foo/Bar.cls" -> "Foo.Bar.CLS" +/// "src/SyncFlat/Flat.cls" -> "SyncFlat.Flat.CLS" +/// "src/inc/MyInc.inc" -> "MyInc.INC" +ClassMethod RelPathToDocName(relPath As %String) As %String +{ + set parts = $length(relPath, "/") + set ext = $$$lcase($piece(relPath, ".", *)) + if ",cls,inc,mac,int," '[ (","_ext_",") { + quit "" + } + // Skip leading path segments that are conventional prefix directories. + // Stop at the first segment that isn't one of these well-known prefixes. + set startSeg = 1 + for i = 1:1:parts { + set seg = $piece(relPath, "/", i) + if ",src,cls,inc,mac,int," [ (","_$$$lcase(seg)_",") { + set startSeg = i + 1 + } else { + quit + } + } + if startSeg > parts { + quit "" + } + // Segments startSeg through (parts-1) are package/directory components. + // Segment parts is "ClassName.ext" — strip the extension for the final component. + set name = "" + for i = startSeg:1:(parts - 1) { + set name = name _ $select(name="":"", 1:".") _ $piece(relPath, "/", i) + } + set lastName = $piece($piece(relPath, "/", parts), ".", 1, *-1) + set name = name _ $select(name="":"", 1:".") _ lastName + quit name _ "." _ $$$UPPER(ext) +} + /// Compute which tracked files changed on disk vs stored baseline. /// Returns modified(relPath)=newHash and deleted(relPath)="" arrays. ClassMethod ComputeChanges(module As %IPM.Storage.Module, ByRef trackedPaths, Output modified, Output deleted) As %Status @@ -90,31 +184,18 @@ ClassMethod ComputeChanges(module As %IPM.Storage.Module, ByRef trackedPaths, Ou continue } - set currentSize = ##class(%File).GetFileSize(fullPath) - set currentTimestamp = ..GetFileTimestamp(fullPath) - set existing = ..ModulePathIndexOpen(module.Name, normalizedRelPath, , .openSC) if '$isobject(existing) { - // No baseline row — file is newly tracked (added since the last sync/stamp). - // Report it as modified so it gets loaded; no prior hash to compare against. - set modified(normalizedRelPath) = ##class(%File).SHA1Hash(fullPath, 1) + // No baseline row — file joined trackedPaths after initial load (e.g. added via + // OnSyncResolveFiles). Stamp it silently so it doesn't trigger a spurious + // "modified" on every sync until someone edits it. + do ..StampOneFile(module.Name, root, fullPath) continue } - // Fast path: size+mtime match means unchanged - if (existing.FileSize = currentSize) && (existing.FileTimestamp = currentTimestamp) { - continue - } - - // Size or mtime differ — read hash to confirm real change + // Hash-only comparison. Avoids mtime unreliability on bind mounts. set newHash = ##class(%File).SHA1Hash(fullPath, 1) - if (existing.Hash '= "") && (newHash = existing.Hash) { - // Hash matches stored — content unchanged despite mtime/size difference; update fast-path - set existing.FileTimestamp = currentTimestamp - set existing.FileSize = currentSize - $$$ThrowOnError(existing.%Save()) - } else { - // Content changed (or no prior hash to confirm otherwise) + if existing.Hash '= newHash { set modified(normalizedRelPath) = newHash } } @@ -179,6 +260,19 @@ ClassMethod HasBaseline(moduleName As %String) As %Boolean quit result.%Next() } +/// Populate paths(normalizedRelPath)="" for all stored baseline rows for this module. +/// Same structure as the trackedPaths array used by GetTrackedPaths and ComputeChanges. +ClassMethod GetStoredPaths(moduleName As %String, Output paths) +{ + kill paths + set result = ##class(%SQL.Statement).%ExecDirect(, + "SELECT RelativePath FROM %IPM_Storage.FileHash WHERE ModuleName = ?", + moduleName) + while result.%Next() { + set paths(result.%Get("RelativePath")) = "" + } +} + /// Normalize a relative path: forward slashes, collapse //, strip leading slash. ClassMethod NormalizePath(path As %String) As %String { diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls index e554673fb..ab2a6293a 100644 --- a/src/cls/IPM/Storage/Module.cls +++ b/src/cls/IPM/Storage/Module.cls @@ -897,6 +897,7 @@ ClassMethod SyncCheckModuleXml( /// Used by SyncRouteChanges to map changed files back to their owning resource processors. ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, ByRef reverseIndex) { + // Pass 1: derive paths from resource metadata (covers standard layouts) set orderedResourceList = module.GetOrderedResourceList() set key = "" for { @@ -932,8 +933,6 @@ ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, ByRef reverseIn set reverseIndex(normalizedRelPath, "Resource") = resource } - // Sync-only enumeration for resources whose files aren't captured by ResolveChildren - // (e.g. FileCopy's directory-scanned source). Does not affect packaging/export. kill syncOnlyPaths set childSC = resource.Processor.OnSyncResolveFiles(.syncOnlyPaths) if $$$ISERR(childSC) { @@ -950,6 +949,46 @@ ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, ByRef reverseIn set reverseIndex(normalizedRelPath, "Resource") = resource } } + + // Pass 2: for stored paths not covered by Pass 1 (non-standard layouts), + // derive the document name, find the owning resource, and add to index. + kill storedPaths + do ##class(%IPM.Storage.FileHash).GetStoredPaths(module.Name, .storedPaths) + set relPath = "" + for { + set relPath = $order(storedPaths(relPath)) + quit:relPath="" + + if $data(reverseIndex(relPath)) { + continue + } + + set docName = ##class(%IPM.Storage.FileHash).RelPathToDocName(relPath) + if docName = "" { + continue + } + + set key = "" + for { + set resource = orderedResourceList.GetNext(.key) + quit:key="" + + if '$isobject(resource.Processor) { + continue + } + + kill childArr + set childSC = resource.ResolveChildren(.childArr) + if $$$ISERR(childSC) || '$data(childArr(docName)) { + continue + } + + set reverseIndex(relPath) = resource.Name + set reverseIndex(relPath, "Processor") = resource.Processor + set reverseIndex(relPath, "Resource") = resource + quit + } + } } /// Partition modified and deleted paths into syncByResource (keyed by resource name) and diff --git a/tests/integration_tests/Test/PM/Integration/Sync.cls b/tests/integration_tests/Test/PM/Integration/Sync.cls index 9ecc11253..78d08a618 100644 --- a/tests/integration_tests/Test/PM/Integration/Sync.cls +++ b/tests/integration_tests/Test/PM/Integration/Sync.cls @@ -54,6 +54,83 @@ Method OnAfterAllTests() As %Status quit $$$OK } +/// GetStoredPaths returns paths that were stamped by StampModule. +Method TestGetStoredPathsReturnsStampedPaths() +{ + do $$$AssertTrue(##class(%IPM.Storage.FileHash).HasBaseline("sync-test"), "Baseline exists after load") + + kill paths + do ##class(%IPM.Storage.FileHash).GetStoredPaths("sync-test", .paths) + + do $$$AssertTrue($data(paths("module.xml")), "module.xml in stored paths") + do $$$AssertTrue($data(paths("src/cls/SyncTest/SuperClass.cls")), "SuperClass.cls in stored paths") + do $$$AssertTrue($data(paths("src/inc/SyncTest.inc")), "SyncTest.inc in stored paths") +} + +/// ComputeChanges must detect a content change even when mtime and size are identical +/// (simulates bind-mount stale mtime — Windows host edit invisible to container mtime). +Method TestChangeDetectedWithStaleMtime() +{ + set filePath = ..TempDir _ "src/cls/SyncTest/SuperClass.cls" + + // Modify content + do ..ReplaceInFile(filePath, "Property BaseValue", "Property BaseValueModified") + + // Forge the FileHash row: update mtime and size to the post-edit values but keep the + // old hash. This is the bind-mount scenario: IRIS sees the new mtime+size but the + // fast-path would conclude "unchanged" because they match the forged row. + set existing = ##class(%IPM.Storage.FileHash).ModulePathIndexOpen("sync-test", "src/cls/SyncTest/SuperClass.cls") + set existing.FileTimestamp = ##class(%IPM.Storage.FileHash).GetFileTimestamp(filePath) + set existing.FileSize = ##class(%File).GetFileSize(filePath) + // Leave existing.Hash as the old value intentionally + $$$ThrowOnError(existing.%Save()) + + kill params + set params("Verbose") = 1 + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + + do $$$AssertStatusOK(sc, "Sync detects content change despite matching mtime/size") + do $$$AssertTrue(..FindInOutput(.output, "Sync complete"), "Content change was detected and applied (not 'Nothing to sync')") +} + +/// Confirm sync detects changes in a module whose file layout doesn't match the +/// SourcesRoot/Directory/package.ext convention. sync-flat-test declares SyncFlat.Flat.CLS +/// with SourcesRoot=src, but the file is at src/SyncFlat/Flat.cls (no cls/ subdirectory). +/// OnItemRelativePath derives src/cls/SyncFlat/Flat.cls, which doesn't exist — so without +/// filesystem-anchored stamping, the file never gets a baseline row and sync always reports +/// "Nothing to sync" regardless of what changed. +// This test manages its own install/uninstall because sync-flat-test uses a separate fixture +// from sync-test and must not interfere with the per-test lifecycle. If the test throws before +// cleanup, sync-flat-test and flatTempDir are leaked (no safety net from OnAfterOneTest). +Method TestNonStandardLayoutDetectsChange() +{ + set flatSource = ..GetModuleDir("sync-flat-test") + set flatTempDir = ##class(%File).NormalizeDirectory(##class(%File).ManagerDirectory() _ "sync-flat-test-" _ $job) + $$$ThrowOnError(##class(%IPM.Utils.File).CreateDirectoryChain(flatTempDir)) + if '##class(%Library.File).CopyDir(flatSource, flatTempDir, 1) { + $$$ThrowOnError($$$ERROR($$$GeneralError, "Failed to copy sync-flat-test to temp dir")) + } + + $$$ThrowOnError(##class(%IPM.Main).Shell("load " _ flatTempDir _ " -dev")) + + set filePath = flatTempDir _ "src/SyncFlat/Flat.cls" + do ..ReplaceInFile(filePath, "As %String", "As %Integer") + + kill params + set params("Verbose") = 1 + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Storage.Module).Sync("sync-flat-test", .params) + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + + do $$$AssertStatusOK(sc, "Sync succeeds for non-standard layout module") + do $$$AssertTrue(..FindInOutput(.output, "Sync complete"), "Sync detects the file change (not 'Nothing to sync')") + + do ##class(%IPM.Main).Shell("uninstall sync-flat-test") + do ##class(%Library.File).RemoveDirectoryTree(flatTempDir) +} + /// Sync with no files changed on disk since the last load/sync reports nothing to do. Method TestNoChangeIsNoOp() { diff --git a/tests/integration_tests/Test/PM/Integration/_data/sync-flat-test/module.xml b/tests/integration_tests/Test/PM/Integration/_data/sync-flat-test/module.xml new file mode 100644 index 000000000..6513b0747 --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/sync-flat-test/module.xml @@ -0,0 +1,12 @@ + + + + + sync-flat-test + 1.0.0 + module + src + + + + diff --git a/tests/integration_tests/Test/PM/Integration/_data/sync-flat-test/src/SyncFlat/Flat.cls b/tests/integration_tests/Test/PM/Integration/_data/sync-flat-test/src/SyncFlat/Flat.cls new file mode 100644 index 000000000..9d8d222e0 --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/sync-flat-test/src/SyncFlat/Flat.cls @@ -0,0 +1,6 @@ +Class SyncFlat.Flat +{ + +Property Value As %String; + +} From a14c809c970d9bfc6dc6c27a470731b1b817bfcc Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Mon, 20 Jul 2026 09:52:49 -0400 Subject: [PATCH 13/39] Use BFS directory scanning --- src/cls/IPM/Lifecycle/Base.cls | 70 +-------- src/cls/IPM/ResourceProcessor/Test.cls | 5 +- src/cls/IPM/Storage/FileHash.cls | 202 +++++++++++++++++++------ src/cls/IPM/Storage/Module.cls | 14 +- 4 files changed, 172 insertions(+), 119 deletions(-) diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls index 1da642e69..b43c9b102 100644 --- a/src/cls/IPM/Lifecycle/Base.cls +++ b/src/cls/IPM/Lifecycle/Base.cls @@ -727,71 +727,6 @@ Method %Reload(ByRef pParams) As %Status quit tSC } -/// Build the set of tracked paths (resource-owned files + module.xml) for sync change detection. -/// Output: trackedPaths(normalizedRelPath)="" -Method GetTrackedPaths(Output trackedPaths) -{ - kill trackedPaths - - // Always derive paths from resource metadata — this discovers new files that aren't - // stamped yet (e.g. a newly-added resource or a new file in a directory-scanned resource). - set trackedPaths(##class(%IPM.Storage.FileHash).NormalizePath("module.xml")) = "" - - set orderedResourceList = ..Module.GetOrderedResourceList() - set key = "" - for { - set resource = orderedResourceList.GetNext(.key) - quit:key="" - - if '$isobject(resource.Processor) { - continue - } - - kill childArr - set sc = resource.ResolveChildren(.childArr) - if $$$ISERR(sc) { - continue - } - - set childName = "" - for { - set childName = $order(childArr(childName)) - quit:childName="" - - set relPath = $get(childArr(childName, "RelativePath")) - if relPath = "" { - set relPath = resource.Processor.OnItemRelativePath(childName) - } - if relPath = "" { - continue - } - - set trackedPaths(##class(%IPM.Storage.FileHash).NormalizePath(relPath)) = "" - } - - kill syncOnlyPaths - set sc = resource.Processor.OnSyncResolveFiles(.syncOnlyPaths) - if $$$ISERR(sc) { - continue - } - set relPath = "" - for { - set relPath = $order(syncOnlyPaths(relPath)) - quit:relPath="" - set trackedPaths(##class(%IPM.Storage.FileHash).NormalizePath(relPath)) = "" - } - } - - // Also merge stored paths. These cover files at non-standard locations that derivation - // would place at a different path (e.g. no cls/ subdir). Without stored paths, a change - // to such a file is never presented to ComputeChanges and sync can't detect it. - if ##class(%IPM.Storage.FileHash).HasBaseline(..Module.Name) { - kill storedPaths - do ##class(%IPM.Storage.FileHash).GetStoredPaths(..Module.Name, .storedPaths) - merge trackedPaths = storedPaths - } -} - Method InstallOrDownloadPythonRequirements( pRoot As %String = "", ByRef pParams, @@ -1271,11 +1206,10 @@ Method %Compile(ByRef pParams) As %Status // Stamp file baselines for sync change detection (dev mode only). // Non-fatal: stamping failure must not break a normal compile cycle. - // Done here after compile so test classes are in ^oddDEF and appear in trackedPaths. + // Done after compile so test classes are in ^oddDEF and pass the namespace filter in StampModule. if tDevMode { try { - do ..GetTrackedPaths(.trackedPaths) - do ##class(%IPM.Storage.FileHash).StampModule(..Module, .trackedPaths) + $$$ThrowOnError(##class(%IPM.Storage.FileHash).StampModule(..Module)) } catch stampEx { write !, "Warning: sync baseline stamping failed: ", $system.Status.GetOneErrorText(stampEx.AsStatus()) } diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls index 5b998db1b..5a3ef7d1d 100644 --- a/src/cls/IPM/ResourceProcessor/Test.cls +++ b/src/cls/IPM/ResourceProcessor/Test.cls @@ -383,9 +383,8 @@ Method SupportsSync() As %Boolean /// Enumerate test files on disk so sync can detect brand-new test classes. /// ResolveChildren (via GetChildren's StudioOpenDialog query) only sees already-compiled /// classes, so a test file that was never compiled — e.g. one just added to the test -/// directory — is otherwise invisible to GetTrackedPaths/ComputeChanges until something -/// else compiles it first. This live directory scan closes that gap, mirroring FileCopy's -/// OnSyncResolveFiles for the same reason. +/// directory — is otherwise invisible to ComputeChanges until something else compiles it +/// first. This live directory scan closes that gap. Method OnSyncResolveFiles(Output relPaths) As %Status { set sc = $$$OK diff --git a/src/cls/IPM/Storage/FileHash.cls b/src/cls/IPM/Storage/FileHash.cls index 285909c9e..4f6f71745 100644 --- a/src/cls/IPM/Storage/FileHash.cls +++ b/src/cls/IPM/Storage/FileHash.cls @@ -27,31 +27,23 @@ Index ModuleNameIndex On ModuleName; ForeignKey ModuleNameFK(ModuleName) References %IPM.Storage.Module(Name) [ OnDelete = cascade ]; -/// Stamp actual file paths for this module by combining two passes: -/// Pass 1 covers convention-derived paths that exist on disk (standard layouts). -/// Pass 2 walks the module root for compilable files not found at their derived location. -ClassMethod StampModule(module As %IPM.Storage.Module, ByRef trackedPaths) As %Status +/// Stamp all tracked files for a module. Two passes: +/// Pass 1 walks the module root for compilable files (cls/inc/mac/int) present in the namespace. +/// Pass 2 calls OnSyncResolveFiles on each processor to stamp non-compilable tracked files (e.g. FileCopy). +/// Also stamps module.xml. +ClassMethod StampModule(module As %IPM.Storage.Module) As %Status { set sc = $$$OK try { set root = ##class(%File).NormalizeDirectory(module.Root) - // Pass 1: stamp all convention-derived paths that exist on disk. - set relPath = "" - for { - set relPath = $order(trackedPaths(relPath)) - quit:relPath="" - - set fullPath = ##class(%File).NormalizeFilename(relPath, root) - if '##class(%File).Exists(fullPath) { - continue - } - $$$ThrowOnError(..StampOneFile(module.Name, root, fullPath)) + // module.xml is always tracked. + set moduleXmlPath = root _ "module.xml" + if ##class(%File).Exists(moduleXmlPath) { + $$$ThrowOnError(..StampOneFile(module.Name, root, moduleXmlPath)) } - // Pass 2: walk the module root for compilable files not already in trackedPaths. - // Catches non-standard layouts where OnItemRelativePath produced a path that doesn't - // match the actual file location on disk. + // Pass 1: BFS the module root for compilable files present in the namespace. // Uses a manual BFS queue because File_FileSet's recursive flag only returns // immediate children — it does not descend through multiple directory levels in one call. kill walkQueue @@ -78,9 +70,6 @@ ClassMethod StampModule(module As %IPM.Storage.Module, ByRef trackedPaths) As %S continue } set normalizedRelPath = ..NormalizePath($extract(entryPath, $length(root) + 1, *)) - if $data(trackedPaths(normalizedRelPath)) { - continue - } set docName = ..RelPathToDocName(normalizedRelPath) if docName = "" { continue @@ -98,6 +87,32 @@ ClassMethod StampModule(module As %IPM.Storage.Module, ByRef trackedPaths) As %S $$$ThrowOnError(..StampOneFile(module.Name, root, entryPath)) } } + + // Pass 2: stamp non-compilable tracked files via OnSyncResolveFiles on each processor. + set orderedResourceList = module.GetOrderedResourceList() + set key = "" + for { + set resource = orderedResourceList.GetNext(.key) + quit:key="" + if '$isobject(resource.Processor) { + continue + } + kill syncOnlyPaths + set childSC = resource.Processor.OnSyncResolveFiles(.syncOnlyPaths) + if $$$ISERR(childSC) { + continue + } + set relPath = "" + for { + set relPath = $order(syncOnlyPaths(relPath)) + quit:relPath="" + set fullPath = ##class(%File).NormalizeFilename(relPath, root) + if '##class(%File).Exists(fullPath) { + continue + } + $$$ThrowOnError(..StampOneFile(module.Name, root, fullPath)) + } + } } catch e { set sc = e.AsStatus() } @@ -160,43 +175,126 @@ ClassMethod RelPathToDocName(relPath As %String) As %String quit name _ "." _ $$$UPPER(ext) } -/// Compute which tracked files changed on disk vs stored baseline. +/// Compute which files changed on disk vs stored baseline. Self-contained: discovers files +/// via BFS (compilable) + OnSyncResolveFiles (non-compilable), then iterates stored rows for deletions. /// Returns modified(relPath)=newHash and deleted(relPath)="" arrays. -ClassMethod ComputeChanges(module As %IPM.Storage.Module, ByRef trackedPaths, Output modified, Output deleted) As %Status +ClassMethod ComputeChanges(module As %IPM.Storage.Module, Output modified, Output deleted) As %Status { set sc = $$$OK kill modified, deleted try { set root = ##class(%File).NormalizeDirectory(module.Root) + kill seen - set relPath = "" - for { - set relPath = $order(trackedPaths(relPath)) - quit:relPath="" + // module.xml + set moduleXmlPath = root _ "module.xml" + if ##class(%File).Exists(moduleXmlPath) { + do ..CheckOneFile(module.Name, root, moduleXmlPath, .modified, .seen) + } - set normalizedRelPath = ..NormalizePath(relPath) - set fullPath = ##class(%File).NormalizeFilename(relPath, root) + // Pass 1: BFS module root for compilable files present in the namespace. + kill walkQueue + set walkHead = 1, walkTail = 1 + set walkQueue(walkTail) = root + for { + quit:(walkHead > walkTail) + set walkDir = walkQueue(walkHead) + set walkHead = walkHead + 1 - if '##class(%File).Exists(fullPath) { - if ..ModulePathIndexExists(module.Name, normalizedRelPath) { - set deleted(normalizedRelPath) = "" + set walkResult = ##class(%SQL.Statement).%ExecDirect(, + "SELECT Name, Type FROM %Library.File_FileSet(?, ?, ?, ?)", + walkDir, "*", "", 0) + while walkResult.%Next() { + set entryType = walkResult.%Get("Type") + set entryPath = walkResult.%Get("Name") + if entryType = "D" { + set walkTail = walkTail + 1 + set walkQueue(walkTail) = ##class(%File).NormalizeDirectory(entryPath) + continue } + set ext = $$$lcase($piece(entryPath, ".", *)) + if ",cls,inc,mac,int," '[ (","_ext_",") { + continue + } + set docName = ..RelPathToDocName(..NormalizePath($extract(entryPath, $length(root) + 1, *))) + if docName = "" { + continue + } + set docExt = $$$lcase($piece(docName, ".", *)) + if docExt = "cls" { + if '$$$comClassDefined($piece(docName, ".", 1, *-1)) { + continue + } + } else { + if '##class(%RoutineMgr).Exists(docName) { + continue + } + } + do ..CheckOneFile(module.Name, root, entryPath, .modified, .seen) + } + } + + // Pass 2: manifest-derived paths for files not yet compiled (e.g. newly-declared + // resource after manifest reload). BFS skips these because they fail the namespace + // filter, but they exist on disk and should be detected as new. + set orderedResourceList = module.GetOrderedResourceList() + set key = "" + for { + set resource = orderedResourceList.GetNext(.key) + quit:key="" + if '$isobject(resource.Processor) { + continue + } + kill childArr + set childSC = resource.ResolveChildren(.childArr) + if $$$ISERR(childSC) { continue } + set childName = "" + for { + set childName = $order(childArr(childName)) + quit:childName="" + set relPath = $get(childArr(childName, "RelativePath")) + if relPath = "" { + set relPath = resource.Processor.OnItemRelativePath(childName) + } + if relPath = "" { + continue + } + set fullPath = ##class(%File).NormalizeFilename(relPath, root) + if '##class(%File).Exists(fullPath) { + continue + } + do ..CheckOneFile(module.Name, root, fullPath, .modified, .seen) + } - set existing = ..ModulePathIndexOpen(module.Name, normalizedRelPath, , .openSC) - if '$isobject(existing) { - // No baseline row — file joined trackedPaths after initial load (e.g. added via - // OnSyncResolveFiles). Stamp it silently so it doesn't trigger a spurious - // "modified" on every sync until someone edits it. - do ..StampOneFile(module.Name, root, fullPath) + // Also check OnSyncResolveFiles for non-compilable tracked files. + kill syncOnlyPaths + set childSC = resource.Processor.OnSyncResolveFiles(.syncOnlyPaths) + if $$$ISERR(childSC) { continue } + set relPath = "" + for { + set relPath = $order(syncOnlyPaths(relPath)) + quit:relPath="" + set fullPath = ##class(%File).NormalizeFilename(relPath, root) + if '##class(%File).Exists(fullPath) { + continue + } + do ..CheckOneFile(module.Name, root, fullPath, .modified, .seen) + } + } - // Hash-only comparison. Avoids mtime unreliability on bind mounts. - set newHash = ##class(%File).SHA1Hash(fullPath, 1) - if existing.Hash '= newHash { - set modified(normalizedRelPath) = newHash + // Pass 3: iterate all stored rows — file missing from disk → deleted. + set result = ##class(%SQL.Statement).%ExecDirect(, + "SELECT RelativePath FROM %IPM_Storage.FileHash WHERE ModuleName = ?", + module.Name) + while result.%Next() { + set storedRelPath = result.%Get("RelativePath") + set fullPath = ##class(%File).NormalizeFilename(storedRelPath, root) + if '##class(%File).Exists(fullPath) { + set deleted(storedRelPath) = "" } } } catch e { @@ -205,6 +303,25 @@ ClassMethod ComputeChanges(module As %IPM.Storage.Module, ByRef trackedPaths, Ou quit sc } +/// Check a single file against its stored baseline. If no row or hash differs, add to modified. +ClassMethod CheckOneFile(moduleName As %String, root As %String, fullPath As %String, ByRef modified, ByRef seen) [ Private ] +{ + set normalizedRelPath = ..NormalizePath($extract(fullPath, $length(root) + 1, *)) + if $data(seen(normalizedRelPath)) { + quit + } + set seen(normalizedRelPath) = "" + set existing = ..ModulePathIndexOpen(moduleName, normalizedRelPath, , .openSC) + if '$isobject(existing) { + set modified(normalizedRelPath) = ##class(%File).SHA1Hash(fullPath, 1) + quit + } + set newHash = ##class(%File).SHA1Hash(fullPath, 1) + if existing.Hash '= newHash { + set modified(normalizedRelPath) = newHash + } +} + /// After a successful sync, commit new hashes for modified files and optionally remove deleted rows. ClassMethod CommitChanges(module As %IPM.Storage.Module, ByRef modified, ByRef deleted, processDeletes As %Boolean = 0) As %Status { @@ -261,7 +378,6 @@ ClassMethod HasBaseline(moduleName As %String) As %Boolean } /// Populate paths(normalizedRelPath)="" for all stored baseline rows for this module. -/// Same structure as the trackedPaths array used by GetTrackedPaths and ComputeChanges. ClassMethod GetStoredPaths(moduleName As %String, Output paths) { kill paths diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls index ab2a6293a..59d05a0c9 100644 --- a/src/cls/IPM/Storage/Module.cls +++ b/src/cls/IPM/Storage/Module.cls @@ -755,17 +755,21 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status set moduleXmlChanged = ..SyncCheckModuleXml(.module, moduleXmlPath, moduleXmlRelPath) set lifecycle = module.Lifecycle - // Step 2: Collect tracked paths and compute disk changes vs baseline - do lifecycle.GetTrackedPaths(.trackedPaths) - if '##class(%IPM.Storage.FileHash).HasBaseline(moduleName) { // Self-heal: establish baseline for modules loaded before this feature - do ##class(%IPM.Storage.FileHash).StampModule(module, .trackedPaths) + $$$ThrowOnError(##class(%IPM.Storage.FileHash).StampModule(module)) write !, "[", moduleName, "] Baseline established. Run sync again to detect changes." quit } - $$$ThrowOnError(##class(%IPM.Storage.FileHash).ComputeChanges(module, .trackedPaths, .modified, .deleted)) + // After a manifest reload, re-stamp so newly-declared resources get baseline rows + // in this same sync call rather than requiring a separate reload -dev. + if moduleXmlChanged { + $$$ThrowOnError(##class(%IPM.Storage.FileHash).StampModule(module)) + } + + // Step 2: Compute disk changes vs baseline (self-contained — BFS + stored rows) + $$$ThrowOnError(##class(%IPM.Storage.FileHash).ComputeChanges(module, .modified, .deleted)) // Exclude module.xml from file routing (handled by SyncCheckModuleXml above) kill modified(moduleXmlRelPath) From 61f30397c53afe0864f28e7f1a550802e46ee09a Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Mon, 20 Jul 2026 11:13:16 -0400 Subject: [PATCH 14/39] Drop the unused mtime/size properties --- CHANGELOG.md | 2 +- src/cls/IPM/Storage/FileHash.cls | 29 ++----------------- src/cls/IPM/Storage/Module.cls | 12 +------- .../Test/PM/Integration/Sync.cls | 28 ------------------ 4 files changed, 4 insertions(+), 67 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16f2799e0..1d160fe6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - #1178: Add `-password-env` and `-token-env` modifiers to the `repo` command to read the password/token from a named environment variable (secure alternatives to `-password` and `-token`). -- Add `sync` command for incremental loading of changed files in dev-mode modules. Detects modified files since last sync and recompiles only what is stale. Supports `-delete` for processing removed files and `-test` for running changed test-phase unit tests. +- #1117: Add `sync` command for incremental loading of changed files in dev-mode modules. Detects modified files since last sync using SHA-1 hash and recompiles only what is stale. Supports `-delete` for processing removed files and `-test` for running changed test-phase unit tests. ### Changed - #1186: Change %IPM.Main:ShellScript() to return a status. diff --git a/src/cls/IPM/Storage/FileHash.cls b/src/cls/IPM/Storage/FileHash.cls index 4f6f71745..f6a2e4f29 100644 --- a/src/cls/IPM/Storage/FileHash.cls +++ b/src/cls/IPM/Storage/FileHash.cls @@ -9,17 +9,8 @@ Property ModuleName As %String(MAXLEN = 255) [ Required ]; /// Used as the lookup key in ModulePathIndex — callers must normalize via NormalizePath before querying. Property RelativePath As %String(MAXLEN = 512) [ Required ]; -/// SHA-1 content hash (hex). Empty string means the row was stamped with mtime/size only and -/// no hash was computed yet. On the next sync, any mtime/size mismatch will compute a fresh hash. -Property Hash As %String(MAXLEN = 64) [ InitialExpression = "" ]; - -/// Last-modified timestamp as returned by ##class(%File).GetFileDateModified — $H format. -/// Used as a fast-path: if mtime and FileSize both match, assume content is unchanged. -Property FileTimestamp As %String(MAXLEN = 64); - -/// File size in bytes. Combined with FileTimestamp forms the fast-path unchanged check. -/// Both fields are needed: size alone misses same-size edits; mtime alone is unreliable on copies. -Property FileSize As %Integer; +/// SHA-1 content hash (hex). +Property Hash As %String(MAXLEN = 64) [ Required ]; Index ModulePathIndex On (ModuleName, RelativePath) [ Unique ]; @@ -132,8 +123,6 @@ ClassMethod StampOneFile(moduleName As %String, root As %String, fullPath As %St set instance.RelativePath = normalizedRelPath } set instance.Hash = ##class(%File).SHA1Hash(fullPath, 1) - set instance.FileSize = ##class(%File).GetFileSize(fullPath) - set instance.FileTimestamp = ..GetFileTimestamp(fullPath) quit instance.%Save() } @@ -345,8 +334,6 @@ ClassMethod CommitChanges(module As %IPM.Storage.Module, ByRef modified, ByRef d } set instance.Hash = newHash - set instance.FileSize = ##class(%File).GetFileSize(fullPath) - set instance.FileTimestamp = ..GetFileTimestamp(fullPath) $$$ThrowOnError(instance.%Save()) } @@ -402,12 +389,6 @@ ClassMethod NormalizePath(path As %String) As %String quit path } -/// Get the last-modified timestamp of a file in $H format (as returned by GetFileDateModified). -ClassMethod GetFileTimestamp(fullPath As %String) As %String -{ - quit ##class(%File).GetFileDateModified(fullPath) -} - Storage Default { @@ -423,12 +404,6 @@ Storage Default Hash - -FileTimestamp - - -FileSize - ^IPM.Storage.FileHashD FileHashDefaultData diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls index 59d05a0c9..019118c54 100644 --- a/src/cls/IPM/Storage/Module.cls +++ b/src/cls/IPM/Storage/Module.cls @@ -876,18 +876,8 @@ ClassMethod SyncCheckModuleXml( if '$isobject(existing) { quit 0 } - set currentSize = ##class(%File).GetFileSize(moduleXmlPath) - set currentTimestamp = ##class(%IPM.Storage.FileHash).GetFileTimestamp(moduleXmlPath) - if (existing.FileSize = currentSize) && (existing.FileTimestamp = currentTimestamp) { - quit 0 - } set newHash = ##class(%File).SHA1Hash(moduleXmlPath, 1) - if (existing.Hash '= "") && (newHash = existing.Hash) { - // Content unchanged despite mtime/size change — update stored metadata to avoid - // re-hashing on every subsequent sync until the file is written again. - set existing.FileSize = currentSize - set existing.FileTimestamp = currentTimestamp - do existing.%Save() + if newHash = existing.Hash { quit 0 } $$$ThrowOnError($system.OBJ.Load(moduleXmlPath, "-d")) diff --git a/tests/integration_tests/Test/PM/Integration/Sync.cls b/tests/integration_tests/Test/PM/Integration/Sync.cls index 78d08a618..e8a18411b 100644 --- a/tests/integration_tests/Test/PM/Integration/Sync.cls +++ b/tests/integration_tests/Test/PM/Integration/Sync.cls @@ -67,34 +67,6 @@ Method TestGetStoredPathsReturnsStampedPaths() do $$$AssertTrue($data(paths("src/inc/SyncTest.inc")), "SyncTest.inc in stored paths") } -/// ComputeChanges must detect a content change even when mtime and size are identical -/// (simulates bind-mount stale mtime — Windows host edit invisible to container mtime). -Method TestChangeDetectedWithStaleMtime() -{ - set filePath = ..TempDir _ "src/cls/SyncTest/SuperClass.cls" - - // Modify content - do ..ReplaceInFile(filePath, "Property BaseValue", "Property BaseValueModified") - - // Forge the FileHash row: update mtime and size to the post-edit values but keep the - // old hash. This is the bind-mount scenario: IRIS sees the new mtime+size but the - // fast-path would conclude "unchanged" because they match the forged row. - set existing = ##class(%IPM.Storage.FileHash).ModulePathIndexOpen("sync-test", "src/cls/SyncTest/SuperClass.cls") - set existing.FileTimestamp = ##class(%IPM.Storage.FileHash).GetFileTimestamp(filePath) - set existing.FileSize = ##class(%File).GetFileSize(filePath) - // Leave existing.Hash as the old value intentionally - $$$ThrowOnError(existing.%Save()) - - kill params - set params("Verbose") = 1 - do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) - do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) - - do $$$AssertStatusOK(sc, "Sync detects content change despite matching mtime/size") - do $$$AssertTrue(..FindInOutput(.output, "Sync complete"), "Content change was detected and applied (not 'Nothing to sync')") -} - /// Confirm sync detects changes in a module whose file layout doesn't match the /// SourcesRoot/Directory/package.ext convention. sync-flat-test declares SyncFlat.Flat.CLS /// with SourcesRoot=src, but the file is at src/SyncFlat/Flat.cls (no cls/ subdirectory). From 8b0969af6214b1071b0c9c5ac77a5b0ff709f536 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Mon, 20 Jul 2026 13:07:52 -0400 Subject: [PATCH 15/39] Refactor --- src/cls/IPM/ResourceProcessor/FileCopy.cls | 23 +-- src/cls/IPM/ResourceProcessor/Test.cls | 51 +----- src/cls/IPM/Storage/FileHash.cls | 170 ++++++++++-------- src/cls/IPM/Storage/Module.cls | 144 ++++++--------- .../Test/PM/Integration/Sync.cls | 38 ++-- 5 files changed, 184 insertions(+), 242 deletions(-) diff --git a/src/cls/IPM/ResourceProcessor/FileCopy.cls b/src/cls/IPM/ResourceProcessor/FileCopy.cls index 546dd89cf..fb8191db2 100644 --- a/src/cls/IPM/ResourceProcessor/FileCopy.cls +++ b/src/cls/IPM/ResourceProcessor/FileCopy.cls @@ -196,22 +196,13 @@ Method OnSyncResolveFiles(Output relPaths) As %Status quit } - set rs = ##class(%SQL.Statement).%ExecDirect(, - "SELECT Name, Type FROM %Library.File_FileSet(?, ?, ?, ?)", - sourceDir, "*", "", 1) - if rs.%SQLCODE < 0 { - $$$ThrowStatus($$$ERROR($$$GeneralError, "FileSet query error: " _ rs.%Message)) - } - while rs.%Next() { - if rs.%Get("Type") = "D" { - continue - } - set fullPath = rs.%Get("Name") - // Compute path relative to module root - set relPath = $extract(fullPath, $length(moduleRoot) + 1, *) - if relPath '= "" { - set relPaths(relPath) = "" - } + kill walkFiles + $$$ThrowOnError(##class(%IPM.Storage.FileHash).WalkFilesRecursive(sourceDir, moduleRoot, .walkFiles)) + set relPath = "" + for { + set relPath = $order(walkFiles(relPath)) + quit:relPath="" + set relPaths(relPath) = "" } } catch e { set sc = e.AsStatus() diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls index 5a3ef7d1d..1e64d5265 100644 --- a/src/cls/IPM/ResourceProcessor/Test.cls +++ b/src/cls/IPM/ResourceProcessor/Test.cls @@ -394,50 +394,13 @@ Method OnSyncResolveFiles(Output relPaths) As %Status if '##class(%File).DirectoryExists(unitTestDir) { quit } - $$$ThrowOnError(..WalkClsFilesRecursive(unitTestDir, moduleRoot, .relPaths)) - } catch e { - set sc = e.AsStatus() - } - quit sc -} - -/// Recursively collects .cls files under pDir into relPaths(relPath), relative to pModuleRoot. -/// %Library.File_FileSet's own recursive flag only returns immediate children per call -/// (confirmed experimentally — it does not descend through multiple directory levels in one -/// call), so each subdirectory must be visited with its own call, same as %UnitTest.Manager's -/// own GetSubDirectories does internally. -ClassMethod WalkClsFilesRecursive( - pDir As %String, - pModuleRoot As %String, - ByRef relPaths) As %Status -{ - set sc = $$$OK - try { - set rs = ##class(%SQL.Statement).%ExecDirect(, - "SELECT Name, Type FROM %Library.File_FileSet(?, ?, ?, ?)", - pDir, "*.cls", "", 0) - if rs.%SQLCODE < 0 { - $$$ThrowStatus($$$ERROR($$$GeneralError, "FileSet query error: " _ rs.%Message)) - } - while rs.%Next() { - continue:rs.%Get("Type")="D" - set fullPath = rs.%Get("Name") - set relPath = $extract(fullPath, $length(pModuleRoot) + 1, *) - if relPath '= "" { - set relPaths(relPath) = "" - } - } - - set dirRs = ##class(%SQL.Statement).%ExecDirect(, - "SELECT Name, Type FROM %Library.File_FileSet(?, ?, ?, ?)", - pDir, "*", "", 0) - if dirRs.%SQLCODE < 0 { - $$$ThrowStatus($$$ERROR($$$GeneralError, "FileSet query error: " _ dirRs.%Message)) - } - while dirRs.%Next() { - continue:dirRs.%Get("Type")'="D" - set subDir = ##class(%File).NormalizeDirectory(dirRs.%Get("Name")) - $$$ThrowOnError(..WalkClsFilesRecursive(subDir, pModuleRoot, .relPaths)) + kill walkFiles + $$$ThrowOnError(##class(%IPM.Storage.FileHash).WalkFilesRecursive(unitTestDir, moduleRoot, .walkFiles, "cls")) + set relPath = "" + for { + set relPath = $order(walkFiles(relPath)) + quit:relPath="" + set relPaths(relPath) = "" } } catch e { set sc = e.AsStatus() diff --git a/src/cls/IPM/Storage/FileHash.cls b/src/cls/IPM/Storage/FileHash.cls index f6a2e4f29..5c76eb73a 100644 --- a/src/cls/IPM/Storage/FileHash.cls +++ b/src/cls/IPM/Storage/FileHash.cls @@ -35,47 +35,29 @@ ClassMethod StampModule(module As %IPM.Storage.Module) As %Status } // Pass 1: BFS the module root for compilable files present in the namespace. - // Uses a manual BFS queue because File_FileSet's recursive flag only returns - // immediate children — it does not descend through multiple directory levels in one call. - kill walkQueue - set walkHead = 1, walkTail = 1 - set walkQueue(walkTail) = root + kill bfsFiles + $$$ThrowOnError(..WalkFilesRecursive(root, root, .bfsFiles, "cls,inc,mac,int")) + set relPath = "" for { - quit:(walkHead > walkTail) - set walkDir = walkQueue(walkHead) - set walkHead = walkHead + 1 - - set walkResult = ##class(%SQL.Statement).%ExecDirect(, - "SELECT Name, Type FROM %Library.File_FileSet(?, ?, ?, ?)", - walkDir, "*", "", 0) - while walkResult.%Next() { - set entryType = walkResult.%Get("Type") - set entryPath = walkResult.%Get("Name") - if entryType = "D" { - set walkTail = walkTail + 1 - set walkQueue(walkTail) = ##class(%File).NormalizeDirectory(entryPath) - continue - } - set ext = $$$lcase($piece(entryPath, ".", *)) - if ",cls,inc,mac,int," '[ (","_ext_",") { + set relPath = $order(bfsFiles(relPath), 1, entryPath) + quit:relPath="" + set docName = ..RelPathToDocName(relPath) + if docName = "" { + continue + } + set docExt = $$$lcase($piece(docName, ".", *)) + if docExt = "cls" { + if '$$$comClassDefined($piece(docName, ".", 1, *-1)) { continue } - set normalizedRelPath = ..NormalizePath($extract(entryPath, $length(root) + 1, *)) - set docName = ..RelPathToDocName(normalizedRelPath) - if docName = "" { + } else { + if '##class(%RoutineMgr).Exists(docName) { continue } - set docExt = $$$lcase($piece(docName, ".", *)) - if docExt = "cls" { - if '$$$comClassDefined($piece(docName, ".", 1, *-1)) { - continue - } - } else { - if '##class(%RoutineMgr).Exists(docName) { - continue - } - } - $$$ThrowOnError(..StampOneFile(module.Name, root, entryPath)) + } + set stampSC = ..StampOneFile(module.Name, root, entryPath) + if $$$ISERR(stampSC) { + write !, "Warning: could not stamp ", relPath, ": ", $system.Status.GetOneErrorText(stampSC) } } @@ -101,7 +83,10 @@ ClassMethod StampModule(module As %IPM.Storage.Module) As %Status if '##class(%File).Exists(fullPath) { continue } - $$$ThrowOnError(..StampOneFile(module.Name, root, fullPath)) + set stampSC = ..StampOneFile(module.Name, root, fullPath) + if $$$ISERR(stampSC) { + write !, "Warning: could not stamp ", relPath, ": ", $system.Status.GetOneErrorText(stampSC) + } } } } catch e { @@ -111,9 +96,14 @@ ClassMethod StampModule(module As %IPM.Storage.Module) As %Status } /// Stamp (or update) the FileHash row for a single file. Computes hash at stamp time. +/// Returns an error if the file cannot be hashed (locked/permission denied). ClassMethod StampOneFile(moduleName As %String, root As %String, fullPath As %String) As %Status { set normalizedRelPath = ..NormalizePath($extract(fullPath, $length(root) + 1, *)) + set hash = ##class(%File).SHA1Hash(fullPath, 1) + if hash = "" { + quit $$$ERROR($$$GeneralError, "Could not compute hash for: " _ fullPath) + } set existing = ..ModulePathIndexOpen(moduleName, normalizedRelPath, , .openSC) if $isobject(existing) { set instance = existing @@ -122,7 +112,7 @@ ClassMethod StampOneFile(moduleName As %String, root As %String, fullPath As %St set instance.ModuleName = moduleName set instance.RelativePath = normalizedRelPath } - set instance.Hash = ##class(%File).SHA1Hash(fullPath, 1) + set instance.Hash = hash quit instance.%Save() } @@ -182,45 +172,27 @@ ClassMethod ComputeChanges(module As %IPM.Storage.Module, Output modified, Outpu } // Pass 1: BFS module root for compilable files present in the namespace. - kill walkQueue - set walkHead = 1, walkTail = 1 - set walkQueue(walkTail) = root + kill bfsFiles + $$$ThrowOnError(..WalkFilesRecursive(root, root, .bfsFiles, "cls,inc,mac,int")) + set relPath = "" for { - quit:(walkHead > walkTail) - set walkDir = walkQueue(walkHead) - set walkHead = walkHead + 1 - - set walkResult = ##class(%SQL.Statement).%ExecDirect(, - "SELECT Name, Type FROM %Library.File_FileSet(?, ?, ?, ?)", - walkDir, "*", "", 0) - while walkResult.%Next() { - set entryType = walkResult.%Get("Type") - set entryPath = walkResult.%Get("Name") - if entryType = "D" { - set walkTail = walkTail + 1 - set walkQueue(walkTail) = ##class(%File).NormalizeDirectory(entryPath) - continue - } - set ext = $$$lcase($piece(entryPath, ".", *)) - if ",cls,inc,mac,int," '[ (","_ext_",") { + set relPath = $order(bfsFiles(relPath), 1, entryPath) + quit:relPath="" + set docName = ..RelPathToDocName(relPath) + if docName = "" { + continue + } + set docExt = $$$lcase($piece(docName, ".", *)) + if docExt = "cls" { + if '$$$comClassDefined($piece(docName, ".", 1, *-1)) { continue } - set docName = ..RelPathToDocName(..NormalizePath($extract(entryPath, $length(root) + 1, *))) - if docName = "" { + } else { + if '##class(%RoutineMgr).Exists(docName) { continue } - set docExt = $$$lcase($piece(docName, ".", *)) - if docExt = "cls" { - if '$$$comClassDefined($piece(docName, ".", 1, *-1)) { - continue - } - } else { - if '##class(%RoutineMgr).Exists(docName) { - continue - } - } - do ..CheckOneFile(module.Name, root, entryPath, .modified, .seen) } + do ..CheckOneFile(module.Name, root, entryPath, .modified, .seen) } // Pass 2: manifest-derived paths for files not yet compiled (e.g. newly-declared @@ -293,6 +265,7 @@ ClassMethod ComputeChanges(module As %IPM.Storage.Module, Output modified, Outpu } /// Check a single file against its stored baseline. If no row or hash differs, add to modified. +/// If hash cannot be computed (locked/permission denied), reports as modified with "" hash and warns. ClassMethod CheckOneFile(moduleName As %String, root As %String, fullPath As %String, ByRef modified, ByRef seen) [ Private ] { set normalizedRelPath = ..NormalizePath($extract(fullPath, $length(root) + 1, *)) @@ -300,12 +273,17 @@ ClassMethod CheckOneFile(moduleName As %String, root As %String, fullPath As %St quit } set seen(normalizedRelPath) = "" + set newHash = ##class(%File).SHA1Hash(fullPath, 1) + if newHash = "" { + write !, "Warning: could not hash ", normalizedRelPath, " (file may be locked)" + set modified(normalizedRelPath) = "" + quit + } set existing = ..ModulePathIndexOpen(moduleName, normalizedRelPath, , .openSC) if '$isobject(existing) { - set modified(normalizedRelPath) = ##class(%File).SHA1Hash(fullPath, 1) + set modified(normalizedRelPath) = newHash quit } - set newHash = ##class(%File).SHA1Hash(fullPath, 1) if existing.Hash '= newHash { set modified(normalizedRelPath) = newHash } @@ -389,6 +367,52 @@ ClassMethod NormalizePath(path As %String) As %String quit path } +/// BFS walk all files under dir, returning paths relative to relativeToRoot. +/// extensionFilter is comma-separated (e.g. "cls,inc,mac,int") or "" for all files. +/// Output: files(normalizedRelPath) = fullPath. +ClassMethod WalkFilesRecursive(dir As %String, relativeToRoot As %String, Output files, extensionFilter As %String = "") As %Status +{ + set sc = $$$OK + try { + set dir = ##class(%File).NormalizeDirectory(dir) + set relativeToRoot = ##class(%File).NormalizeDirectory(relativeToRoot) + kill walkQueue + set walkHead = 1, walkTail = 1 + set walkQueue(walkTail) = dir + for { + quit:(walkHead > walkTail) + set walkDir = walkQueue(walkHead) + set walkHead = walkHead + 1 + + set rs = ##class(%SQL.Statement).%ExecDirect(, + "SELECT Name, Type FROM %Library.File_FileSet(?, ?, ?, ?)", + walkDir, "*", "", 0) + while rs.%Next() { + set entryType = rs.%Get("Type") + set entryPath = rs.%Get("Name") + if entryType = "D" { + set walkTail = walkTail + 1 + set walkQueue(walkTail) = ##class(%File).NormalizeDirectory(entryPath) + continue + } + if extensionFilter '= "" { + set ext = $$$lcase($piece(entryPath, ".", *)) + if (","_extensionFilter_",") '[ (","_ext_",") { + continue + } + } + set relPath = ..NormalizePath($extract(entryPath, $length(relativeToRoot) + 1, *)) + if relPath '= "" { + set files(relPath) = entryPath + } + } + } + } catch e { + set sc = e.AsStatus() + } + quit sc +} + Storage Default { diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls index 019118c54..4e56dbb31 100644 --- a/src/cls/IPM/Storage/Module.cls +++ b/src/cls/IPM/Storage/Module.cls @@ -753,7 +753,6 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status set moduleXmlRelPath = ##class(%IPM.Storage.FileHash).NormalizePath("module.xml") set moduleXmlPath = root _ "module.xml" set moduleXmlChanged = ..SyncCheckModuleXml(.module, moduleXmlPath, moduleXmlRelPath) - set lifecycle = module.Lifecycle if '##class(%IPM.Storage.FileHash).HasBaseline(moduleName) { // Self-heal: establish baseline for modules loaded before this feature @@ -786,13 +785,16 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status } // Step 3: Build reverse index (relPath -> owning resource + processor) - kill reverseIndex - do ..SyncBuildReverseIndex(module, .reverseIndex) set orderedResourceList = module.GetOrderedResourceList() + kill reverseIndex + do ..SyncBuildReverseIndex(module, orderedResourceList, .reverseIndex) // Step 4: Partition changes by resource, separating unsupported processors kill syncByResource, unsupportedWarnings - do ..SyncRouteChanges(.modified, .deleted, .reverseIndex, processDeletes, .syncByResource, .unsupportedWarnings) + do ..SyncRoutePathSet(.modified, .reverseIndex, "modified", .syncByResource, .unsupportedWarnings) + if processDeletes { + do ..SyncRoutePathSet(.deleted, .reverseIndex, "deleted", .syncByResource, .unsupportedWarnings) + } // Changes routed to a processor that doesn't support sync were never applied. // Drop them from modified/deleted so CommitChanges (Step 8) doesn't advance their @@ -804,13 +806,13 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status // Step 6: Compile the full resource set with u-flag to pick up dependent recompiles if loadItems > 0 { - $$$ThrowOnError(..SyncCompile(module, verbose, .params)) + $$$ThrowOnError(..SyncCompile(module, orderedResourceList, verbose, .params)) } // Step 7: Delete server-side documents for removed files, then recompile if processDeletes && ($data(deleted) > 1) { do ..SyncApplyDeletes(.deleted, .reverseIndex, .syncByResource, verbose) - $$$ThrowOnError(..SyncCompile(module, verbose, .params)) + $$$ThrowOnError(..SyncCompile(module, orderedResourceList, verbose, .params)) } // Step 8: Commit new hashes on success (skipped on error so next sync re-detects). @@ -888,11 +890,19 @@ ClassMethod SyncCheckModuleXml( } /// Build a reverse index: normalizedRelPath -> resource name, Processor, Resource object. -/// Used by SyncRouteChanges to map changed files back to their owning resource processors. -ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, ByRef reverseIndex) +/// Used by SyncRoutePathSet to map changed files back to their owning resource processors. +/// +/// Step 1 builds docToResource (docName → owner) from ResolveChildren and maps +/// OnSyncResolveFiles paths directly into reverseIndex (non-compilable resources). +/// Step 2 maps all stored baseline paths to owners via RelPathToDocName → docToResource. +/// This is sufficient because every path in modified/deleted has a stored baseline row +/// (guaranteed by the self-heal stamping on first sync). +ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, orderedResourceList As %ListOfObjects, ByRef reverseIndex) { - // Pass 1: derive paths from resource metadata (covers standard layouts) - set orderedResourceList = module.GetOrderedResourceList() + // docToResource maps server document names (e.g. "SyncTest.SuperClass.CLS") to the + // resource that owns them. This lets us resolve filesystem paths → owners in O(1) below, + // since RelPathToDocName converts a relPath to a docName deterministically. + kill docToResource set key = "" for { set resource = orderedResourceList.GetNext(.key) @@ -902,6 +912,7 @@ ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, ByRef reverseIn continue } + // ResolveChildren returns childArr keyed by document name (server-side identifier). kill childArr set childSC = resource.ResolveChildren(.childArr) if $$$ISERR(childSC) { @@ -912,21 +923,27 @@ ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, ByRef reverseIn for { set childName = $order(childArr(childName)) quit:childName="" + set docToResource(childName) = resource.Name + set docToResource(childName, "Processor") = resource.Processor + set docToResource(childName, "Resource") = resource + // Newly-declared resources have no baseline row yet (StampModule skips uncompiled + // classes), so GetStoredPaths below won't find them. Map their relPath directly. set relPath = $get(childArr(childName, "RelativePath")) if relPath = "" { set relPath = resource.Processor.OnItemRelativePath(childName) } - if relPath = "" { - continue + if relPath '= "" { + set normalizedRelPath = ##class(%IPM.Storage.FileHash).NormalizePath(relPath) + set reverseIndex(normalizedRelPath) = resource.Name + set reverseIndex(normalizedRelPath, "Processor") = resource.Processor + set reverseIndex(normalizedRelPath, "Resource") = resource } - - set normalizedRelPath = ##class(%IPM.Storage.FileHash).NormalizePath(relPath) - set reverseIndex(normalizedRelPath) = resource.Name - set reverseIndex(normalizedRelPath, "Processor") = resource.Processor - set reverseIndex(normalizedRelPath, "Resource") = resource } + // OnSyncResolveFiles returns filesystem-relative paths for non-compilable resources + // (FileCopy directories, test directories). These have no document name, so they go + // directly into reverseIndex keyed by relPath. kill syncOnlyPaths set childSC = resource.Processor.OnSyncResolveFiles(.syncOnlyPaths) if $$$ISERR(childSC) { @@ -936,7 +953,6 @@ ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, ByRef reverseIn for { set relPath = $order(syncOnlyPaths(relPath)) quit:relPath="" - set normalizedRelPath = ##class(%IPM.Storage.FileHash).NormalizePath(relPath) set reverseIndex(normalizedRelPath) = resource.Name set reverseIndex(normalizedRelPath, "Processor") = resource.Processor @@ -944,8 +960,10 @@ ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, ByRef reverseIn } } - // Pass 2: for stored paths not covered by Pass 1 (non-standard layouts), - // derive the document name, find the owning resource, and add to index. + // Resolve compilable paths: every baseline path not already claimed by OnSyncResolveFiles + // above gets mapped through RelPathToDocName → docToResource. This handles both standard + // and non-standard directory layouts uniformly (the path on disk doesn't matter — only + // the derived document name needs to match what ResolveChildren reported). kill storedPaths do ##class(%IPM.Storage.FileHash).GetStoredPaths(module.Name, .storedPaths) set relPath = "" @@ -961,43 +979,13 @@ ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, ByRef reverseIn if docName = "" { continue } - - set key = "" - for { - set resource = orderedResourceList.GetNext(.key) - quit:key="" - - if '$isobject(resource.Processor) { - continue - } - - kill childArr - set childSC = resource.ResolveChildren(.childArr) - if $$$ISERR(childSC) || '$data(childArr(docName)) { - continue - } - - set reverseIndex(relPath) = resource.Name - set reverseIndex(relPath, "Processor") = resource.Processor - set reverseIndex(relPath, "Resource") = resource - quit + if '$data(docToResource(docName)) { + continue } - } -} -/// Partition modified and deleted paths into syncByResource (keyed by resource name) and -/// unsupportedWarnings (for processors that don't support sync). -ClassMethod SyncRouteChanges( - ByRef modified, - ByRef deleted, - ByRef reverseIndex, - processDeletes As %Boolean, - ByRef syncByResource, - ByRef unsupportedWarnings) -{ - do ..SyncRoutePathSet(.modified, .reverseIndex, "modified", .syncByResource, .unsupportedWarnings) - if processDeletes { - do ..SyncRoutePathSet(.deleted, .reverseIndex, "deleted", .syncByResource, .unsupportedWarnings) + set reverseIndex(relPath) = docToResource(docName) + set reverseIndex(relPath, "Processor") = docToResource(docName, "Processor") + set reverseIndex(relPath, "Resource") = docToResource(docName, "Resource") } } @@ -1053,6 +1041,8 @@ ClassMethod SyncRoutePathSet( /// Call OnSync on each processor, then load any unhandled compilable files. /// Returns the number of files loaded (used to decide whether SyncCompile is needed). +/// If a processor sets handled=1 in OnSync, it fully owns the sync for that resource +/// (e.g. Test reloads its own way). Otherwise the default path loads+compiles each file. ClassMethod SyncDispatchProcessors( module As %IPM.Storage.Module, root As %String, @@ -1105,6 +1095,8 @@ ClassMethod SyncDispatchProcessors( } /// Delete server-side documents for compilable deleted files. +/// Guards skip paths with no owner, no processor, or non-compilable processors — those are +/// handled elsewhere (unsupported warnings) or aren't server-side documents at all. ClassMethod SyncApplyDeletes( ByRef deleted, ByRef reverseIndex, @@ -1127,7 +1119,7 @@ ClassMethod SyncApplyDeletes( if 'processor.%IsA("%IPM.ResourceProcessor.AbstractCompilable") { continue } - set docName = ..RelPathToDocName(relPath) + set docName = ##class(%IPM.Storage.FileHash).RelPathToDocName(relPath) if docName '= "" { set delFlags = $select(verbose:"d", 1:"-d") set delSC = $system.OBJ.Delete(docName, delFlags) @@ -1235,12 +1227,14 @@ ClassMethod SyncPrintUnsupportedWarnings(moduleName As %String, ByRef unsupporte } } -/// Compile all compilable resources in a module with the 'u' (skip-up-to-date) flag. -ClassMethod SyncCompile(module As %IPM.Storage.Module, verbose As %Boolean = 0, ByRef params) As %Status +/// Recompile all compilable resources in the module to catch dependents invalidated by +/// changes loaded in SyncDispatchProcessors. Skips CompileFromProject resources — those are +/// deployed code loaded via Studio project files (.prj); compiling them individually would +/// fail because deployed classes have no source in the routine database. +ClassMethod SyncCompile(module As %IPM.Storage.Module, orderedResourceList As %ListOfObjects, verbose As %Boolean = 0, ByRef params) As %Status { set sc = $$$OK try { - set orderedResourceList = module.GetOrderedResourceList() kill compileArray set key = "" @@ -1293,38 +1287,6 @@ ClassMethod SyncCompile(module As %IPM.Storage.Module, verbose As %Boolean = 0, quit sc } -/// Convert a normalized relative path to an IRIS server document name. -ClassMethod RelPathToDocName(relPath As %String) As %String -{ - set fileName = $piece(relPath, "/", *) - set ext = $zconvert($piece(fileName, ".", *), "U") - set baseName = $piece(fileName, ".", 1, *-1) - - if ext = "CLS" { - // Convert path like src/cls/My/Package/Class.cls -> My.Package.Class.cls - set parts = $length(relPath, "/") - set className = "" - // Skip leading source prefixes (e.g. "src/cls/") to get class package path - set startPiece = 1 - for i = 1:1:parts-1 { - set piece = $piece(relPath, "/", i) - if $listfind($listbuild("src", "cls"), $zconvert(piece, "L")) { - set startPiece = i + 1 - } else { - quit - } - } - for i = startPiece:1:parts-1 { - set className = className _ $select(className="":"", 1:".") _ $piece(relPath, "/", i) - } - set className = className _ $select(className="":"", 1:".") _ baseName - quit className _ ".cls" - } elseif (ext = "MAC") || (ext = "INC") || (ext = "INT") { - quit baseName _ "." _ $zconvert(ext, "L") - } - quit "" -} - /// Uninstalls a named module (pModuleName). /// May optionally force installation (uninstalling even if required by other modules) if pForce is 1. /// May optionally recurse to also uninstall dependencies that are not required by other modules if pRecurse is 1. diff --git a/tests/integration_tests/Test/PM/Integration/Sync.cls b/tests/integration_tests/Test/PM/Integration/Sync.cls index e8a18411b..c5f78bb06 100644 --- a/tests/integration_tests/Test/PM/Integration/Sync.cls +++ b/tests/integration_tests/Test/PM/Integration/Sync.cls @@ -74,31 +74,33 @@ Method TestGetStoredPathsReturnsStampedPaths() /// filesystem-anchored stamping, the file never gets a baseline row and sync always reports /// "Nothing to sync" regardless of what changed. // This test manages its own install/uninstall because sync-flat-test uses a separate fixture -// from sync-test and must not interfere with the per-test lifecycle. If the test throws before -// cleanup, sync-flat-test and flatTempDir are leaked (no safety net from OnAfterOneTest). +// from sync-test and must not interfere with the per-test lifecycle. Method TestNonStandardLayoutDetectsChange() { - set flatSource = ..GetModuleDir("sync-flat-test") set flatTempDir = ##class(%File).NormalizeDirectory(##class(%File).ManagerDirectory() _ "sync-flat-test-" _ $job) - $$$ThrowOnError(##class(%IPM.Utils.File).CreateDirectoryChain(flatTempDir)) - if '##class(%Library.File).CopyDir(flatSource, flatTempDir, 1) { - $$$ThrowOnError($$$ERROR($$$GeneralError, "Failed to copy sync-flat-test to temp dir")) - } - - $$$ThrowOnError(##class(%IPM.Main).Shell("load " _ flatTempDir _ " -dev")) + try { + set flatSource = ..GetModuleDir("sync-flat-test") + $$$ThrowOnError(##class(%IPM.Utils.File).CreateDirectoryChain(flatTempDir)) + if '##class(%Library.File).CopyDir(flatSource, flatTempDir, 1) { + $$$ThrowOnError($$$ERROR($$$GeneralError, "Failed to copy sync-flat-test to temp dir")) + } - set filePath = flatTempDir _ "src/SyncFlat/Flat.cls" - do ..ReplaceInFile(filePath, "As %String", "As %Integer") + $$$ThrowOnError(##class(%IPM.Main).Shell("load " _ flatTempDir _ " -dev")) - kill params - set params("Verbose") = 1 - do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Storage.Module).Sync("sync-flat-test", .params) - do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + set filePath = flatTempDir _ "src/SyncFlat/Flat.cls" + do ..ReplaceInFile(filePath, "As %String", "As %Integer") - do $$$AssertStatusOK(sc, "Sync succeeds for non-standard layout module") - do $$$AssertTrue(..FindInOutput(.output, "Sync complete"), "Sync detects the file change (not 'Nothing to sync')") + kill params + set params("Verbose") = 1 + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Storage.Module).Sync("sync-flat-test", .params) + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + do $$$AssertStatusOK(sc, "Sync succeeds for non-standard layout module") + do $$$AssertTrue(..FindInOutput(.output, "Sync complete"), "Sync detects the file change (not 'Nothing to sync')") + } catch e { + do $$$AssertStatusOK(e.AsStatus(), "TestNonStandardLayoutDetectsChange threw unexpectedly") + } do ##class(%IPM.Main).Shell("uninstall sync-flat-test") do ##class(%Library.File).RemoveDirectoryTree(flatTempDir) } From fd4eb65f4e67817aa8f993ee8c81d602e6e3d85e Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Mon, 20 Jul 2026 13:55:03 -0400 Subject: [PATCH 16/39] Refactor to improve performance, reduce false positive warnings, and flesh out verbose mode output --- src/cls/IPM/Storage/FileHash.cls | 65 ++------ src/cls/IPM/Storage/Module.cls | 143 ++++++++++-------- .../Test/PM/Integration/Sync.cls | 67 -------- 3 files changed, 95 insertions(+), 180 deletions(-) diff --git a/src/cls/IPM/Storage/FileHash.cls b/src/cls/IPM/Storage/FileHash.cls index 5c76eb73a..e40c56c67 100644 --- a/src/cls/IPM/Storage/FileHash.cls +++ b/src/cls/IPM/Storage/FileHash.cls @@ -154,10 +154,13 @@ ClassMethod RelPathToDocName(relPath As %String) As %String quit name _ "." _ $$$UPPER(ext) } -/// Compute which files changed on disk vs stored baseline. Self-contained: discovers files -/// via BFS (compilable) + OnSyncResolveFiles (non-compilable), then iterates stored rows for deletions. +/// Compute which files changed on disk vs stored baseline. +/// Pass 1: BFS for compilable files present in the namespace. +/// Pass 2: check manifest-derived paths (from reverseIndex) for files not yet compiled +/// or non-compilable tracked files — avoids re-calling ResolveChildren/OnSyncResolveFiles. +/// Pass 3: iterate stored rows to detect deletions. /// Returns modified(relPath)=newHash and deleted(relPath)="" arrays. -ClassMethod ComputeChanges(module As %IPM.Storage.Module, Output modified, Output deleted) As %Status +ClassMethod ComputeChanges(module As %IPM.Storage.Module, Output modified, Output deleted, ByRef manifestPaths) As %Status { set sc = $$$OK kill modified, deleted @@ -195,56 +198,18 @@ ClassMethod ComputeChanges(module As %IPM.Storage.Module, Output modified, Outpu do ..CheckOneFile(module.Name, root, entryPath, .modified, .seen) } - // Pass 2: manifest-derived paths for files not yet compiled (e.g. newly-declared - // resource after manifest reload). BFS skips these because they fail the namespace - // filter, but they exist on disk and should be detected as new. - set orderedResourceList = module.GetOrderedResourceList() - set key = "" + // Pass 2: check manifest-derived paths for files BFS missed (uncompiled new files, + // non-compilable tracked files). These come from reverseIndex which already called + // ResolveChildren + OnSyncResolveFiles, avoiding redundant iteration here. + set relPath = "" for { - set resource = orderedResourceList.GetNext(.key) - quit:key="" - if '$isobject(resource.Processor) { - continue - } - kill childArr - set childSC = resource.ResolveChildren(.childArr) - if $$$ISERR(childSC) { - continue - } - set childName = "" - for { - set childName = $order(childArr(childName)) - quit:childName="" - set relPath = $get(childArr(childName, "RelativePath")) - if relPath = "" { - set relPath = resource.Processor.OnItemRelativePath(childName) - } - if relPath = "" { - continue - } - set fullPath = ##class(%File).NormalizeFilename(relPath, root) - if '##class(%File).Exists(fullPath) { - continue - } - do ..CheckOneFile(module.Name, root, fullPath, .modified, .seen) - } - - // Also check OnSyncResolveFiles for non-compilable tracked files. - kill syncOnlyPaths - set childSC = resource.Processor.OnSyncResolveFiles(.syncOnlyPaths) - if $$$ISERR(childSC) { + set relPath = $order(manifestPaths(relPath)) + quit:relPath="" + set fullPath = ##class(%File).NormalizeFilename(relPath, root) + if '##class(%File).Exists(fullPath) { continue } - set relPath = "" - for { - set relPath = $order(syncOnlyPaths(relPath)) - quit:relPath="" - set fullPath = ##class(%File).NormalizeFilename(relPath, root) - if '##class(%File).Exists(fullPath) { - continue - } - do ..CheckOneFile(module.Name, root, fullPath, .modified, .seen) - } + do ..CheckOneFile(module.Name, root, fullPath, .modified, .seen) } // Pass 3: iterate all stored rows — file missing from disk → deleted. diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls index 4e56dbb31..ae52d718d 100644 --- a/src/cls/IPM/Storage/Module.cls +++ b/src/cls/IPM/Storage/Module.cls @@ -725,6 +725,7 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status { set sc = $$$OK try { + set syncStart = $zhorolog set verbose = $get(params("Verbose"), 0) set processDeletes = $get(params("ProcessDeletes"), 0) set runTests = $get(params("RunTests"), 0) @@ -767,8 +768,34 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status $$$ThrowOnError(##class(%IPM.Storage.FileHash).StampModule(module)) } - // Step 2: Compute disk changes vs baseline (self-contained — BFS + stored rows) - $$$ThrowOnError(##class(%IPM.Storage.FileHash).ComputeChanges(module, .modified, .deleted)) + // Step 2: Build reverse index (relPath -> owning resource + processor). + // Done before ComputeChanges so we can pass manifest-derived paths to it, + // avoiding a redundant ResolveChildren + OnSyncResolveFiles iteration. + set orderedResourceList = module.GetOrderedResourceList() + kill reverseIndex, unsupportedResources + do ..SyncBuildReverseIndex(module, orderedResourceList, .reverseIndex, .unsupportedResources) + + // Collect manifest-derived paths from reverseIndex for ComputeChanges. + // These supplement the BFS walk (which only finds compiled files). + kill manifestPaths + set riCount = 0 + set riKey = "" + for { + set riKey = $order(reverseIndex(riKey)) + quit:riKey="" + set manifestPaths(riKey) = "" + set riCount = riCount + 1 + } + if verbose { + write !, "[", moduleName, "] Reverse index: ", riCount, " tracked path(s)" + } + + // Step 3: Compute disk changes vs baseline + set scanStart = $zhorolog + $$$ThrowOnError(##class(%IPM.Storage.FileHash).ComputeChanges(module, .modified, .deleted, .manifestPaths)) + if verbose { + write !, "[", moduleName, "] Change detection: ", $fnumber($zhorolog - scanStart, "", 2), "s" + } // Exclude module.xml from file routing (handled by SyncCheckModuleXml above) kill modified(moduleXmlRelPath) @@ -781,27 +808,31 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status do ..SyncCommitModuleXml(module, moduleXmlPath, moduleXmlRelPath) do ..SyncPrintModuleXmlWarning(moduleName) } + if verbose && $data(unsupportedResources) { + do ..SyncPrintUnsupportedNote(moduleName, .unsupportedResources) + } + write !, "[", moduleName, "] Done in ", $fnumber($zhorolog - syncStart, "", 2), "s" quit } - // Step 3: Build reverse index (relPath -> owning resource + processor) - set orderedResourceList = module.GetOrderedResourceList() - kill reverseIndex - do ..SyncBuildReverseIndex(module, orderedResourceList, .reverseIndex) - - // Step 4: Partition changes by resource, separating unsupported processors - kill syncByResource, unsupportedWarnings - do ..SyncRoutePathSet(.modified, .reverseIndex, "modified", .syncByResource, .unsupportedWarnings) + // Step 4: Partition changes by owning resource + kill syncByResource + do ..SyncRoutePathSet(.modified, .reverseIndex, "modified", .syncByResource) if processDeletes { - do ..SyncRoutePathSet(.deleted, .reverseIndex, "deleted", .syncByResource, .unsupportedWarnings) + do ..SyncRoutePathSet(.deleted, .reverseIndex, "deleted", .syncByResource) } - // Changes routed to a processor that doesn't support sync were never applied. - // Drop them from modified/deleted so CommitChanges (Step 8) doesn't advance their - // baseline — otherwise the next sync would see "no change" despite the pending edit. - do ..SyncExcludeUnsupported(.unsupportedWarnings, .modified, .deleted) - // Step 5: Dispatch OnSync to each processor; load unhandled compilable files + if verbose { + set resCount = 0 + set resName = "" + for { + set resName = $order(syncByResource(resName)) + quit:resName="" + set resCount = resCount + 1 + } + write !, "[", moduleName, "] Dispatching to ", resCount, " resource(s)" + } $$$ThrowOnError(..SyncDispatchProcessors(module, root, verbose, .syncByResource, .params, .loadItems)) // Step 6: Compile the full resource set with u-flag to pick up dependent recompiles @@ -850,7 +881,11 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status if moduleXmlChanged { do ..SyncPrintModuleXmlWarning(moduleName) } - do ..SyncPrintUnsupportedWarnings(moduleName, .unsupportedWarnings) + if verbose && $data(unsupportedResources) { + do ..SyncPrintUnsupportedNote(moduleName, .unsupportedResources) + } + + write !, "[", moduleName, "] Done in ", $fnumber($zhorolog - syncStart, "", 2), "s" // Step 9: Run changed test-phase tests if -test flag is set (after sync is reported) if runTests { @@ -891,14 +926,15 @@ ClassMethod SyncCheckModuleXml( /// Build a reverse index: normalizedRelPath -> resource name, Processor, Resource object. /// Used by SyncRoutePathSet to map changed files back to their owning resource processors. +/// Skips resources whose processor does not support sync — those are collected in +/// unsupportedResources(resourceName)="" for informational display. /// /// Step 1 builds docToResource (docName → owner) from ResolveChildren and maps /// OnSyncResolveFiles paths directly into reverseIndex (non-compilable resources). /// Step 2 maps all stored baseline paths to owners via RelPathToDocName → docToResource. -/// This is sufficient because every path in modified/deleted has a stored baseline row -/// (guaranteed by the self-heal stamping on first sync). -ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, orderedResourceList As %ListOfObjects, ByRef reverseIndex) +ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, orderedResourceList As %ListOfObjects, ByRef reverseIndex, Output unsupportedResources) { + kill unsupportedResources // docToResource maps server document names (e.g. "SyncTest.SuperClass.CLS") to the // resource that owns them. This lets us resolve filesystem paths → owners in O(1) below, // since RelPathToDocName converts a relPath to a docName deterministically. @@ -912,6 +948,11 @@ ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, orderedResource continue } + if 'resource.Processor.SupportsSync() { + set unsupportedResources(resource.Name) = "" + continue + } + // ResolveChildren returns childArr keyed by document name (server-side identifier). kill childArr set childSC = resource.ResolveChildren(.childArr) @@ -989,35 +1030,14 @@ ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, orderedResource } } -/// Remove paths routed to a non-sync-supporting processor from modified/deleted, since -/// those changes were never applied and must not advance the FileHash baseline. -ClassMethod SyncExcludeUnsupported( - ByRef unsupportedWarnings, - ByRef modified, - ByRef deleted) -{ - set resName = "" - for { - set resName = $order(unsupportedWarnings(resName)) - quit:resName="" - - set relPath = "" - for { - set relPath = $order(unsupportedWarnings(resName, relPath)) - quit:relPath="" - kill modified(relPath) - kill deleted(relPath) - } - } -} /// Route a set of changed paths to their owning resources, categorized by type (modified/deleted). +/// Only sync-supporting resources appear in reverseIndex, so no filtering needed here. ClassMethod SyncRoutePathSet( ByRef paths, ByRef reverseIndex, category As %String, - ByRef syncByResource, - ByRef unsupportedWarnings) [ Private ] + ByRef syncByResource) [ Private ] { set relPath = "" for { @@ -1028,14 +1048,9 @@ ClassMethod SyncRoutePathSet( continue } set resName = reverseIndex(relPath) - set processor = reverseIndex(relPath, "Processor") - if 'processor.SupportsSync() { - set unsupportedWarnings(resName, relPath) = "" - } else { - set syncByResource(resName, category, relPath) = "" - set syncByResource(resName, "Processor") = processor - set syncByResource(resName, "Resource") = reverseIndex(relPath, "Resource") - } + set syncByResource(resName, category, relPath) = "" + set syncByResource(resName, "Processor") = reverseIndex(relPath, "Processor") + set syncByResource(resName, "Resource") = reverseIndex(relPath, "Resource") } } @@ -1209,22 +1224,24 @@ ClassMethod SyncPrintModuleXmlWarning(moduleName As %String) write !, " manifest-level changes (mappings, dependencies, defaults)." } -ClassMethod SyncPrintUnsupportedWarnings(moduleName As %String, ByRef unsupportedWarnings) +ClassMethod SyncPrintUnsupportedNote(moduleName As %String, ByRef unsupportedResources) { - set warnRes = "" + set count = 0 + set names = "" + set resName = "" for { - set warnRes = $order(unsupportedWarnings(warnRes)) - quit:warnRes="" - - write !, "Warning: resource '", warnRes, "' does not support sync. Changed files:" - set warnPath = "" - for { - set warnPath = $order(unsupportedWarnings(warnRes, warnPath)) - quit:warnPath="" - write !, " ", warnPath + set resName = $order(unsupportedResources(resName)) + quit:resName="" + set count = count + 1 + if count <= 3 { + set names = names _ $select(names="":"", 1:", ") _ resName } - write !, " Run `reload ", moduleName, "` to apply these changes." } + if count > 3 { + set names = names _ ", ... (" _ (count - 3) _ " more)" + } + write !, "[", moduleName, "] ", count, " resource(s) skipped (no sync support): ", names + write !, " Use `reload ", moduleName, "` to apply changes to those resources." } /// Recompile all compilable resources in the module to catch dependents invalidated by diff --git a/tests/integration_tests/Test/PM/Integration/Sync.cls b/tests/integration_tests/Test/PM/Integration/Sync.cls index c5f78bb06..218752cf9 100644 --- a/tests/integration_tests/Test/PM/Integration/Sync.cls +++ b/tests/integration_tests/Test/PM/Integration/Sync.cls @@ -284,22 +284,6 @@ Method TestModuleXmlChangedWarning() do $$$AssertTrue(..FindInOutput(.output, "module.xml changed"), "Warning about module.xml change is shown") } -/// A changed file owned by a resource processor that doesn't support sync produces a -/// warning directing the user to run a full reload. -Method TestNonSyncProcessorWarning() -{ - set filePath = ..TempDir _ "static/config.txt" - do ..ReplaceInFile(filePath, "static file content", "modified static content") - - kill params - set params("Verbose") = 1 - do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) - do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) - - do $$$AssertStatusOK(sc, "Sync with non-sync processor file change succeeds") - do $$$AssertTrue(..FindInOutput(.output, "does not support sync"), "Warning about non-sync resource shown") -} /// Without -test, a changed test-phase class is loaded but not executed. With -test, it is /// executed and its results are shown. @@ -377,57 +361,6 @@ Method TestSyncTestFlagBatchesMultipleChangedClasses() do $$$AssertEquals(testResultsCount, 1, "Both changed classes ran in a single batched Test Results block, not one per class") } -/// A change to a file owned by a processor that doesn't support sync is never applied, so -/// its baseline must stay unchanged. The warning should reappear on every subsequent sync -/// until the change is actually applied (e.g. via a full reload). -Method TestUnsupportedProcessorChangeNotCommitted() -{ - set filePath = ..TempDir _ "static/config.txt" - do ..ReplaceInFile(filePath, "static file content", "modified static content") - - kill params - set params("Verbose") = 1 - do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) - do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) - do $$$AssertStatusOK(sc, "Sync with non-sync processor file change succeeds") - do $$$AssertTrue(..FindInOutput(.output, "does not support sync"), "Warning about non-sync resource shown") - - // Baseline must NOT reflect the on-disk change, since it was never applied. - // If it were committed, running sync again would report "Nothing to sync" and - // silently drop the warning even though config.txt server-side content is still stale. - kill params2 - set params2("Verbose") = 1 - do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie2) - set sc2 = ##class(%IPM.Storage.Module).Sync("sync-test", .params2) - do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie2, .output2) - do $$$AssertStatusOK(sc2, "Second sync succeeds") - do $$$AssertTrue(..FindInOutput(.output2, "does not support sync"), "Warning still shown on next sync (baseline was not falsely committed)") -} - -/// A brand-new file added under a directory-scanned FileCopy resource is recognized as a -/// change on the next sync. FileCopy resources are directory-scanned (OnResolveChildren -/// queries the source directory live), so a new file here is discoverable without any -/// module.xml change — unlike individually-declared classes, which require a -/// manifest entry regardless of sync (see TestModuleXmlAddsResourcePicksUpNewFile). -Method TestNewFileInFileCopyResourceDetected() -{ - set filePath = ..TempDir _ "static/new-file.txt" - set stream = ##class(%Stream.FileCharacter).%New() - $$$ThrowOnError(stream.LinkToFile(filePath)) - $$$ThrowOnError(stream.Write("new static file")) - $$$ThrowOnError(stream.%Save()) - set stream = "" - - kill params - set params("Verbose") = 1 - do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) - do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) - - do $$$AssertStatusOK(sc, "Sync with a new FileCopy file succeeds") - do $$$AssertTrue(..FindInOutput(.output, "does not support sync"), "New file under a non-sync-supporting resource is recognized as a change (warned, not silently ignored)") -} /// With two UnitTest resources present, a changed test class in one resource's directory /// must only be dispatched through that owning resource — not through every Test processor From 180723498d371e822ceb58c3ee207aa36cf3b1b3 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Mon, 20 Jul 2026 14:23:48 -0400 Subject: [PATCH 17/39] Handle sync IPM edge case with % classes --- src/cls/IPM/Storage/FileHash.cls | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/cls/IPM/Storage/FileHash.cls b/src/cls/IPM/Storage/FileHash.cls index e40c56c67..b5799175e 100644 --- a/src/cls/IPM/Storage/FileHash.cls +++ b/src/cls/IPM/Storage/FileHash.cls @@ -47,11 +47,12 @@ ClassMethod StampModule(module As %IPM.Storage.Module) As %Status } set docExt = $$$lcase($piece(docName, ".", *)) if docExt = "cls" { - if '$$$comClassDefined($piece(docName, ".", 1, *-1)) { + set className = $piece(docName, ".", 1, *-1) + if '$$$comClassDefined(className) && '$$$comClassDefined("%" _ className) { continue } } else { - if '##class(%RoutineMgr).Exists(docName) { + if '##class(%RoutineMgr).Exists(docName) && '##class(%RoutineMgr).Exists("%" _ docName) { continue } } @@ -187,11 +188,12 @@ ClassMethod ComputeChanges(module As %IPM.Storage.Module, Output modified, Outpu } set docExt = $$$lcase($piece(docName, ".", *)) if docExt = "cls" { - if '$$$comClassDefined($piece(docName, ".", 1, *-1)) { + set className = $piece(docName, ".", 1, *-1) + if '$$$comClassDefined(className) && '$$$comClassDefined("%" _ className) { continue } } else { - if '##class(%RoutineMgr).Exists(docName) { + if '##class(%RoutineMgr).Exists(docName) && '##class(%RoutineMgr).Exists("%" _ docName) { continue } } From bef916511015b11cdbd952c3d991b616f823d2eb Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Mon, 20 Jul 2026 15:13:02 -0400 Subject: [PATCH 18/39] Even more speedup --- src/cls/IPM/ResourceProcessor/Abstract.cls | 14 ++- src/cls/IPM/ResourceProcessor/FileCopy.cls | 19 ++-- src/cls/IPM/ResourceProcessor/Test.cls | 20 ++-- src/cls/IPM/Storage/FileHash.cls | 107 +++++++++++++++++---- src/cls/IPM/Storage/Module.cls | 50 +++++++--- 5 files changed, 148 insertions(+), 62 deletions(-) diff --git a/src/cls/IPM/ResourceProcessor/Abstract.cls b/src/cls/IPM/ResourceProcessor/Abstract.cls index 9f2bdb420..79a647d92 100644 --- a/src/cls/IPM/ResourceProcessor/Abstract.cls +++ b/src/cls/IPM/ResourceProcessor/Abstract.cls @@ -218,14 +218,12 @@ Method SupportsSync() As %Boolean quit 0 } -/// Called by sync's tracked-path scan (only) to discover files owned by this resource that -/// aren't captured by OnResolveChildren/OnItemRelativePath — e.g. a directory-scanned resource -/// whose file set isn't declared as individual module.xml resources. Output relPaths(relPath)="" -/// relative to the module root. Base returns nothing. Do NOT populate ResolveChildren's shared -/// pResourceArray for this purpose — that array is also consumed by packaging/export -/// (GetResolvedReferences/ExportSingleModule), which expects entries keyed by resource name with -/// owning-module context, not by raw filesystem path. -Method OnSyncResolveFiles(Output relPaths) As %Status +/// Called by sync to discover files owned by this resource that aren't captured by +/// OnResolveChildren/OnItemRelativePath — e.g. a directory-scanned resource whose file set +/// isn't declared as individual module.xml resources. Output relPaths(relPath)="" relative to +/// the module root. allFiles(relPath)=fullPath contains the pre-walked module root — filter +/// from it via $order prefix scan rather than doing filesystem I/O. Base returns nothing. +Method OnSyncResolveFiles(Output relPaths, ByRef allFiles) As %Status { quit $$$OK } diff --git a/src/cls/IPM/ResourceProcessor/FileCopy.cls b/src/cls/IPM/ResourceProcessor/FileCopy.cls index fb8191db2..41216c8a5 100644 --- a/src/cls/IPM/ResourceProcessor/FileCopy.cls +++ b/src/cls/IPM/ResourceProcessor/FileCopy.cls @@ -186,22 +186,27 @@ Method DoCopy( /// Enumerate source files so sync can detect changes to FileCopy resources. /// Populates relPaths(relPath)="" for each file under the source directory, relative to module root. -Method OnSyncResolveFiles(Output relPaths) As %Status +Method OnSyncResolveFiles(Output relPaths, ByRef allFiles) As %Status { set sc = $$$OK try { set sourceDir = ##class(%File).NormalizeDirectory(..GetSource()) set moduleRoot = ##class(%File).NormalizeDirectory(..ResourceReference.Module.Root) - if '##class(%File).DirectoryExists(sourceDir) { + // Source must be under module root to appear in allFiles. + if $$$lcase($extract(sourceDir, 1, $length(moduleRoot))) '= $$$lcase(moduleRoot) { quit } - - kill walkFiles - $$$ThrowOnError(##class(%IPM.Storage.FileHash).WalkFilesRecursive(sourceDir, moduleRoot, .walkFiles)) - set relPath = "" + set relSourceDir = ##class(%IPM.Storage.FileHash).NormalizePath($extract(sourceDir, $length(moduleRoot) + 1, *)) + if relSourceDir = "" { + quit + } + set prefix = relSourceDir _ "/" + set prefixLen = $length(prefix) + set relPath = prefix for { - set relPath = $order(walkFiles(relPath)) + set relPath = $order(allFiles(relPath)) quit:relPath="" + quit:($extract(relPath, 1, prefixLen) '= prefix) set relPaths(relPath) = "" } } catch e { diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls index 1e64d5265..8af8fd7b9 100644 --- a/src/cls/IPM/ResourceProcessor/Test.cls +++ b/src/cls/IPM/ResourceProcessor/Test.cls @@ -385,22 +385,20 @@ Method SupportsSync() As %Boolean /// classes, so a test file that was never compiled — e.g. one just added to the test /// directory — is otherwise invisible to ComputeChanges until something else compiles it /// first. This live directory scan closes that gap. -Method OnSyncResolveFiles(Output relPaths) As %Status +Method OnSyncResolveFiles(Output relPaths, ByRef allFiles) As %Status { set sc = $$$OK try { - set unitTestDir = ##class(%File).NormalizeDirectory(..ResourceReference.Module.Root _ ..ResourceReference.Name) - set moduleRoot = ##class(%File).NormalizeDirectory(..ResourceReference.Module.Root) - if '##class(%File).DirectoryExists(unitTestDir) { - quit - } - kill walkFiles - $$$ThrowOnError(##class(%IPM.Storage.FileHash).WalkFilesRecursive(unitTestDir, moduleRoot, .walkFiles, "cls")) - set relPath = "" + set prefix = ##class(%IPM.Storage.FileHash).NormalizePath(..ResourceReference.Name _ "/") + set prefixLen = $length(prefix) + set relPath = prefix for { - set relPath = $order(walkFiles(relPath)) + set relPath = $order(allFiles(relPath)) quit:relPath="" - set relPaths(relPath) = "" + quit:($extract(relPath, 1, prefixLen) '= prefix) + if $$$lcase($piece(relPath, ".", *)) = "cls" { + set relPaths(relPath) = "" + } } } catch e { set sc = e.AsStatus() diff --git a/src/cls/IPM/Storage/FileHash.cls b/src/cls/IPM/Storage/FileHash.cls index b5799175e..d7ba3ed4a 100644 --- a/src/cls/IPM/Storage/FileHash.cls +++ b/src/cls/IPM/Storage/FileHash.cls @@ -34,13 +34,19 @@ ClassMethod StampModule(module As %IPM.Storage.Module) As %Status $$$ThrowOnError(..StampOneFile(module.Name, root, moduleXmlPath)) } - // Pass 1: BFS the module root for compilable files present in the namespace. - kill bfsFiles - $$$ThrowOnError(..WalkFilesRecursive(root, root, .bfsFiles, "cls,inc,mac,int")) + // Single walk of the entire module root (uses Python os.walk for speed). + kill allFiles + $$$ThrowOnError(..WalkFilesRecursive(root, root, .allFiles)) + + // Pass 1: filter for compilable files present in the namespace. set relPath = "" for { - set relPath = $order(bfsFiles(relPath), 1, entryPath) + set relPath = $order(allFiles(relPath), 1, entryPath) quit:relPath="" + set ext = $$$lcase($piece(relPath, ".", *)) + if ",cls,inc,mac,int," '[ (","_ext_",") { + continue + } set docName = ..RelPathToDocName(relPath) if docName = "" { continue @@ -72,7 +78,7 @@ ClassMethod StampModule(module As %IPM.Storage.Module) As %Status continue } kill syncOnlyPaths - set childSC = resource.Processor.OnSyncResolveFiles(.syncOnlyPaths) + set childSC = resource.Processor.OnSyncResolveFiles(.syncOnlyPaths, .allFiles) if $$$ISERR(childSC) { continue } @@ -156,12 +162,14 @@ ClassMethod RelPathToDocName(relPath As %String) As %String } /// Compute which files changed on disk vs stored baseline. -/// Pass 1: BFS for compilable files present in the namespace. +/// Pass 1: filter pre-walked BFS data for compilable files present in the namespace. /// Pass 2: check manifest-derived paths (from reverseIndex) for files not yet compiled /// or non-compilable tracked files — avoids re-calling ResolveChildren/OnSyncResolveFiles. /// Pass 3: iterate stored rows to detect deletions. +/// allFiles(relPath)=fullPath is the full pre-walked module root (all extensions). +/// bfsFiles(relPath)=fullPath is the compilable subset (cls,inc,mac,int). /// Returns modified(relPath)=newHash and deleted(relPath)="" arrays. -ClassMethod ComputeChanges(module As %IPM.Storage.Module, Output modified, Output deleted, ByRef manifestPaths) As %Status +ClassMethod ComputeChanges(module As %IPM.Storage.Module, Output modified, Output deleted, ByRef manifestPaths, ByRef bfsFiles, ByRef allFiles) As %Status { set sc = $$$OK kill modified, deleted @@ -175,9 +183,7 @@ ClassMethod ComputeChanges(module As %IPM.Storage.Module, Output modified, Outpu do ..CheckOneFile(module.Name, root, moduleXmlPath, .modified, .seen) } - // Pass 1: BFS module root for compilable files present in the namespace. - kill bfsFiles - $$$ThrowOnError(..WalkFilesRecursive(root, root, .bfsFiles, "cls,inc,mac,int")) + // Pass 1: filter pre-walked compilable files by namespace presence. set relPath = "" for { set relPath = $order(bfsFiles(relPath), 1, entryPath) @@ -201,27 +207,25 @@ ClassMethod ComputeChanges(module As %IPM.Storage.Module, Output modified, Outpu } // Pass 2: check manifest-derived paths for files BFS missed (uncompiled new files, - // non-compilable tracked files). These come from reverseIndex which already called - // ResolveChildren + OnSyncResolveFiles, avoiding redundant iteration here. + // non-compilable tracked files). Use allFiles for existence check (O(1) $data) + // instead of filesystem calls. set relPath = "" for { set relPath = $order(manifestPaths(relPath)) quit:relPath="" - set fullPath = ##class(%File).NormalizeFilename(relPath, root) - if '##class(%File).Exists(fullPath) { + if '$data(allFiles(relPath)) { continue } - do ..CheckOneFile(module.Name, root, fullPath, .modified, .seen) + do ..CheckOneFile(module.Name, root, allFiles(relPath), .modified, .seen) } - // Pass 3: iterate all stored rows — file missing from disk → deleted. + // Pass 3: iterate all stored rows — file not in allFiles → deleted. set result = ##class(%SQL.Statement).%ExecDirect(, "SELECT RelativePath FROM %IPM_Storage.FileHash WHERE ModuleName = ?", module.Name) while result.%Next() { set storedRelPath = result.%Get("RelativePath") - set fullPath = ##class(%File).NormalizeFilename(storedRelPath, root) - if '##class(%File).Exists(fullPath) { + if '$data(allFiles(storedRelPath)) { set deleted(storedRelPath) = "" } } @@ -334,10 +338,73 @@ ClassMethod NormalizePath(path As %String) As %String quit path } -/// BFS walk all files under dir, returning paths relative to relativeToRoot. +/// Walk all files under dir using Python os.walk() for performance. +/// Returns files(normalizedRelPath) = fullPath, relative to relativeToRoot. /// extensionFilter is comma-separated (e.g. "cls,inc,mac,int") or "" for all files. -/// Output: files(normalizedRelPath) = fullPath. +/// Falls back to SQL-based BFS if Python is unavailable. ClassMethod WalkFilesRecursive(dir As %String, relativeToRoot As %String, Output files, extensionFilter As %String = "") As %Status +{ + set sc = ..WalkFilesRecursivePython(dir, relativeToRoot, .files, extensionFilter) + if $$$ISERR(sc) { + set sc = ..WalkFilesRecursiveSQL(dir, relativeToRoot, .files, extensionFilter) + } + quit sc +} + +/// Python implementation of recursive file walk using os.walk(). +/// ~10x faster than SQL-based FileSet iteration. +ClassMethod WalkFilesRecursivePython(dir As %String, relativeToRoot As %String, Output files, extensionFilter As %String = "") As %Status +{ + set sc = $$$OK + try { + set dir = ##class(%File).NormalizeDirectory(dir) + set relativeToRoot = ##class(%File).NormalizeDirectory(relativeToRoot) + set jsonStr = ..WalkFilesRecursivePythonImpl(dir, relativeToRoot, extensionFilter) + set result = ##class(%DynamicArray).%FromJSON(jsonStr) + set count = result.%Size() + for i = 0:1:(count - 1) { + set entry = result.%Get(i) + set relPath = entry.%Get("rel") + set fullPath = entry.%Get("full") + set files(relPath) = fullPath + } + } catch e { + set sc = e.AsStatus() + } + quit sc +} + +ClassMethod WalkFilesRecursivePythonImpl(dir As %String, relativeToRoot As %String, extensionFilter As %String = "") As %String [ Language = python ] +{ +import os +import json + +SKIP_DIRS = {'.git', '__pycache__', 'node_modules'} + +results = [] +ext_filter = set() +if extensionFilter: + ext_filter = set(extensionFilter.lower().split(',')) + +root_len = len(relativeToRoot.rstrip(os.sep)) + 1 + +for dirpath, dirnames, filenames in os.walk(dir): + dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS] + for fname in filenames: + if ext_filter: + ext = fname.rsplit('.', 1)[-1].lower() if '.' in fname else '' + if ext not in ext_filter: + continue + full_path = os.path.join(dirpath, fname) + rel_path = full_path[root_len:].replace('\\', '/') + if rel_path: + results.append({"rel": rel_path, "full": full_path}) + +return json.dumps(results) +} + +/// SQL-based BFS fallback for environments without Python. +ClassMethod WalkFilesRecursiveSQL(dir As %String, relativeToRoot As %String, Output files, extensionFilter As %String = "") As %Status { set sc = $$$OK try { diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls index ae52d718d..92b5d0c2c 100644 --- a/src/cls/IPM/Storage/Module.cls +++ b/src/cls/IPM/Storage/Module.cls @@ -768,15 +768,32 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status $$$ThrowOnError(##class(%IPM.Storage.FileHash).StampModule(module)) } - // Step 2: Build reverse index (relPath -> owning resource + processor). - // Done before ComputeChanges so we can pass manifest-derived paths to it, - // avoiding a redundant ResolveChildren + OnSyncResolveFiles iteration. + // Step 2: Walk the entire module root once (Python os.walk, fast). + // This pre-walked data feeds both the reverse index and change detection, + // eliminating redundant filesystem walks by processors. + set walkStart = $zhorolog + kill allFiles, bfsFiles + $$$ThrowOnError(##class(%IPM.Storage.FileHash).WalkFilesRecursive(root, root, .allFiles)) + // Also build a compilable-only subset for ComputeChanges Pass 1. + set relPath = "" + for { + set relPath = $order(allFiles(relPath), 1, fullPath) + quit:relPath="" + set ext = $$$lcase($piece(relPath, ".", *)) + if ",cls,inc,mac,int," [ (","_ext_",") { + set bfsFiles(relPath) = fullPath + } + } + if verbose { + write !, "[", moduleName, "] File walk: ", $fnumber($zhorolog - walkStart, "", 2), "s" + } + + // Step 3: Build reverse index (relPath -> owning resource + processor). set orderedResourceList = module.GetOrderedResourceList() kill reverseIndex, unsupportedResources - do ..SyncBuildReverseIndex(module, orderedResourceList, .reverseIndex, .unsupportedResources) + do ..SyncBuildReverseIndex(module, orderedResourceList, .reverseIndex, .unsupportedResources, .allFiles) // Collect manifest-derived paths from reverseIndex for ComputeChanges. - // These supplement the BFS walk (which only finds compiled files). kill manifestPaths set riCount = 0 set riKey = "" @@ -790,9 +807,9 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status write !, "[", moduleName, "] Reverse index: ", riCount, " tracked path(s)" } - // Step 3: Compute disk changes vs baseline + // Step 4: Compute disk changes vs baseline set scanStart = $zhorolog - $$$ThrowOnError(##class(%IPM.Storage.FileHash).ComputeChanges(module, .modified, .deleted, .manifestPaths)) + $$$ThrowOnError(##class(%IPM.Storage.FileHash).ComputeChanges(module, .modified, .deleted, .manifestPaths, .bfsFiles, .allFiles)) if verbose { write !, "[", moduleName, "] Change detection: ", $fnumber($zhorolog - scanStart, "", 2), "s" } @@ -815,14 +832,14 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status quit } - // Step 4: Partition changes by owning resource + // Step 5: Partition changes by owning resource kill syncByResource do ..SyncRoutePathSet(.modified, .reverseIndex, "modified", .syncByResource) if processDeletes { do ..SyncRoutePathSet(.deleted, .reverseIndex, "deleted", .syncByResource) } - // Step 5: Dispatch OnSync to each processor; load unhandled compilable files + // Step 6: Dispatch OnSync to each processor; load unhandled compilable files if verbose { set resCount = 0 set resName = "" @@ -835,18 +852,18 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status } $$$ThrowOnError(..SyncDispatchProcessors(module, root, verbose, .syncByResource, .params, .loadItems)) - // Step 6: Compile the full resource set with u-flag to pick up dependent recompiles + // Step 7: Compile the full resource set with u-flag to pick up dependent recompiles if loadItems > 0 { $$$ThrowOnError(..SyncCompile(module, orderedResourceList, verbose, .params)) } - // Step 7: Delete server-side documents for removed files, then recompile + // Step 8: Delete server-side documents for removed files, then recompile if processDeletes && ($data(deleted) > 1) { do ..SyncApplyDeletes(.deleted, .reverseIndex, .syncByResource, verbose) $$$ThrowOnError(..SyncCompile(module, orderedResourceList, verbose, .params)) } - // Step 8: Commit new hashes on success (skipped on error so next sync re-detects). + // Step 9: Commit new hashes on success (skipped on error so next sync re-detects). // Done before running tests below so a test failure doesn't prevent the file-sync // outcome from being committed and reported — file sync and test results are independent. $$$ThrowOnError(##class(%IPM.Storage.FileHash).CommitChanges(module, .modified, .deleted, processDeletes)) @@ -887,7 +904,7 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status write !, "[", moduleName, "] Done in ", $fnumber($zhorolog - syncStart, "", 2), "s" - // Step 9: Run changed test-phase tests if -test flag is set (after sync is reported) + // Step 10: Run changed test-phase tests if -test flag is set (after sync is reported) if runTests { $$$ThrowOnError(..SyncRunTests(orderedResourceList, verbose, .params)) } @@ -932,7 +949,7 @@ ClassMethod SyncCheckModuleXml( /// Step 1 builds docToResource (docName → owner) from ResolveChildren and maps /// OnSyncResolveFiles paths directly into reverseIndex (non-compilable resources). /// Step 2 maps all stored baseline paths to owners via RelPathToDocName → docToResource. -ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, orderedResourceList As %ListOfObjects, ByRef reverseIndex, Output unsupportedResources) +ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, orderedResourceList As %ListOfObjects, ByRef reverseIndex, Output unsupportedResources, ByRef allFiles) { kill unsupportedResources // docToResource maps server document names (e.g. "SyncTest.SuperClass.CLS") to the @@ -984,9 +1001,10 @@ ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, orderedResource // OnSyncResolveFiles returns filesystem-relative paths for non-compilable resources // (FileCopy directories, test directories). These have no document name, so they go - // directly into reverseIndex keyed by relPath. + // directly into reverseIndex keyed by relPath. Passes pre-walked allFiles so + // processors filter in-memory instead of re-walking the filesystem. kill syncOnlyPaths - set childSC = resource.Processor.OnSyncResolveFiles(.syncOnlyPaths) + set childSC = resource.Processor.OnSyncResolveFiles(.syncOnlyPaths, .allFiles) if $$$ISERR(childSC) { continue } From 16fc56a8c0955de39c36475425d95601771ecf96 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Mon, 20 Jul 2026 16:21:21 -0400 Subject: [PATCH 19/39] Fix casing bug and merge walk+hash in Python --- src/cls/IPM/Storage/FileHash.cls | 157 +++++++++++++++---------------- src/cls/IPM/Storage/Module.cls | 34 ++++--- 2 files changed, 93 insertions(+), 98 deletions(-) diff --git a/src/cls/IPM/Storage/FileHash.cls b/src/cls/IPM/Storage/FileHash.cls index d7ba3ed4a..26b03a96f 100644 --- a/src/cls/IPM/Storage/FileHash.cls +++ b/src/cls/IPM/Storage/FileHash.cls @@ -28,20 +28,20 @@ ClassMethod StampModule(module As %IPM.Storage.Module) As %Status try { set root = ##class(%File).NormalizeDirectory(module.Root) + // Single walk+hash of the entire module root. + kill allFiles, allHashes + $$$ThrowOnError(..WalkAndHashFiles(root, root, .allFiles, .allHashes)) + // module.xml is always tracked. - set moduleXmlPath = root _ "module.xml" - if ##class(%File).Exists(moduleXmlPath) { - $$$ThrowOnError(..StampOneFile(module.Name, root, moduleXmlPath)) + set moduleXmlRel = "module.xml" + if $data(allHashes(moduleXmlRel)) { + $$$ThrowOnError(..StampOneFileWithHash(module.Name, moduleXmlRel, allHashes(moduleXmlRel))) } - // Single walk of the entire module root (uses Python os.walk for speed). - kill allFiles - $$$ThrowOnError(..WalkFilesRecursive(root, root, .allFiles)) - // Pass 1: filter for compilable files present in the namespace. set relPath = "" for { - set relPath = $order(allFiles(relPath), 1, entryPath) + set relPath = $order(allFiles(relPath)) quit:relPath="" set ext = $$$lcase($piece(relPath, ".", *)) if ",cls,inc,mac,int," '[ (","_ext_",") { @@ -54,15 +54,19 @@ ClassMethod StampModule(module As %IPM.Storage.Module) As %Status set docExt = $$$lcase($piece(docName, ".", *)) if docExt = "cls" { set className = $piece(docName, ".", 1, *-1) - if '$$$comClassDefined(className) && '$$$comClassDefined("%" _ className) { + if '$$$comClassDefined(className) { continue } } else { - if '##class(%RoutineMgr).Exists(docName) && '##class(%RoutineMgr).Exists("%" _ docName) { + if '##class(%RoutineMgr).Exists(docName) { continue } } - set stampSC = ..StampOneFile(module.Name, root, entryPath) + set hash = $get(allHashes(relPath)) + if hash = "" { + continue + } + set stampSC = ..StampOneFileWithHash(module.Name, relPath, hash) if $$$ISERR(stampSC) { write !, "Warning: could not stamp ", relPath, ": ", $system.Status.GetOneErrorText(stampSC) } @@ -86,11 +90,11 @@ ClassMethod StampModule(module As %IPM.Storage.Module) As %Status for { set relPath = $order(syncOnlyPaths(relPath)) quit:relPath="" - set fullPath = ##class(%File).NormalizeFilename(relPath, root) - if '##class(%File).Exists(fullPath) { + set hash = $get(allHashes(relPath)) + if hash = "" { continue } - set stampSC = ..StampOneFile(module.Name, root, fullPath) + set stampSC = ..StampOneFileWithHash(module.Name, relPath, hash) if $$$ISERR(stampSC) { write !, "Warning: could not stamp ", relPath, ": ", $system.Status.GetOneErrorText(stampSC) } @@ -102,22 +106,19 @@ ClassMethod StampModule(module As %IPM.Storage.Module) As %Status quit sc } -/// Stamp (or update) the FileHash row for a single file. Computes hash at stamp time. -/// Returns an error if the file cannot be hashed (locked/permission denied). -ClassMethod StampOneFile(moduleName As %String, root As %String, fullPath As %String) As %Status +/// Stamp (or update) the FileHash row for a single file using a pre-computed hash. +ClassMethod StampOneFileWithHash(moduleName As %String, relPath As %String, hash As %String) As %Status { - set normalizedRelPath = ..NormalizePath($extract(fullPath, $length(root) + 1, *)) - set hash = ##class(%File).SHA1Hash(fullPath, 1) if hash = "" { - quit $$$ERROR($$$GeneralError, "Could not compute hash for: " _ fullPath) + quit $$$ERROR($$$GeneralError, "Empty hash for: " _ relPath) } - set existing = ..ModulePathIndexOpen(moduleName, normalizedRelPath, , .openSC) + set existing = ..ModulePathIndexOpen(moduleName, relPath, , .openSC) if $isobject(existing) { set instance = existing } else { set instance = ..%New() set instance.ModuleName = moduleName - set instance.RelativePath = normalizedRelPath + set instance.RelativePath = relPath } set instance.Hash = hash quit instance.%Save() @@ -164,29 +165,30 @@ ClassMethod RelPathToDocName(relPath As %String) As %String /// Compute which files changed on disk vs stored baseline. /// Pass 1: filter pre-walked BFS data for compilable files present in the namespace. /// Pass 2: check manifest-derived paths (from reverseIndex) for files not yet compiled -/// or non-compilable tracked files — avoids re-calling ResolveChildren/OnSyncResolveFiles. +/// or non-compilable tracked files. /// Pass 3: iterate stored rows to detect deletions. /// allFiles(relPath)=fullPath is the full pre-walked module root (all extensions). +/// allHashes(relPath)=sha1hex are pre-computed hashes from the same walk. /// bfsFiles(relPath)=fullPath is the compilable subset (cls,inc,mac,int). /// Returns modified(relPath)=newHash and deleted(relPath)="" arrays. -ClassMethod ComputeChanges(module As %IPM.Storage.Module, Output modified, Output deleted, ByRef manifestPaths, ByRef bfsFiles, ByRef allFiles) As %Status +ClassMethod ComputeChanges(module As %IPM.Storage.Module, Output modified, Output deleted, ByRef manifestPaths, ByRef bfsFiles, ByRef allFiles, ByRef allHashes) As %Status { set sc = $$$OK kill modified, deleted try { - set root = ##class(%File).NormalizeDirectory(module.Root) + set moduleName = module.Name kill seen // module.xml - set moduleXmlPath = root _ "module.xml" - if ##class(%File).Exists(moduleXmlPath) { - do ..CheckOneFile(module.Name, root, moduleXmlPath, .modified, .seen) + set moduleXmlRel = "module.xml" + if $data(allHashes(moduleXmlRel)) { + do ..CompareOneFile(moduleName, moduleXmlRel, $get(allHashes(moduleXmlRel)), .modified, .seen) } // Pass 1: filter pre-walked compilable files by namespace presence. set relPath = "" for { - set relPath = $order(bfsFiles(relPath), 1, entryPath) + set relPath = $order(bfsFiles(relPath)) quit:relPath="" set docName = ..RelPathToDocName(relPath) if docName = "" { @@ -195,20 +197,19 @@ ClassMethod ComputeChanges(module As %IPM.Storage.Module, Output modified, Outpu set docExt = $$$lcase($piece(docName, ".", *)) if docExt = "cls" { set className = $piece(docName, ".", 1, *-1) - if '$$$comClassDefined(className) && '$$$comClassDefined("%" _ className) { + if '$$$comClassDefined(className) { continue } } else { - if '##class(%RoutineMgr).Exists(docName) && '##class(%RoutineMgr).Exists("%" _ docName) { + if '##class(%RoutineMgr).Exists(docName) { continue } } - do ..CheckOneFile(module.Name, root, entryPath, .modified, .seen) + do ..CompareOneFile(moduleName, relPath, $get(allHashes(relPath)), .modified, .seen) } // Pass 2: check manifest-derived paths for files BFS missed (uncompiled new files, - // non-compilable tracked files). Use allFiles for existence check (O(1) $data) - // instead of filesystem calls. + // non-compilable tracked files). set relPath = "" for { set relPath = $order(manifestPaths(relPath)) @@ -216,13 +217,13 @@ ClassMethod ComputeChanges(module As %IPM.Storage.Module, Output modified, Outpu if '$data(allFiles(relPath)) { continue } - do ..CheckOneFile(module.Name, root, allFiles(relPath), .modified, .seen) + do ..CompareOneFile(moduleName, relPath, $get(allHashes(relPath)), .modified, .seen) } // Pass 3: iterate all stored rows — file not in allFiles → deleted. set result = ##class(%SQL.Statement).%ExecDirect(, "SELECT RelativePath FROM %IPM_Storage.FileHash WHERE ModuleName = ?", - module.Name) + moduleName) while result.%Next() { set storedRelPath = result.%Get("RelativePath") if '$data(allFiles(storedRelPath)) { @@ -235,28 +236,24 @@ ClassMethod ComputeChanges(module As %IPM.Storage.Module, Output modified, Outpu quit sc } -/// Check a single file against its stored baseline. If no row or hash differs, add to modified. -/// If hash cannot be computed (locked/permission denied), reports as modified with "" hash and warns. -ClassMethod CheckOneFile(moduleName As %String, root As %String, fullPath As %String, ByRef modified, ByRef seen) [ Private ] +/// Compare a file's pre-computed hash against stored baseline. No filesystem I/O. +ClassMethod CompareOneFile(moduleName As %String, relPath As %String, newHash As %String, ByRef modified, ByRef seen) [ Private ] { - set normalizedRelPath = ..NormalizePath($extract(fullPath, $length(root) + 1, *)) - if $data(seen(normalizedRelPath)) { + if $data(seen(relPath)) { quit } - set seen(normalizedRelPath) = "" - set newHash = ##class(%File).SHA1Hash(fullPath, 1) + set seen(relPath) = "" if newHash = "" { - write !, "Warning: could not hash ", normalizedRelPath, " (file may be locked)" - set modified(normalizedRelPath) = "" + set modified(relPath) = "" quit } - set existing = ..ModulePathIndexOpen(moduleName, normalizedRelPath, , .openSC) + set existing = ..ModulePathIndexOpen(moduleName, relPath, , .openSC) if '$isobject(existing) { - set modified(normalizedRelPath) = newHash + set modified(relPath) = newHash quit } if existing.Hash '= newHash { - set modified(normalizedRelPath) = newHash + set modified(relPath) = newHash } } @@ -265,14 +262,11 @@ ClassMethod CommitChanges(module As %IPM.Storage.Module, ByRef modified, ByRef d { set sc = $$$OK try { - set root = ##class(%File).NormalizeDirectory(module.Root) - set relPath = "" for { set relPath = $order(modified(relPath), 1, newHash) quit:relPath="" - set fullPath = ##class(%File).NormalizeFilename(relPath, root) set existing = ..ModulePathIndexOpen(module.Name, relPath, , .openSC) if $isobject(existing) { set instance = existing @@ -338,35 +332,32 @@ ClassMethod NormalizePath(path As %String) As %String quit path } -/// Walk all files under dir using Python os.walk() for performance. -/// Returns files(normalizedRelPath) = fullPath, relative to relativeToRoot. -/// extensionFilter is comma-separated (e.g. "cls,inc,mac,int") or "" for all files. -/// Falls back to SQL-based BFS if Python is unavailable. -ClassMethod WalkFilesRecursive(dir As %String, relativeToRoot As %String, Output files, extensionFilter As %String = "") As %Status +/// Walk all files and compute SHA1 hashes in a single pass. +/// Returns files(relPath) = fullPath and hashes(relPath) = sha1hex (lowercase). +/// Uses Python os.walk + hashlib for speed; falls back to SQL BFS + %File.SHA1Hash. +ClassMethod WalkAndHashFiles(dir As %String, relativeToRoot As %String, Output files, Output hashes) As %Status { - set sc = ..WalkFilesRecursivePython(dir, relativeToRoot, .files, extensionFilter) + set sc = ..WalkAndHashFilesPython(dir, relativeToRoot, .files, .hashes) if $$$ISERR(sc) { - set sc = ..WalkFilesRecursiveSQL(dir, relativeToRoot, .files, extensionFilter) + set sc = ..WalkAndHashFilesSQL(dir, relativeToRoot, .files, .hashes) } quit sc } -/// Python implementation of recursive file walk using os.walk(). -/// ~10x faster than SQL-based FileSet iteration. -ClassMethod WalkFilesRecursivePython(dir As %String, relativeToRoot As %String, Output files, extensionFilter As %String = "") As %Status +ClassMethod WalkAndHashFilesPython(dir As %String, relativeToRoot As %String, Output files, Output hashes) As %Status [ Private ] { set sc = $$$OK try { set dir = ##class(%File).NormalizeDirectory(dir) set relativeToRoot = ##class(%File).NormalizeDirectory(relativeToRoot) - set jsonStr = ..WalkFilesRecursivePythonImpl(dir, relativeToRoot, extensionFilter) + set jsonStr = ..WalkAndHashFilesPythonImpl(dir, relativeToRoot) set result = ##class(%DynamicArray).%FromJSON(jsonStr) set count = result.%Size() for i = 0:1:(count - 1) { set entry = result.%Get(i) set relPath = entry.%Get("rel") - set fullPath = entry.%Get("full") - set files(relPath) = fullPath + set files(relPath) = entry.%Get("full") + set hashes(relPath) = entry.%Get("hash") } } catch e { set sc = e.AsStatus() @@ -374,37 +365,38 @@ ClassMethod WalkFilesRecursivePython(dir As %String, relativeToRoot As %String, quit sc } -ClassMethod WalkFilesRecursivePythonImpl(dir As %String, relativeToRoot As %String, extensionFilter As %String = "") As %String [ Language = python ] +ClassMethod WalkAndHashFilesPythonImpl(dir As %String, relativeToRoot As %String) As %String [ Language = python ] { import os import json +import hashlib SKIP_DIRS = {'.git', '__pycache__', 'node_modules'} results = [] -ext_filter = set() -if extensionFilter: - ext_filter = set(extensionFilter.lower().split(',')) - root_len = len(relativeToRoot.rstrip(os.sep)) + 1 for dirpath, dirnames, filenames in os.walk(dir): dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS] for fname in filenames: - if ext_filter: - ext = fname.rsplit('.', 1)[-1].lower() if '.' in fname else '' - if ext not in ext_filter: - continue full_path = os.path.join(dirpath, fname) rel_path = full_path[root_len:].replace('\\', '/') - if rel_path: - results.append({"rel": rel_path, "full": full_path}) + if not rel_path: + continue + try: + h = hashlib.sha1() + with open(full_path, 'rb') as f: + while chunk := f.read(65536): + h.update(chunk) + results.append({"rel": rel_path, "full": full_path, "hash": h.hexdigest()}) + except (OSError, PermissionError): + results.append({"rel": rel_path, "full": full_path, "hash": ""}) return json.dumps(results) } -/// SQL-based BFS fallback for environments without Python. -ClassMethod WalkFilesRecursiveSQL(dir As %String, relativeToRoot As %String, Output files, extensionFilter As %String = "") As %Status +/// SQL-based BFS fallback that walks and hashes without Python. +ClassMethod WalkAndHashFilesSQL(dir As %String, relativeToRoot As %String, Output files, Output hashes) As %Status [ Private ] { set sc = $$$OK try { @@ -425,19 +417,18 @@ ClassMethod WalkFilesRecursiveSQL(dir As %String, relativeToRoot As %String, Out set entryType = rs.%Get("Type") set entryPath = rs.%Get("Name") if entryType = "D" { + set childDirName = ##class(%File).GetFilename(entryPath) + if ",.git,__pycache__,node_modules," [ (","_childDirName_",") { + continue + } set walkTail = walkTail + 1 set walkQueue(walkTail) = ##class(%File).NormalizeDirectory(entryPath) continue } - if extensionFilter '= "" { - set ext = $$$lcase($piece(entryPath, ".", *)) - if (","_extensionFilter_",") '[ (","_ext_",") { - continue - } - } set relPath = ..NormalizePath($extract(entryPath, $length(relativeToRoot) + 1, *)) if relPath '= "" { set files(relPath) = entryPath + set hashes(relPath) = $$$lcase(##class(%File).SHA1Hash(entryPath, 1)) } } } diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls index 92b5d0c2c..f9420bf8b 100644 --- a/src/cls/IPM/Storage/Module.cls +++ b/src/cls/IPM/Storage/Module.cls @@ -768,13 +768,13 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status $$$ThrowOnError(##class(%IPM.Storage.FileHash).StampModule(module)) } - // Step 2: Walk the entire module root once (Python os.walk, fast). - // This pre-walked data feeds both the reverse index and change detection, - // eliminating redundant filesystem walks by processors. + // Step 2: Walk the entire module root and hash all files in one Python pass. + // This pre-walked data feeds the reverse index, change detection, and deletion checks — + // no further filesystem I/O needed from ObjectScript. set walkStart = $zhorolog - kill allFiles, bfsFiles - $$$ThrowOnError(##class(%IPM.Storage.FileHash).WalkFilesRecursive(root, root, .allFiles)) - // Also build a compilable-only subset for ComputeChanges Pass 1. + kill allFiles, allHashes, bfsFiles + $$$ThrowOnError(##class(%IPM.Storage.FileHash).WalkAndHashFiles(root, root, .allFiles, .allHashes)) + // Build a compilable-only subset for ComputeChanges Pass 1. set relPath = "" for { set relPath = $order(allFiles(relPath), 1, fullPath) @@ -784,8 +784,16 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status set bfsFiles(relPath) = fullPath } } + // Count files walked for verbose output. + set fileCount = 0 + set countKey = "" + for { + set countKey = $order(allFiles(countKey)) + quit:countKey="" + set fileCount = fileCount + 1 + } if verbose { - write !, "[", moduleName, "] File walk: ", $fnumber($zhorolog - walkStart, "", 2), "s" + write !, "[", moduleName, "] Scanned ", fileCount, " file(s) in ", $fnumber($zhorolog - walkStart, "", 2), "s" } // Step 3: Build reverse index (relPath -> owning resource + processor). @@ -804,15 +812,11 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status set riCount = riCount + 1 } if verbose { - write !, "[", moduleName, "] Reverse index: ", riCount, " tracked path(s)" + write !, "[", moduleName, "] Tracking ", riCount, " path(s) across ", orderedResourceList.Count(), " resource(s)" } // Step 4: Compute disk changes vs baseline - set scanStart = $zhorolog - $$$ThrowOnError(##class(%IPM.Storage.FileHash).ComputeChanges(module, .modified, .deleted, .manifestPaths, .bfsFiles, .allFiles)) - if verbose { - write !, "[", moduleName, "] Change detection: ", $fnumber($zhorolog - scanStart, "", 2), "s" - } + $$$ThrowOnError(##class(%IPM.Storage.FileHash).ComputeChanges(module, .modified, .deleted, .manifestPaths, .bfsFiles, .allFiles, .allHashes)) // Exclude module.xml from file routing (handled by SyncCheckModuleXml above) kill modified(moduleXmlRelPath) @@ -930,7 +934,7 @@ ClassMethod SyncCheckModuleXml( if '$isobject(existing) { quit 0 } - set newHash = ##class(%File).SHA1Hash(moduleXmlPath, 1) + set newHash = $$$lcase(##class(%File).SHA1Hash(moduleXmlPath, 1)) if newHash = existing.Hash { quit 0 } @@ -1227,7 +1231,7 @@ ClassMethod SyncCommitModuleXml( moduleXmlRelPath As %String) { kill moduleXmlMod, emptyDel - set moduleXmlMod(moduleXmlRelPath) = ##class(%File).SHA1Hash(moduleXmlPath, 1) + set moduleXmlMod(moduleXmlRelPath) = $$$lcase(##class(%File).SHA1Hash(moduleXmlPath, 1)) set commitSC = ##class(%IPM.Storage.FileHash).CommitChanges(module, .moduleXmlMod, .emptyDel, 0) if $$$ISERR(commitSC) { write !, "Warning: failed to record module.xml hash: ", $system.Status.GetOneErrorText(commitSC) From c741873fc7699340ce47ec2b9dd68a4a17d15b08 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Tue, 21 Jul 2026 11:05:06 -0400 Subject: [PATCH 20/39] Fix a few small issues and add some unit tests --- src/cls/IPM/ResourceProcessor/Abstract.cls | 5 +- src/cls/IPM/ResourceProcessor/FileCopy.cls | 30 ---------- src/cls/IPM/Storage/FileHash.cls | 6 ++ src/cls/IPM/Storage/Module.cls | 60 +++++++++---------- .../Test/PM/Integration/Sync.cls | 23 +++++++ tests/unit_tests/Test/PM/Unit/FileHash.cls | 35 +++++++++++ 6 files changed, 97 insertions(+), 62 deletions(-) create mode 100644 tests/unit_tests/Test/PM/Unit/FileHash.cls diff --git a/src/cls/IPM/ResourceProcessor/Abstract.cls b/src/cls/IPM/ResourceProcessor/Abstract.cls index 79a647d92..c9f600ea4 100644 --- a/src/cls/IPM/ResourceProcessor/Abstract.cls +++ b/src/cls/IPM/ResourceProcessor/Abstract.cls @@ -222,7 +222,10 @@ Method SupportsSync() As %Boolean /// OnResolveChildren/OnItemRelativePath — e.g. a directory-scanned resource whose file set /// isn't declared as individual module.xml resources. Output relPaths(relPath)="" relative to /// the module root. allFiles(relPath)=fullPath contains the pre-walked module root — filter -/// from it via $order prefix scan rather than doing filesystem I/O. Base returns nothing. +/// from it via $order prefix scan rather than doing filesystem I/O. +/// Base implementation returns an empty relPaths (no files claimed). Only processors with +/// SupportsSync()=1 should override this — SyncBuildReverseIndex skips processors that +/// return 0 from SupportsSync(), so an override on a non-sync processor has no effect. Method OnSyncResolveFiles(Output relPaths, ByRef allFiles) As %Status { quit $$$OK diff --git a/src/cls/IPM/ResourceProcessor/FileCopy.cls b/src/cls/IPM/ResourceProcessor/FileCopy.cls index 41216c8a5..fe62659ad 100644 --- a/src/cls/IPM/ResourceProcessor/FileCopy.cls +++ b/src/cls/IPM/ResourceProcessor/FileCopy.cls @@ -184,36 +184,6 @@ Method DoCopy( quit tSC } -/// Enumerate source files so sync can detect changes to FileCopy resources. -/// Populates relPaths(relPath)="" for each file under the source directory, relative to module root. -Method OnSyncResolveFiles(Output relPaths, ByRef allFiles) As %Status -{ - set sc = $$$OK - try { - set sourceDir = ##class(%File).NormalizeDirectory(..GetSource()) - set moduleRoot = ##class(%File).NormalizeDirectory(..ResourceReference.Module.Root) - // Source must be under module root to appear in allFiles. - if $$$lcase($extract(sourceDir, 1, $length(moduleRoot))) '= $$$lcase(moduleRoot) { - quit - } - set relSourceDir = ##class(%IPM.Storage.FileHash).NormalizePath($extract(sourceDir, $length(moduleRoot) + 1, *)) - if relSourceDir = "" { - quit - } - set prefix = relSourceDir _ "/" - set prefixLen = $length(prefix) - set relPath = prefix - for { - set relPath = $order(allFiles(relPath)) - quit:relPath="" - quit:($extract(relPath, 1, prefixLen) '= prefix) - set relPaths(relPath) = "" - } - } catch e { - set sc = e.AsStatus() - } - quit sc -} Method OnExportItem( pFullExportPath As %String, diff --git a/src/cls/IPM/Storage/FileHash.cls b/src/cls/IPM/Storage/FileHash.cls index 26b03a96f..969a9dc64 100644 --- a/src/cls/IPM/Storage/FileHash.cls +++ b/src/cls/IPM/Storage/FileHash.cls @@ -304,6 +304,9 @@ ClassMethod HasBaseline(moduleName As %String) As %Boolean set result = ##class(%SQL.Statement).%ExecDirect(, "SELECT TOP 1 1 FROM %IPM_Storage.FileHash WHERE ModuleName = ?", moduleName) + if result.%SQLCODE < 0 { + quit 0 + } quit result.%Next() } @@ -314,6 +317,9 @@ ClassMethod GetStoredPaths(moduleName As %String, Output paths) set result = ##class(%SQL.Statement).%ExecDirect(, "SELECT RelativePath FROM %IPM_Storage.FileHash WHERE ModuleName = ?", moduleName) + if result.%SQLCODE < 0 { + quit + } while result.%Next() { set paths(result.%Get("RelativePath")) = "" } diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls index f9420bf8b..b65b7227a 100644 --- a/src/cls/IPM/Storage/Module.cls +++ b/src/cls/IPM/Storage/Module.cls @@ -762,11 +762,13 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status quit } - // After a manifest reload, re-stamp so newly-declared resources get baseline rows - // in this same sync call rather than requiring a separate reload -dev. - if moduleXmlChanged { - $$$ThrowOnError(##class(%IPM.Storage.FileHash).StampModule(module)) - } + // No StampModule call here after a manifest reload: StampModule would overwrite current + // hashes for ALL files (including ones the user just edited), causing ComputeChanges to + // see current-vs-current and report zero changes for co-edited files. + // Newly-declared resources are handled without a full stamp: SyncBuildReverseIndex + // (step 3) calls ResolveChildren, which adds their derived relPaths to reverseIndex → + // manifestPaths. ComputeChanges Pass 2 finds those paths with no baseline row and + // reports them as modified, so they are loaded in this same sync call. // Step 2: Walk the entire module root and hash all files in one Python pass. // This pre-walked data feeds the reverse index, change detection, and deletion checks — @@ -774,24 +776,17 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status set walkStart = $zhorolog kill allFiles, allHashes, bfsFiles $$$ThrowOnError(##class(%IPM.Storage.FileHash).WalkAndHashFiles(root, root, .allFiles, .allHashes)) - // Build a compilable-only subset for ComputeChanges Pass 1. - set relPath = "" + // Build a compilable-only subset for ComputeChanges Pass 1; count total for verbose. + set relPath = "", fileCount = 0 for { set relPath = $order(allFiles(relPath), 1, fullPath) quit:relPath="" + set fileCount = fileCount + 1 set ext = $$$lcase($piece(relPath, ".", *)) if ",cls,inc,mac,int," [ (","_ext_",") { set bfsFiles(relPath) = fullPath } } - // Count files walked for verbose output. - set fileCount = 0 - set countKey = "" - for { - set countKey = $order(allFiles(countKey)) - quit:countKey="" - set fileCount = fileCount + 1 - } if verbose { write !, "[", moduleName, "] Scanned ", fileCount, " file(s) in ", $fnumber($zhorolog - walkStart, "", 2), "s" } @@ -925,7 +920,7 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status ClassMethod SyncCheckModuleXml( ByRef module As %IPM.Storage.Module, moduleXmlPath As %String, - moduleXmlRelPath As %String) As %Boolean + moduleXmlRelPath As %String) As %Boolean [ Private ] { if '##class(%File).Exists(moduleXmlPath) { quit 0 @@ -939,7 +934,6 @@ ClassMethod SyncCheckModuleXml( quit 0 } $$$ThrowOnError($system.OBJ.Load(moduleXmlPath, "-d")) - $$$ThrowOnError(module.%Reload()) set module = ..NameOpen(module.Name, , .openSC) $$$ThrowOnError(openSC) quit 1 @@ -950,10 +944,10 @@ ClassMethod SyncCheckModuleXml( /// Skips resources whose processor does not support sync — those are collected in /// unsupportedResources(resourceName)="" for informational display. /// -/// Step 1 builds docToResource (docName → owner) from ResolveChildren and maps -/// OnSyncResolveFiles paths directly into reverseIndex (non-compilable resources). +/// Step 1 builds docToResource (docName → owner) from ResolveChildren and calls +/// OnSyncResolveFiles for directory-based resources (e.g. test dirs) that have no doc name. /// Step 2 maps all stored baseline paths to owners via RelPathToDocName → docToResource. -ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, orderedResourceList As %ListOfObjects, ByRef reverseIndex, Output unsupportedResources, ByRef allFiles) +ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, orderedResourceList As %ListOfObjects, ByRef reverseIndex, Output unsupportedResources, ByRef allFiles) [ Private ] { kill unsupportedResources // docToResource maps server document names (e.g. "SyncTest.SuperClass.CLS") to the @@ -1003,10 +997,10 @@ ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, orderedResource } } - // OnSyncResolveFiles returns filesystem-relative paths for non-compilable resources - // (FileCopy directories, test directories). These have no document name, so they go - // directly into reverseIndex keyed by relPath. Passes pre-walked allFiles so - // processors filter in-memory instead of re-walking the filesystem. + // OnSyncResolveFiles returns filesystem-relative paths for directory-based resources + // (e.g. test directories). These have no document name, so they go directly into + // reverseIndex keyed by relPath. Passes pre-walked allFiles so processors filter + // in-memory instead of re-walking the filesystem. kill syncOnlyPaths set childSC = resource.Processor.OnSyncResolveFiles(.syncOnlyPaths, .allFiles) if $$$ISERR(childSC) { @@ -1086,7 +1080,7 @@ ClassMethod SyncDispatchProcessors( verbose As %Boolean, ByRef syncByResource, ByRef params, - Output loadItems As %Integer = 0) As %Status + Output loadItems As %Integer = 0) As %Status [ Private ] { set sc = $$$OK try { @@ -1138,7 +1132,7 @@ ClassMethod SyncApplyDeletes( ByRef deleted, ByRef reverseIndex, ByRef syncByResource, - verbose As %Boolean) + verbose As %Boolean) [ Private ] { set relPath = "" for { @@ -1160,6 +1154,10 @@ ClassMethod SyncApplyDeletes( if docName '= "" { set delFlags = $select(verbose:"d", 1:"-d") set delSC = $system.OBJ.Delete(docName, delFlags) + // Delete failures are non-fatal: the SyncCompile pass that immediately follows + // will fail to compile any class that still references the deleted doc, surfacing + // the error with full context. Aborting the delete loop here would leave other + // deletions unapplied and make the overall error harder to diagnose. if $$$ISERR(delSC) { write !, "Warning: could not delete ", docName, ": ", $system.Status.GetOneErrorText(delSC) } @@ -1174,7 +1172,7 @@ ClassMethod SyncApplyDeletes( ClassMethod SyncRunTests( orderedResourceList As %ListOfObjects, verbose As %Boolean, - ByRef params) As %Status + ByRef params) As %Status [ Private ] { set sc = $$$OK try { @@ -1228,7 +1226,7 @@ ClassMethod SyncRunTests( ClassMethod SyncCommitModuleXml( module As %IPM.Storage.Module, moduleXmlPath As %String, - moduleXmlRelPath As %String) + moduleXmlRelPath As %String) [ Private ] { kill moduleXmlMod, emptyDel set moduleXmlMod(moduleXmlRelPath) = $$$lcase(##class(%File).SHA1Hash(moduleXmlPath, 1)) @@ -1238,7 +1236,7 @@ ClassMethod SyncCommitModuleXml( } } -ClassMethod SyncPrintModuleXmlWarning(moduleName As %String) +ClassMethod SyncPrintModuleXmlWarning(moduleName As %String) [ Private ] { write ! write !, "Warning: module.xml changed and was reloaded." @@ -1246,7 +1244,7 @@ ClassMethod SyncPrintModuleXmlWarning(moduleName As %String) write !, " manifest-level changes (mappings, dependencies, defaults)." } -ClassMethod SyncPrintUnsupportedNote(moduleName As %String, ByRef unsupportedResources) +ClassMethod SyncPrintUnsupportedNote(moduleName As %String, ByRef unsupportedResources) [ Private ] { set count = 0 set names = "" @@ -1270,7 +1268,7 @@ ClassMethod SyncPrintUnsupportedNote(moduleName As %String, ByRef unsupportedRes /// changes loaded in SyncDispatchProcessors. Skips CompileFromProject resources — those are /// deployed code loaded via Studio project files (.prj); compiling them individually would /// fail because deployed classes have no source in the routine database. -ClassMethod SyncCompile(module As %IPM.Storage.Module, orderedResourceList As %ListOfObjects, verbose As %Boolean = 0, ByRef params) As %Status +ClassMethod SyncCompile(module As %IPM.Storage.Module, orderedResourceList As %ListOfObjects, verbose As %Boolean = 0, ByRef params) As %Status [ Private ] { set sc = $$$OK try { diff --git a/tests/integration_tests/Test/PM/Integration/Sync.cls b/tests/integration_tests/Test/PM/Integration/Sync.cls index 218752cf9..b4e5befa2 100644 --- a/tests/integration_tests/Test/PM/Integration/Sync.cls +++ b/tests/integration_tests/Test/PM/Integration/Sync.cls @@ -473,6 +473,29 @@ Method TestFailedCompileRetries() do $$$AssertStatusOK(sc, "Sync succeeds after fixing syntax error (retry works)") } +/// Editing module.xml and a source class in the same operation must sync BOTH changes. +/// The StampModule call in the moduleXmlChanged branch must not clobber existing baseline +/// rows for files edited at the same time — if it does, the class change is lost silently. +Method TestModuleXmlAndClassEditedTogether() +{ + set moduleXmlPath = ..TempDir _ "module.xml" + set clsPath = ..TempDir _ "src/cls/SyncTest/SuperClass.cls" + + // Edit both files before running sync + do ..ReplaceInFile(moduleXmlPath, "1.0.0", "1.0.1") + do ..ReplaceInFile(clsPath, """original""", """modified-with-xml""") + + kill params + set params("Verbose") = 1 + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + + do $$$AssertStatusOK(sc, "Sync after co-editing module.xml and a class succeeds") + do $$$AssertTrue(..FindInOutput(.output, "Sync complete"), "Sync reports changes (not 'Nothing to sync')") + do $$$AssertTrue(..FindInOutput(.output, "SuperClass.cls"), "SuperClass.cls appears in sync output as updated") +} + /// Returns the path to a fixture directory under _data/. /// Matches the convention from Test.PM.Integration.Base.GetModuleDir. ClassMethod GetModuleDir(subfolders... As %String) As %String diff --git a/tests/unit_tests/Test/PM/Unit/FileHash.cls b/tests/unit_tests/Test/PM/Unit/FileHash.cls new file mode 100644 index 000000000..1a2d31d8c --- /dev/null +++ b/tests/unit_tests/Test/PM/Unit/FileHash.cls @@ -0,0 +1,35 @@ +Class Test.PM.Unit.FileHash Extends %UnitTest.TestCase +{ + +Method TestNormalizePath() +{ + do $$$AssertEquals(##class(%IPM.Storage.FileHash).NormalizePath("src\cls\Foo\Bar.cls"), "src/cls/Foo/Bar.cls", "Backslashes -> forward slashes") + do $$$AssertEquals(##class(%IPM.Storage.FileHash).NormalizePath("src//cls///Foo.cls"), "src/cls/Foo.cls", "Consecutive slashes collapsed") + do $$$AssertEquals(##class(%IPM.Storage.FileHash).NormalizePath("/src/cls/Foo.cls"), "src/cls/Foo.cls", "Leading slash stripped") + do $$$AssertEquals(##class(%IPM.Storage.FileHash).NormalizePath("//src/Foo.cls"), "src/Foo.cls", "Multiple leading slashes stripped") + do $$$AssertEquals(##class(%IPM.Storage.FileHash).NormalizePath("src/cls/Foo.cls"), "src/cls/Foo.cls", "Already-normalized path unchanged") +} + +Method TestRelPathToDocName() +{ + // Standard layouts + do $$$AssertEquals(##class(%IPM.Storage.FileHash).RelPathToDocName("src/cls/Foo/Bar.cls"), "Foo.Bar.CLS", "Standard src/cls/ layout") + do $$$AssertEquals(##class(%IPM.Storage.FileHash).RelPathToDocName("src/inc/SyncTest.inc"), "SyncTest.INC", "Include file") + do $$$AssertEquals(##class(%IPM.Storage.FileHash).RelPathToDocName("src/mac/MyRoutine.mac"), "MyRoutine.MAC", "MAC routine") + do $$$AssertEquals(##class(%IPM.Storage.FileHash).RelPathToDocName("src/int/MyRoutine.int"), "MyRoutine.INT", "INT routine") + + // Non-compilable files return "" so they are never routed as server-side documents + do $$$AssertEquals(##class(%IPM.Storage.FileHash).RelPathToDocName("static/config.txt"), "", "Non-compilable extension") + do $$$AssertEquals(##class(%IPM.Storage.FileHash).RelPathToDocName("module.xml"), "", "module.xml") + + // Flat layout: 'SyncFlat' is not a prefix dir, becomes the package component + do $$$AssertEquals(##class(%IPM.Storage.FileHash).RelPathToDocName("src/SyncFlat/Flat.cls"), "SyncFlat.Flat.CLS", "Flat layout (no cls/ subdir)") + + // Edge cases + do $$$AssertEquals(##class(%IPM.Storage.FileHash).RelPathToDocName("Foo.cls"), "Foo.CLS", "Single-segment path") + do $$$AssertEquals(##class(%IPM.Storage.FileHash).RelPathToDocName("src/cls/A/B/C/Deep.cls"), "A.B.C.Deep.CLS", "Deep package hierarchy") + // IPM own classes: callers handle %-prefix namespace fallback; this method doesn't add % + do $$$AssertEquals(##class(%IPM.Storage.FileHash).RelPathToDocName("src/cls/IPM/Main.cls"), "IPM.Main.CLS", "IPM class — no % prefix added") +} + +} From 6f43b4ed4931f2cf683ed4fe9677d0b057551d61 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Wed, 22 Jul 2026 11:32:42 -0400 Subject: [PATCH 21/39] Only support canonical paths --- src/cls/IPM/ResourceProcessor/Abstract.cls | 16 +- .../ResourceProcessor/Default/Document.cls | 22 ++ src/cls/IPM/ResourceProcessor/Test.cls | 27 +-- src/cls/IPM/Storage/FileHash.cls | 213 +++++++++++------- src/cls/IPM/Storage/Module.cls | 86 ++++--- .../Test/PM/Integration/Sync.cls | 42 +--- .../_data/sync-flat-test/module.xml | 12 - .../sync-flat-test/src/SyncFlat/Flat.cls | 6 - 8 files changed, 221 insertions(+), 203 deletions(-) delete mode 100644 tests/integration_tests/Test/PM/Integration/_data/sync-flat-test/module.xml delete mode 100644 tests/integration_tests/Test/PM/Integration/_data/sync-flat-test/src/SyncFlat/Flat.cls diff --git a/src/cls/IPM/ResourceProcessor/Abstract.cls b/src/cls/IPM/ResourceProcessor/Abstract.cls index c9f600ea4..38bee611a 100644 --- a/src/cls/IPM/ResourceProcessor/Abstract.cls +++ b/src/cls/IPM/ResourceProcessor/Abstract.cls @@ -218,17 +218,13 @@ Method SupportsSync() As %Boolean quit 0 } -/// Called by sync to discover files owned by this resource that aren't captured by -/// OnResolveChildren/OnItemRelativePath — e.g. a directory-scanned resource whose file set -/// isn't declared as individual module.xml resources. Output relPaths(relPath)="" relative to -/// the module root. allFiles(relPath)=fullPath contains the pre-walked module root — filter -/// from it via $order prefix scan rather than doing filesystem I/O. -/// Base implementation returns an empty relPaths (no files claimed). Only processors with -/// SupportsSync()=1 should override this — SyncBuildReverseIndex skips processors that -/// return 0 from SupportsSync(), so an override on a non-sync processor has no effect. -Method OnSyncResolveFiles(Output relPaths, ByRef allFiles) As %Status +/// Returns the directory (relative to module root, normalized, no leading/trailing slash) +/// that this processor owns on disk. Sync walks only declared directories rather than the +/// full module root. Base returns "" (no owned directory; files are declared individually +/// via OnItemRelativePath). Processors that own a directory override this. +Method GetSyncDirectory() As %String { - quit $$$OK + quit "" } /// Returns the path relative to the module root for item pItemName within this resource. diff --git a/src/cls/IPM/ResourceProcessor/Default/Document.cls b/src/cls/IPM/ResourceProcessor/Default/Document.cls index 4891c3a11..663a9eac6 100644 --- a/src/cls/IPM/ResourceProcessor/Default/Document.cls +++ b/src/cls/IPM/ResourceProcessor/Default/Document.cls @@ -512,6 +512,28 @@ Method OnItemRelativePath(pItemName As %String) As %String quit $select(..ResourceReference.Preload:"preload/",1:"")_ tSourceRoot _ $select(directory=$char(0):"", 1:directory _ "/") _ $translate($piece(pItemName,".",1,*-1),..FilenameTranslateIdentifier,..FilenameTranslateAssociator)_tFileExtension } +/// Returns the directory this resource owns on disk, relative to module root (no trailing slash). +/// For directory-style resources (Name starts with "/"), the Name IS the directory. +/// For normal doc resources, it's {SourcesRoot}/{Directory}. +Method GetSyncDirectory() As %String +{ + if ..LoadAsDirectory { + set dir = ##class(%IPM.Storage.FileHash).NormalizePath(..ResourceReference.Name) + } else { + set sourcesRoot = ..ResourceReference.Module.SourcesRoot + set dir = ..Directory + if dir = "" { + quit "" + } + if sourcesRoot '= "" { + set dir = ##class(%IPM.Storage.FileHash).NormalizePath(sourcesRoot _ "/" _ dir) + } else { + set dir = ##class(%IPM.Storage.FileHash).NormalizePath(dir) + } + } + quit dir +} + Method GetSourceControlInfo(Output pInfo As %IPM.ExtensionBase.SourceControl.ResourceInfo) As %Status { set pInfo = ##class(%IPM.ExtensionBase.SourceControl.ResourceInfo).%New() diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls index 8af8fd7b9..8f824a3e7 100644 --- a/src/cls/IPM/ResourceProcessor/Test.cls +++ b/src/cls/IPM/ResourceProcessor/Test.cls @@ -380,30 +380,11 @@ Method SupportsSync() As %Boolean quit 1 } -/// Enumerate test files on disk so sync can detect brand-new test classes. -/// ResolveChildren (via GetChildren's StudioOpenDialog query) only sees already-compiled -/// classes, so a test file that was never compiled — e.g. one just added to the test -/// directory — is otherwise invisible to ComputeChanges until something else compiles it -/// first. This live directory scan closes that gap. -Method OnSyncResolveFiles(Output relPaths, ByRef allFiles) As %Status +/// The test directory (resource Name, e.g. "/tests/unit/") is the owned scan directory. +/// Sync walks this directory directly so new test classes are detected before compilation. +Method GetSyncDirectory() As %String { - set sc = $$$OK - try { - set prefix = ##class(%IPM.Storage.FileHash).NormalizePath(..ResourceReference.Name _ "/") - set prefixLen = $length(prefix) - set relPath = prefix - for { - set relPath = $order(allFiles(relPath)) - quit:relPath="" - quit:($extract(relPath, 1, prefixLen) '= prefix) - if $$$lcase($piece(relPath, ".", *)) = "cls" { - set relPaths(relPath) = "" - } - } - } catch e { - set sc = e.AsStatus() - } - quit sc + quit ##class(%IPM.Storage.FileHash).NormalizePath(..ResourceReference.Name) } Method OnSync(ByRef modifiedPaths, ByRef deletedPaths, ByRef params, Output handled As %Boolean = 0) As %Status diff --git a/src/cls/IPM/Storage/FileHash.cls b/src/cls/IPM/Storage/FileHash.cls index 969a9dc64..6834e15dd 100644 --- a/src/cls/IPM/Storage/FileHash.cls +++ b/src/cls/IPM/Storage/FileHash.cls @@ -18,27 +18,51 @@ Index ModuleNameIndex On ModuleName; ForeignKey ModuleNameFK(ModuleName) References %IPM.Storage.Module(Name) [ OnDelete = cascade ]; -/// Stamp all tracked files for a module. Two passes: -/// Pass 1 walks the module root for compilable files (cls/inc/mac/int) present in the namespace. -/// Pass 2 calls OnSyncResolveFiles on each processor to stamp non-compilable tracked files (e.g. FileCopy). -/// Also stamps module.xml. +/// Stamp all tracked files for a module. +/// Collects scan directories from each resource processor via GetSyncDirectory(), deduplicates, +/// walks only those directories, then stamps compilable files present in the namespace. +/// Also stamps module.xml from the module root. ClassMethod StampModule(module As %IPM.Storage.Module) As %Status { set sc = $$$OK try { set root = ##class(%File).NormalizeDirectory(module.Root) - // Single walk+hash of the entire module root. - kill allFiles, allHashes - $$$ThrowOnError(..WalkAndHashFiles(root, root, .allFiles, .allHashes)) + // Collect and deduplicate scan directories from resource processors. + kill scanDirs + set orderedResourceList = module.GetOrderedResourceList() + set key = "" + for { + set resource = orderedResourceList.GetNext(.key) + quit:key="" + if '$isobject(resource.Processor) { + continue + } + set syncDir = resource.Processor.GetSyncDirectory() + if syncDir '= "" { + set scanDirs(syncDir) = "" + } + } + do ..DeduplicateScanDirs(.scanDirs) - // module.xml is always tracked. - set moduleXmlRel = "module.xml" - if $data(allHashes(moduleXmlRel)) { - $$$ThrowOnError(..StampOneFileWithHash(module.Name, moduleXmlRel, allHashes(moduleXmlRel))) + // Walk each declared directory. + kill allFiles, allHashes + $$$ThrowOnError(..WalkAndHashDirs(root, .scanDirs, .allFiles, .allHashes)) + + // module.xml is always tracked — hash it directly from root. + set moduleXmlPath = root _ "module.xml" + if ##class(%File).Exists(moduleXmlPath) { + set moduleXmlHash = $$$lcase(##class(%File).SHA1Hash(moduleXmlPath, 1)) + if moduleXmlHash '= "" { + $$$ThrowOnError(..StampOneFileWithHash(module.Name, "module.xml", moduleXmlHash)) + } } - // Pass 1: filter for compilable files present in the namespace. + // Stamp all compilable files found in scan dirs. + // Targeted walks only visit declared resource directories, so every compilable file + // found here belongs to the module and should be tracked. No namespace presence check + // needed — directory-based resources (e.g. test dirs) use paths that RelPathToDocName + // can't resolve to the correct class name, so a namespace check would wrongly skip them. set relPath = "" for { set relPath = $order(allFiles(relPath)) @@ -47,21 +71,6 @@ ClassMethod StampModule(module As %IPM.Storage.Module) As %Status if ",cls,inc,mac,int," '[ (","_ext_",") { continue } - set docName = ..RelPathToDocName(relPath) - if docName = "" { - continue - } - set docExt = $$$lcase($piece(docName, ".", *)) - if docExt = "cls" { - set className = $piece(docName, ".", 1, *-1) - if '$$$comClassDefined(className) { - continue - } - } else { - if '##class(%RoutineMgr).Exists(docName) { - continue - } - } set hash = $get(allHashes(relPath)) if hash = "" { continue @@ -71,35 +80,6 @@ ClassMethod StampModule(module As %IPM.Storage.Module) As %Status write !, "Warning: could not stamp ", relPath, ": ", $system.Status.GetOneErrorText(stampSC) } } - - // Pass 2: stamp non-compilable tracked files via OnSyncResolveFiles on each processor. - set orderedResourceList = module.GetOrderedResourceList() - set key = "" - for { - set resource = orderedResourceList.GetNext(.key) - quit:key="" - if '$isobject(resource.Processor) { - continue - } - kill syncOnlyPaths - set childSC = resource.Processor.OnSyncResolveFiles(.syncOnlyPaths, .allFiles) - if $$$ISERR(childSC) { - continue - } - set relPath = "" - for { - set relPath = $order(syncOnlyPaths(relPath)) - quit:relPath="" - set hash = $get(allHashes(relPath)) - if hash = "" { - continue - } - set stampSC = ..StampOneFileWithHash(module.Name, relPath, hash) - if $$$ISERR(stampSC) { - write !, "Warning: could not stamp ", relPath, ": ", $system.Status.GetOneErrorText(stampSC) - } - } - } } catch e { set sc = e.AsStatus() } @@ -338,25 +318,90 @@ ClassMethod NormalizePath(path As %String) As %String quit path } -/// Walk all files and compute SHA1 hashes in a single pass. -/// Returns files(relPath) = fullPath and hashes(relPath) = sha1hex (lowercase). -/// Uses Python os.walk + hashlib for speed; falls back to SQL BFS + %File.SHA1Hash. -ClassMethod WalkAndHashFiles(dir As %String, relativeToRoot As %String, Output files, Output hashes) As %Status +/// Walk each directory in scanDirs(relDir)="" under moduleRoot and hash all files. +/// Returns files(relPath)=fullPath and hashes(relPath)=sha1hex (lowercase). +/// Uses a single Python os.walk call across all directories for speed; falls back to SQL BFS. +ClassMethod WalkAndHashDirs(moduleRoot As %String, ByRef scanDirs, Output files, Output hashes) As %Status { - set sc = ..WalkAndHashFilesPython(dir, relativeToRoot, .files, .hashes) - if $$$ISERR(sc) { - set sc = ..WalkAndHashFilesSQL(dir, relativeToRoot, .files, .hashes) + set sc = $$$OK + kill files, hashes + try { + set moduleRoot = ##class(%File).NormalizeDirectory(moduleRoot) + + // Collect existing absolute directories to walk. + kill absDirList + set dirCount = 0 + set relDir = "" + for { + set relDir = $order(scanDirs(relDir)) + quit:relDir="" + set absDir = ##class(%File).NormalizeDirectory(moduleRoot _ relDir) + if ##class(%File).DirectoryExists(absDir) { + set dirCount = dirCount + 1 + set absDirList(dirCount) = absDir + } + } + quit:dirCount=0 + + // Single Python call across all directories — avoids per-dir interpreter overhead. + set dirsJson = "[" + for i = 1:1:dirCount { + if i > 1 { set dirsJson = dirsJson _ "," } + set dirsJson = dirsJson _ """" _ $replace(absDirList(i), "\", "\\") _ """" + } + set dirsJson = dirsJson _ "]" + + set walkSC = ..WalkAndHashFilesPython(dirsJson, moduleRoot, .files, .hashes) + if $$$ISERR(walkSC) { + // SQL fallback: walk each directory individually. + kill files, hashes + for i = 1:1:dirCount { + kill dirFiles, dirHashes + $$$ThrowOnError(..WalkAndHashFilesSQL(absDirList(i), moduleRoot, .dirFiles, .dirHashes)) + merge files = dirFiles + merge hashes = dirHashes + } + } + } catch e { + set sc = e.AsStatus() } quit sc } -ClassMethod WalkAndHashFilesPython(dir As %String, relativeToRoot As %String, Output files, Output hashes) As %Status [ Private ] +/// Remove entries from scanDirs that are already covered by a shorter prefix. +/// Sorts alphabetically; skips any entry that starts with a previously-kept entry + "/". +ClassMethod DeduplicateScanDirs(ByRef scanDirs) +{ + kill kept + set relDir = "" + for { + set relDir = $order(scanDirs(relDir)) + quit:relDir="" + set covered = 0 + set keptDir = "" + for { + set keptDir = $order(kept(keptDir)) + quit:keptDir="" + set keptPrefix = ..NormalizePath(keptDir _ "/") + if $extract(relDir, 1, $length(keptPrefix)) = keptPrefix { + set covered = 1 + quit + } + } + if 'covered { + set kept(relDir) = "" + } + } + kill scanDirs + merge scanDirs = kept +} + +ClassMethod WalkAndHashFilesPython(dirsJson As %String, relativeToRoot As %String, Output files, Output hashes) As %Status [ Private ] { set sc = $$$OK try { - set dir = ##class(%File).NormalizeDirectory(dir) set relativeToRoot = ##class(%File).NormalizeDirectory(relativeToRoot) - set jsonStr = ..WalkAndHashFilesPythonImpl(dir, relativeToRoot) + set jsonStr = ..WalkAndHashFilesPythonImpl(dirsJson, relativeToRoot) set result = ##class(%DynamicArray).%FromJSON(jsonStr) set count = result.%Size() for i = 0:1:(count - 1) { @@ -371,7 +416,7 @@ ClassMethod WalkAndHashFilesPython(dir As %String, relativeToRoot As %String, Ou quit sc } -ClassMethod WalkAndHashFilesPythonImpl(dir As %String, relativeToRoot As %String) As %String [ Language = python ] +ClassMethod WalkAndHashFilesPythonImpl(dirsJson As %String, relativeToRoot As %String) As %String [ Language = python ] { import os import json @@ -379,24 +424,26 @@ import hashlib SKIP_DIRS = {'.git', '__pycache__', 'node_modules'} -results = [] +dirs = json.loads(dirsJson) root_len = len(relativeToRoot.rstrip(os.sep)) + 1 +results = [] -for dirpath, dirnames, filenames in os.walk(dir): - dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS] - for fname in filenames: - full_path = os.path.join(dirpath, fname) - rel_path = full_path[root_len:].replace('\\', '/') - if not rel_path: - continue - try: - h = hashlib.sha1() - with open(full_path, 'rb') as f: - while chunk := f.read(65536): - h.update(chunk) - results.append({"rel": rel_path, "full": full_path, "hash": h.hexdigest()}) - except (OSError, PermissionError): - results.append({"rel": rel_path, "full": full_path, "hash": ""}) +for dir in dirs: + for dirpath, dirnames, filenames in os.walk(dir): + dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS] + for fname in filenames: + full_path = os.path.join(dirpath, fname) + rel_path = full_path[root_len:].replace('\\', '/') + if not rel_path: + continue + try: + h = hashlib.sha1() + with open(full_path, 'rb') as f: + while chunk := f.read(65536): + h.update(chunk) + results.append({"rel": rel_path, "full": full_path, "hash": h.hexdigest()}) + except (OSError, PermissionError): + results.append({"rel": rel_path, "full": full_path, "hash": ""}) return json.dumps(results) } diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls index b65b7227a..5f695d649 100644 --- a/src/cls/IPM/Storage/Module.cls +++ b/src/cls/IPM/Storage/Module.cls @@ -770,13 +770,39 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status // manifestPaths. ComputeChanges Pass 2 finds those paths with no baseline row and // reports them as modified, so they are loaded in this same sync call. - // Step 2: Walk the entire module root and hash all files in one Python pass. - // This pre-walked data feeds the reverse index, change detection, and deletion checks — - // no further filesystem I/O needed from ObjectScript. + // Step 2: Collect scan directories from resource processors, then walk only those. + // Each processor declares its owned directory via GetSyncDirectory(); sync never + // touches files outside declared locations. + set orderedResourceList = module.GetOrderedResourceList() + kill scanDirs + set rlKey = "" + for { + set rlResource = orderedResourceList.GetNext(.rlKey) + quit:rlKey="" + if '$isobject(rlResource.Processor) { + continue + } + if 'rlResource.Processor.SupportsSync() { + continue + } + set syncDir = rlResource.Processor.GetSyncDirectory() + if syncDir '= "" { + set scanDirs(syncDir) = "" + } + } + do ##class(%IPM.Storage.FileHash).DeduplicateScanDirs(.scanDirs) + set walkStart = $zhorolog kill allFiles, allHashes, bfsFiles - $$$ThrowOnError(##class(%IPM.Storage.FileHash).WalkAndHashFiles(root, root, .allFiles, .allHashes)) - // Build a compilable-only subset for ComputeChanges Pass 1; count total for verbose. + $$$ThrowOnError(##class(%IPM.Storage.FileHash).WalkAndHashDirs(root, .scanDirs, .allFiles, .allHashes)) + + // module.xml is always tracked — add it to allFiles/allHashes explicitly. + if ##class(%File).Exists(root _ "module.xml") { + set allFiles("module.xml") = root _ "module.xml" + set allHashes("module.xml") = $$$lcase(##class(%File).SHA1Hash(root _ "module.xml", 1)) + } + + // Build compilable-only subset for ComputeChanges Pass 1; count total for verbose. set relPath = "", fileCount = 0 for { set relPath = $order(allFiles(relPath), 1, fullPath) @@ -788,11 +814,13 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status } } if verbose { - write !, "[", moduleName, "] Scanned ", fileCount, " file(s) in ", $fnumber($zhorolog - walkStart, "", 2), "s" + set dirCount = 0 + set tmpDir = "" + for { set tmpDir = $order(scanDirs(tmpDir)) quit:tmpDir="" set dirCount = dirCount + 1 } + write !, "[", moduleName, "] Scanned ", fileCount, " file(s) across ", dirCount, " director(ies) in ", $fnumber($zhorolog - walkStart, "", 2), "s" } // Step 3: Build reverse index (relPath -> owning resource + processor). - set orderedResourceList = module.GetOrderedResourceList() kill reverseIndex, unsupportedResources do ..SyncBuildReverseIndex(module, orderedResourceList, .reverseIndex, .unsupportedResources, .allFiles) @@ -944,8 +972,8 @@ ClassMethod SyncCheckModuleXml( /// Skips resources whose processor does not support sync — those are collected in /// unsupportedResources(resourceName)="" for informational display. /// -/// Step 1 builds docToResource (docName → owner) from ResolveChildren and calls -/// OnSyncResolveFiles for directory-based resources (e.g. test dirs) that have no doc name. +/// Step 1 builds docToResource (docName → owner) from ResolveChildren, and prefix-scans +/// allFiles for directory-owned resources (e.g. test dirs) via GetSyncDirectory(). /// Step 2 maps all stored baseline paths to owners via RelPathToDocName → docToResource. ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, orderedResourceList As %ListOfObjects, ByRef reverseIndex, Output unsupportedResources, ByRef allFiles) [ Private ] { @@ -997,30 +1025,28 @@ ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, orderedResource } } - // OnSyncResolveFiles returns filesystem-relative paths for directory-based resources - // (e.g. test directories). These have no document name, so they go directly into - // reverseIndex keyed by relPath. Passes pre-walked allFiles so processors filter - // in-memory instead of re-walking the filesystem. - kill syncOnlyPaths - set childSC = resource.Processor.OnSyncResolveFiles(.syncOnlyPaths, .allFiles) - if $$$ISERR(childSC) { - continue - } - set relPath = "" - for { - set relPath = $order(syncOnlyPaths(relPath)) - quit:relPath="" - set normalizedRelPath = ##class(%IPM.Storage.FileHash).NormalizePath(relPath) - set reverseIndex(normalizedRelPath) = resource.Name - set reverseIndex(normalizedRelPath, "Processor") = resource.Processor - set reverseIndex(normalizedRelPath, "Resource") = resource + // Directory-based resources (e.g. test dirs) own all files under their declared + // directory. Prefix-scan allFiles to map every file under that dir to this resource. + set syncDir = resource.Processor.GetSyncDirectory() + if syncDir '= "" { + set prefix = ##class(%IPM.Storage.FileHash).NormalizePath(syncDir _ "/") + set prefixLen = $length(prefix) + set dirRelPath = prefix + for { + set dirRelPath = $order(allFiles(dirRelPath)) + quit:dirRelPath="" + quit:($extract(dirRelPath, 1, prefixLen) '= prefix) + if '$data(reverseIndex(dirRelPath)) { + set reverseIndex(dirRelPath) = resource.Name + set reverseIndex(dirRelPath, "Processor") = resource.Processor + set reverseIndex(dirRelPath, "Resource") = resource + } + } } } - // Resolve compilable paths: every baseline path not already claimed by OnSyncResolveFiles - // above gets mapped through RelPathToDocName → docToResource. This handles both standard - // and non-standard directory layouts uniformly (the path on disk doesn't matter — only - // the derived document name needs to match what ResolveChildren reported). + // Resolve compilable paths: every baseline path not already claimed above gets mapped + // through RelPathToDocName → docToResource. kill storedPaths do ##class(%IPM.Storage.FileHash).GetStoredPaths(module.Name, .storedPaths) set relPath = "" diff --git a/tests/integration_tests/Test/PM/Integration/Sync.cls b/tests/integration_tests/Test/PM/Integration/Sync.cls index b4e5befa2..7bcfad78e 100644 --- a/tests/integration_tests/Test/PM/Integration/Sync.cls +++ b/tests/integration_tests/Test/PM/Integration/Sync.cls @@ -67,43 +67,6 @@ Method TestGetStoredPathsReturnsStampedPaths() do $$$AssertTrue($data(paths("src/inc/SyncTest.inc")), "SyncTest.inc in stored paths") } -/// Confirm sync detects changes in a module whose file layout doesn't match the -/// SourcesRoot/Directory/package.ext convention. sync-flat-test declares SyncFlat.Flat.CLS -/// with SourcesRoot=src, but the file is at src/SyncFlat/Flat.cls (no cls/ subdirectory). -/// OnItemRelativePath derives src/cls/SyncFlat/Flat.cls, which doesn't exist — so without -/// filesystem-anchored stamping, the file never gets a baseline row and sync always reports -/// "Nothing to sync" regardless of what changed. -// This test manages its own install/uninstall because sync-flat-test uses a separate fixture -// from sync-test and must not interfere with the per-test lifecycle. -Method TestNonStandardLayoutDetectsChange() -{ - set flatTempDir = ##class(%File).NormalizeDirectory(##class(%File).ManagerDirectory() _ "sync-flat-test-" _ $job) - try { - set flatSource = ..GetModuleDir("sync-flat-test") - $$$ThrowOnError(##class(%IPM.Utils.File).CreateDirectoryChain(flatTempDir)) - if '##class(%Library.File).CopyDir(flatSource, flatTempDir, 1) { - $$$ThrowOnError($$$ERROR($$$GeneralError, "Failed to copy sync-flat-test to temp dir")) - } - - $$$ThrowOnError(##class(%IPM.Main).Shell("load " _ flatTempDir _ " -dev")) - - set filePath = flatTempDir _ "src/SyncFlat/Flat.cls" - do ..ReplaceInFile(filePath, "As %String", "As %Integer") - - kill params - set params("Verbose") = 1 - do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Storage.Module).Sync("sync-flat-test", .params) - do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) - - do $$$AssertStatusOK(sc, "Sync succeeds for non-standard layout module") - do $$$AssertTrue(..FindInOutput(.output, "Sync complete"), "Sync detects the file change (not 'Nothing to sync')") - } catch e { - do $$$AssertStatusOK(e.AsStatus(), "TestNonStandardLayoutDetectsChange threw unexpectedly") - } - do ##class(%IPM.Main).Shell("uninstall sync-flat-test") - do ##class(%Library.File).RemoveDirectoryTree(flatTempDir) -} /// Sync with no files changed on disk since the last load/sync reports nothing to do. Method TestNoChangeIsNoOp() @@ -205,8 +168,9 @@ Method TestIncludeEditRecompilesConsumer() do $$$AssertEquals(##class(SyncTest.Consumer).GetMacroValue(), "modified-include", "Consumer reflects updated macro value after include sync") } -/// A file outside every tracked resource (e.g. under node_modules/) is ignored by sync -/// and never given a baseline row. +/// A file outside every tracked resource directory (e.g. under node_modules/) is ignored +/// by sync and never given a baseline row — sync only walks declared resource directories, +/// so files in undeclared locations are never seen. Method TestUntrackedFileIgnored() { set junkDir = ..TempDir _ "node_modules/" diff --git a/tests/integration_tests/Test/PM/Integration/_data/sync-flat-test/module.xml b/tests/integration_tests/Test/PM/Integration/_data/sync-flat-test/module.xml deleted file mode 100644 index 6513b0747..000000000 --- a/tests/integration_tests/Test/PM/Integration/_data/sync-flat-test/module.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - sync-flat-test - 1.0.0 - module - src - - - - diff --git a/tests/integration_tests/Test/PM/Integration/_data/sync-flat-test/src/SyncFlat/Flat.cls b/tests/integration_tests/Test/PM/Integration/_data/sync-flat-test/src/SyncFlat/Flat.cls deleted file mode 100644 index 9d8d222e0..000000000 --- a/tests/integration_tests/Test/PM/Integration/_data/sync-flat-test/src/SyncFlat/Flat.cls +++ /dev/null @@ -1,6 +0,0 @@ -Class SyncFlat.Flat -{ - -Property Value As %String; - -} From 6ab5ab7db6a2e2039e7a19b8364f2bb4bc9282ab Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Wed, 22 Jul 2026 11:41:23 -0400 Subject: [PATCH 22/39] Hash in parallel --- src/cls/IPM/Storage/FileHash.cls | 43 ++++++++++++++++++++------------ src/cls/IPM/Storage/Module.cls | 4 +-- 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/src/cls/IPM/Storage/FileHash.cls b/src/cls/IPM/Storage/FileHash.cls index 6834e15dd..55d33c530 100644 --- a/src/cls/IPM/Storage/FileHash.cls +++ b/src/cls/IPM/Storage/FileHash.cls @@ -421,31 +421,42 @@ ClassMethod WalkAndHashFilesPythonImpl(dirsJson As %String, relativeToRoot As %S import os import json import hashlib +import concurrent.futures SKIP_DIRS = {'.git', '__pycache__', 'node_modules'} dirs = json.loads(dirsJson) root_len = len(relativeToRoot.rstrip(os.sep)) + 1 -results = [] -for dir in dirs: - for dirpath, dirnames, filenames in os.walk(dir): - dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS] +# Walk all directories first, collecting (rel, full) pairs. +file_list = [] +for d in dirs: + for dirpath, dirnames, filenames in os.walk(d): + dirnames[:] = [x for x in dirnames if x not in SKIP_DIRS] for fname in filenames: full_path = os.path.join(dirpath, fname) rel_path = full_path[root_len:].replace('\\', '/') - if not rel_path: - continue - try: - h = hashlib.sha1() - with open(full_path, 'rb') as f: - while chunk := f.read(65536): - h.update(chunk) - results.append({"rel": rel_path, "full": full_path, "hash": h.hexdigest()}) - except (OSError, PermissionError): - results.append({"rel": rel_path, "full": full_path, "hash": ""}) - -return json.dumps(results) + if rel_path: + file_list.append((rel_path, full_path)) + +def hash_file(full_path): + try: + h = hashlib.sha1() + with open(full_path, 'rb') as f: + while chunk := f.read(65536): + h.update(chunk) + return h.hexdigest() + except (OSError, PermissionError): + return "" + +# Hash in parallel — SHA1 releases the GIL, so threads overlap I/O wait. +with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + hash_results = list(executor.map(hash_file, [f for _, f in file_list])) + +return json.dumps([ + {"rel": rel, "full": full, "hash": h} + for (rel, full), h in zip(file_list, hash_results) +]) } /// SQL-based BFS fallback that walks and hashes without Python. diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls index 5f695d649..c3af949f6 100644 --- a/src/cls/IPM/Storage/Module.cls +++ b/src/cls/IPM/Storage/Module.cls @@ -855,7 +855,7 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status if verbose && $data(unsupportedResources) { do ..SyncPrintUnsupportedNote(moduleName, .unsupportedResources) } - write !, "[", moduleName, "] Done in ", $fnumber($zhorolog - syncStart, "", 2), "s" + write !, "[", moduleName, "] Sync done in ", $fnumber($zhorolog - syncStart, "", 2), "s" quit } @@ -929,7 +929,7 @@ ClassMethod Sync(moduleName As %String, ByRef params) As %Status do ..SyncPrintUnsupportedNote(moduleName, .unsupportedResources) } - write !, "[", moduleName, "] Done in ", $fnumber($zhorolog - syncStart, "", 2), "s" + write !, "[", moduleName, "] Sync done in ", $fnumber($zhorolog - syncStart, "", 2), "s" // Step 10: Run changed test-phase tests if -test flag is set (after sync is reported) if runTests { From ae6b5e1511654aa4b72db94c2e5e40c47ac4576e Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Wed, 22 Jul 2026 13:55:30 -0400 Subject: [PATCH 23/39] Refactor sync to be an actual lifecycle phase --- src/cls/IPM/DataType/PhaseName.cls | 2 +- src/cls/IPM/Lifecycle/Base.cls | 632 ++++++++++++++++++ src/cls/IPM/Main.cls | 4 +- src/cls/IPM/Storage/FileHash.cls | 8 +- src/cls/IPM/Storage/Module.cls | 631 ----------------- .../Test/PM/Integration/Sync.cls | 88 +-- 6 files changed, 664 insertions(+), 701 deletions(-) diff --git a/src/cls/IPM/DataType/PhaseName.cls b/src/cls/IPM/DataType/PhaseName.cls index 2107dd277..53098385f 100644 --- a/src/cls/IPM/DataType/PhaseName.cls +++ b/src/cls/IPM/DataType/PhaseName.cls @@ -10,6 +10,6 @@ Parameter MAXLEN As INTEGER = 50; /// If a non-null value is present, then the attribute is restricted to values /// in the list, and the validation code simply checks to see if the value is in the list. /// Should be kept in sync with %IPM.Lifecycle.Base.Phases -Parameter VALUELIST = ",Clean,Initialize,Reload,*,Validate,ExportData,Compile,Activate,Document,MakeDeployed,Test,Package,Verify,Publish,Configure,Unconfigure,ApplyUpdateSteps"; +Parameter VALUELIST = ",Clean,Initialize,Reload,*,Validate,ExportData,Compile,Activate,Document,MakeDeployed,Test,Package,Verify,Publish,Configure,Unconfigure,ApplyUpdateSteps,Sync"; } diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls index b43c9b102..1bd0fdd36 100644 --- a/src/cls/IPM/Lifecycle/Base.cls +++ b/src/cls/IPM/Lifecycle/Base.cls @@ -183,6 +183,7 @@ ClassMethod GetCompletePhasesForOne(pOnePhase As %String) As %List "applyupdatesteps": $listbuild("Initialize","Reload","*","Validate","Compile","Activate","ApplyUpdateSteps"), "configure": $listbuild("Configure"), "unconfigure": $listbuild("Unconfigure"), + "sync": $listbuild("Sync"), : "" ) } @@ -209,6 +210,7 @@ ClassMethod MatchSinglePhase(pOnePhase As %String) As %String "configure": "Configure", "unconfigure": "Unconfigure", "applyupdatesteps": "ApplyUpdateSteps", + "sync": "Sync", : pOnePhase // return the phase as-is if it's a custom phase name ) } @@ -570,6 +572,636 @@ Method %Unconfigure(ByRef pParams) As %Status quit tSC } +/// Incrementally sync changed files from disk into the namespace for a dev-mode module. +/// Detects files changed since last load/sync, routes them to resource processors, and recompiles. +Method %Sync(ByRef params) As %Status +{ + set sc = $$$OK + try { + set syncStart = $zhorolog + set verbose = $get(params("Verbose"), 0) + set processDeletes = $get(params("ProcessDeletes"), 0) + set runTests = $get(params("RunTests"), 0) + + // params may be reused across modules by Main.Sync's sync-all-dev-mode-modules loop. + // Clear this module's own scratch subtree so a prior module's recorded test-case + // changes can't leak into this module's SyncRunTests dispatch. + kill params("Sync") + + // The lifecycle already opened and validated this module; use it directly. + // SyncCheckModuleXml may replace this local with a freshly-reloaded instance. + set module = ..Module + set moduleName = module.Name + + if 'module.DeveloperMode { + $$$ThrowStatus($$$ERROR($$$GeneralError, $$$FormatText("Module '%1' is not in development mode. Sync requires dev mode.", moduleName))) + } + + set root = ##class(%File).NormalizeDirectory(module.Root) + if root = "" { + $$$ThrowStatus($$$ERROR($$$GeneralError, $$$FormatText("Module '%1' has no root directory configured.", moduleName))) + } + + // Step 1: Check if module.xml changed; reload manifest if so + set moduleXmlRelPath = ##class(%IPM.Storage.FileHash).NormalizePath("module.xml") + set moduleXmlPath = root _ "module.xml" + set moduleXmlChanged = ..SyncCheckModuleXml(.module, moduleXmlPath, moduleXmlRelPath) + + if '##class(%IPM.Storage.FileHash).HasBaseline(moduleName) { + // Self-heal: establish baseline for modules loaded before this feature + $$$ThrowOnError(##class(%IPM.Storage.FileHash).StampModule(module)) + write !, "[", moduleName, "] Baseline established. Run sync again to detect changes." + quit + } + + // No StampModule call here after a manifest reload: StampModule would overwrite current + // hashes for ALL files (including ones the user just edited), causing ComputeChanges to + // see current-vs-current and report zero changes for co-edited files. + // Newly-declared resources are handled without a full stamp: SyncBuildReverseIndex + // (step 3) calls ResolveChildren, which adds their derived relPaths to reverseIndex → + // manifestPaths. ComputeChanges Pass 2 finds those paths with no baseline row and + // reports them as modified, so they are loaded in this same sync call. + + // Step 2: Collect scan directories from resource processors, then walk only those. + // Each processor declares its owned directory via GetSyncDirectory(); sync never + // touches files outside declared locations. + set orderedResourceList = module.GetOrderedResourceList() + kill scanDirs + set rlKey = "" + for { + set rlResource = orderedResourceList.GetNext(.rlKey) + quit:rlKey="" + if '$isobject(rlResource.Processor) { + continue + } + if 'rlResource.Processor.SupportsSync() { + continue + } + set syncDir = rlResource.Processor.GetSyncDirectory() + if syncDir '= "" { + set scanDirs(syncDir) = "" + } + } + do ##class(%IPM.Storage.FileHash).DeduplicateScanDirs(.scanDirs) + + set walkStart = $zhorolog + kill allFiles, allHashes, bfsFiles + $$$ThrowOnError(##class(%IPM.Storage.FileHash).WalkAndHashDirs(root, .scanDirs, .allFiles, .allHashes)) + + // module.xml is always tracked — add it to allFiles/allHashes explicitly. + if ##class(%File).Exists(root _ "module.xml") { + set allFiles("module.xml") = root _ "module.xml" + set allHashes("module.xml") = $$$lcase(##class(%File).SHA1Hash(root _ "module.xml", 1)) + } + + // Build compilable-only subset for ComputeChanges Pass 1; count total for verbose. + set relPath = "", fileCount = 0 + for { + set relPath = $order(allFiles(relPath), 1, fullPath) + quit:relPath="" + set fileCount = fileCount + 1 + set ext = $$$lcase($piece(relPath, ".", *)) + if ",cls,inc,mac,int," [ (","_ext_",") { + set bfsFiles(relPath) = fullPath + } + } + if verbose { + set dirCount = 0 + set tmpDir = "" + for { set tmpDir = $order(scanDirs(tmpDir)) quit:tmpDir="" set dirCount = dirCount + 1 } + write !, "[", moduleName, "] Scanned ", fileCount, " file(s) across ", dirCount, " director(ies) in ", $fnumber($zhorolog - walkStart, "", 2), "s" + } + + // Step 3: Build reverse index (relPath -> owning resource + processor). + kill reverseIndex, unsupportedResources + do ..SyncBuildReverseIndex(module, orderedResourceList, .reverseIndex, .unsupportedResources, .allFiles) + + // Collect manifest-derived paths from reverseIndex for ComputeChanges. + kill manifestPaths + set riCount = 0 + set riKey = "" + for { + set riKey = $order(reverseIndex(riKey)) + quit:riKey="" + set manifestPaths(riKey) = "" + set riCount = riCount + 1 + } + if verbose { + write !, "[", moduleName, "] Tracking ", riCount, " path(s) across ", orderedResourceList.Count(), " resource(s)" + } + + // Step 4: Compute disk changes vs baseline + $$$ThrowOnError(##class(%IPM.Storage.FileHash).ComputeChanges(module, .modified, .deleted, .manifestPaths, .bfsFiles, .allFiles, .allHashes)) + + // Exclude module.xml from file routing (handled by SyncCheckModuleXml above) + kill modified(moduleXmlRelPath) + kill deleted(moduleXmlRelPath) + + // Falls through when only deletes exist and processDeletes=1 + if '$data(modified) && ('$data(deleted) || 'processDeletes) { + write !, "[", moduleName, "] Nothing to sync." + if moduleXmlChanged { + do ..SyncCommitModuleXml(module, moduleXmlPath, moduleXmlRelPath) + do ..SyncPrintModuleXmlWarning(moduleName) + } + if verbose && $data(unsupportedResources) { + do ..SyncPrintUnsupportedNote(moduleName, .unsupportedResources) + } + write !, "[", moduleName, "] Sync done in ", $fnumber($zhorolog - syncStart, "", 2), "s" + quit + } + + // Step 5: Partition changes by owning resource + kill syncByResource + do ..SyncRoutePathSet(.modified, .reverseIndex, "modified", .syncByResource) + if processDeletes { + do ..SyncRoutePathSet(.deleted, .reverseIndex, "deleted", .syncByResource) + } + + // Step 6: Dispatch OnSync to each processor; load unhandled compilable files + if verbose { + set resCount = 0 + set resName = "" + for { + set resName = $order(syncByResource(resName)) + quit:resName="" + set resCount = resCount + 1 + } + write !, "[", moduleName, "] Dispatching to ", resCount, " resource(s)" + } + $$$ThrowOnError(..SyncDispatchProcessors(module, root, verbose, .syncByResource, .params, .loadItems)) + + // Step 7: Compile the full resource set with u-flag to pick up dependent recompiles + if loadItems > 0 { + $$$ThrowOnError(..SyncCompile(module, orderedResourceList, verbose, .params)) + } + + // Step 8: Delete server-side documents for removed files, then recompile + if processDeletes && ($data(deleted) > 1) { + do ..SyncApplyDeletes(.deleted, .reverseIndex, .syncByResource, verbose) + $$$ThrowOnError(..SyncCompile(module, orderedResourceList, verbose, .params)) + } + + // Step 9: Commit new hashes on success (skipped on error so next sync re-detects). + // Done before running tests below so a test failure doesn't prevent the file-sync + // outcome from being committed and reported — file sync and test results are independent. + $$$ThrowOnError(##class(%IPM.Storage.FileHash).CommitChanges(module, .modified, .deleted, processDeletes)) + if moduleXmlChanged { + do ..SyncCommitModuleXml(module, moduleXmlPath, moduleXmlRelPath) + } + + set modCount = 0 + set key = "" + for { + set key = $order(modified(key)) + quit:key="" + set modCount = modCount + 1 + write !, " Updated: ", key + } + set delCount = 0 + if processDeletes { + set key = "" + for { + set key = $order(deleted(key)) + quit:key="" + set delCount = delCount + 1 + write !, " Deleted: ", key + } + } + write !, "[", moduleName, "] Sync complete: ", modCount, " file(s) updated" + if delCount > 0 { + write ", ", delCount, " deleted" + } + write "." + + if moduleXmlChanged { + do ..SyncPrintModuleXmlWarning(moduleName) + } + if verbose && $data(unsupportedResources) { + do ..SyncPrintUnsupportedNote(moduleName, .unsupportedResources) + } + + write !, "[", moduleName, "] Sync done in ", $fnumber($zhorolog - syncStart, "", 2), "s" + + // Step 10: Run changed test-phase tests if -test flag is set (after sync is reported) + if runTests { + $$$ThrowOnError(..SyncRunTests(orderedResourceList, verbose, .params)) + } + + } catch e { + set sc = e.AsStatus() + } + quit sc +} + +/// Check if module.xml changed and reload the manifest if so. +/// Returns 1 if module.xml changed and was reloaded, 0 otherwise. +/// On reload, replaces module with the freshly-opened instance. +ClassMethod SyncCheckModuleXml( + ByRef module As %IPM.Storage.Module, + moduleXmlPath As %String, + moduleXmlRelPath As %String) As %Boolean [ Private ] +{ + if '##class(%File).Exists(moduleXmlPath) { + quit 0 + } + set existing = ##class(%IPM.Storage.FileHash).ModulePathIndexOpen(module.Name, moduleXmlRelPath) + if '$isobject(existing) { + quit 0 + } + set newHash = $$$lcase(##class(%File).SHA1Hash(moduleXmlPath, 1)) + if newHash = existing.Hash { + quit 0 + } + $$$ThrowOnError($system.OBJ.Load(moduleXmlPath, "-d")) + set module = ##class(%IPM.Storage.Module).NameOpen(module.Name, , .openSC) + $$$ThrowOnError(openSC) + quit 1 +} + +/// Build a reverse index: normalizedRelPath -> resource name, Processor, Resource object. +/// Used by SyncRoutePathSet to map changed files back to their owning resource processors. +/// Skips resources whose processor does not support sync — those are collected in +/// unsupportedResources(resourceName)="" for informational display. +/// +/// Step 1 builds docToResource (docName → owner) from ResolveChildren, and prefix-scans +/// allFiles for directory-owned resources (e.g. test dirs) via GetSyncDirectory(). +/// Step 2 maps all stored baseline paths to owners via RelPathToDocName → docToResource. +ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, orderedResourceList As %ListOfObjects, ByRef reverseIndex, Output unsupportedResources, ByRef allFiles) [ Private ] +{ + kill unsupportedResources + // docToResource maps server document names (e.g. "SyncTest.SuperClass.CLS") to the + // resource that owns them. This lets us resolve filesystem paths → owners in O(1) below, + // since RelPathToDocName converts a relPath to a docName deterministically. + kill docToResource + set key = "" + for { + set resource = orderedResourceList.GetNext(.key) + quit:key="" + + if '$isobject(resource.Processor) { + continue + } + + if 'resource.Processor.SupportsSync() { + set unsupportedResources(resource.Name) = "" + continue + } + + // ResolveChildren returns childArr keyed by document name (server-side identifier). + kill childArr + set childSC = resource.ResolveChildren(.childArr) + if $$$ISERR(childSC) { + continue + } + + set childName = "" + for { + set childName = $order(childArr(childName)) + quit:childName="" + set docToResource(childName) = resource.Name + set docToResource(childName, "Processor") = resource.Processor + set docToResource(childName, "Resource") = resource + + // Newly-declared resources have no baseline row yet (StampModule skips uncompiled + // classes), so GetStoredPaths below won't find them. Map their relPath directly. + set relPath = $get(childArr(childName, "RelativePath")) + if relPath = "" { + set relPath = resource.Processor.OnItemRelativePath(childName) + } + if relPath '= "" { + set normalizedRelPath = ##class(%IPM.Storage.FileHash).NormalizePath(relPath) + set reverseIndex(normalizedRelPath) = resource.Name + set reverseIndex(normalizedRelPath, "Processor") = resource.Processor + set reverseIndex(normalizedRelPath, "Resource") = resource + } + } + + // Directory-based resources (e.g. test dirs) own all files under their declared + // directory. Prefix-scan allFiles to map every file under that dir to this resource. + set syncDir = resource.Processor.GetSyncDirectory() + if syncDir '= "" { + set prefix = ##class(%IPM.Storage.FileHash).NormalizePath(syncDir _ "/") + set prefixLen = $length(prefix) + set dirRelPath = prefix + for { + set dirRelPath = $order(allFiles(dirRelPath)) + quit:dirRelPath="" + quit:($extract(dirRelPath, 1, prefixLen) '= prefix) + if '$data(reverseIndex(dirRelPath)) { + set reverseIndex(dirRelPath) = resource.Name + set reverseIndex(dirRelPath, "Processor") = resource.Processor + set reverseIndex(dirRelPath, "Resource") = resource + } + } + } + } + + // Resolve compilable paths: every baseline path not already claimed above gets mapped + // through RelPathToDocName → docToResource. + kill storedPaths + do ##class(%IPM.Storage.FileHash).GetStoredPaths(module.Name, .storedPaths) + set relPath = "" + for { + set relPath = $order(storedPaths(relPath)) + quit:relPath="" + + if $data(reverseIndex(relPath)) { + continue + } + + set docName = ##class(%IPM.Storage.FileHash).RelPathToDocName(relPath) + if docName = "" { + continue + } + if '$data(docToResource(docName)) { + continue + } + + set reverseIndex(relPath) = docToResource(docName) + set reverseIndex(relPath, "Processor") = docToResource(docName, "Processor") + set reverseIndex(relPath, "Resource") = docToResource(docName, "Resource") + } +} + + +/// Route a set of changed paths to their owning resources, categorized by type (modified/deleted). +/// Only sync-supporting resources appear in reverseIndex, so no filtering needed here. +ClassMethod SyncRoutePathSet( + ByRef paths, + ByRef reverseIndex, + category As %String, + ByRef syncByResource) [ Private ] +{ + set relPath = "" + for { + set relPath = $order(paths(relPath)) + quit:relPath="" + + if '$data(reverseIndex(relPath)) { + continue + } + set resName = reverseIndex(relPath) + set syncByResource(resName, category, relPath) = "" + set syncByResource(resName, "Processor") = reverseIndex(relPath, "Processor") + set syncByResource(resName, "Resource") = reverseIndex(relPath, "Resource") + } +} + +/// Call OnSync on each processor, then load any unhandled compilable files. +/// Returns the number of files loaded (used to decide whether SyncCompile is needed). +/// If a processor sets handled=1 in OnSync, it fully owns the sync for that resource +/// (e.g. Test reloads its own way). Otherwise the default path loads+compiles each file. +ClassMethod SyncDispatchProcessors( + module As %IPM.Storage.Module, + root As %String, + verbose As %Boolean, + ByRef syncByResource, + ByRef params, + Output loadItems As %Integer = 0) As %Status [ Private ] +{ + set sc = $$$OK + try { + set resName = "" + for { + set resName = $order(syncByResource(resName)) + quit:resName="" + + if '$data(syncByResource(resName, "Processor")) { + continue + } + set processor = syncByResource(resName, "Processor") + + kill resModified, resDeleted + merge resModified = syncByResource(resName, "modified") + merge resDeleted = syncByResource(resName, "deleted") + + set handled = 0 + $$$ThrowOnError(processor.OnSync(.resModified, .resDeleted, .params, .handled)) + + if handled || 'processor.%IsA("%IPM.ResourceProcessor.AbstractCompilable") { + continue + } + + // Default: load each changed file; SyncCompile will do a full compile with u-flag + set loadRelPath = "" + for { + set loadRelPath = $order(resModified(loadRelPath)) + quit:loadRelPath="" + + set fullPath = ##class(%File).NormalizeFilename(loadRelPath, root) + if ##class(%File).Exists(fullPath) { + set loadFlags = $select(verbose:"d", 1:"-d") + $$$ThrowOnError($system.OBJ.Load(fullPath, loadFlags _ "c")) + set loadItems = loadItems + 1 + } + } + } + } catch e { + set sc = e.AsStatus() + } + quit sc +} + +/// Delete server-side documents for compilable deleted files. +/// Guards skip paths with no owner, no processor, or non-compilable processors — those are +/// handled elsewhere (unsupported warnings) or aren't server-side documents at all. +ClassMethod SyncApplyDeletes( + ByRef deleted, + ByRef reverseIndex, + ByRef syncByResource, + verbose As %Boolean) [ Private ] +{ + set relPath = "" + for { + set relPath = $order(deleted(relPath)) + quit:relPath="" + + if '$data(reverseIndex(relPath)) { + continue + } + set resName = reverseIndex(relPath) + if '$data(syncByResource(resName, "Processor")) { + continue + } + set processor = syncByResource(resName, "Processor") + if 'processor.%IsA("%IPM.ResourceProcessor.AbstractCompilable") { + continue + } + set docName = ##class(%IPM.Storage.FileHash).RelPathToDocName(relPath) + if docName '= "" { + set delFlags = $select(verbose:"d", 1:"-d") + set delSC = $system.OBJ.Delete(docName, delFlags) + // Delete failures are non-fatal: the SyncCompile pass that immediately follows + // will fail to compile any class that still references the deleted doc, surfacing + // the error with full context. Aborting the delete loop here would leave other + // deletions unapplied and make the overall error harder to diagnose. + if $$$ISERR(delSC) { + write !, "Warning: could not delete ", docName, ": ", $system.Status.GetOneErrorText(delSC) + } + } + } +} + +/// Run test-phase tests for changed test case classes recorded in params("Sync","ChangedTestCases"). +/// Groups changed classes by owning resource first, then dispatches one batched RunTest call +/// per resource (via OnSyncRunTests) instead of one call per class — avoids N separate +/// reload/compile/run cycles when several test classes in the same resource changed together. +ClassMethod SyncRunTests( + orderedResourceList As %ListOfObjects, + verbose As %Boolean, + ByRef params) As %Status [ Private ] +{ + set sc = $$$OK + try { + // Step 1: group changed classes by owning resource: byResource(resourceName, className) = "" + kill byResource + set className = "" + for { + set className = $order(params("Sync", "ChangedTestCases", className), 1, owningResource) + quit:className="" + set byResource(owningResource, className) = "" + } + + // Step 2: one OnSyncRunTests call per resource + set testKey = "" + for { + set testResource = orderedResourceList.GetNext(.testKey) + quit:testKey="" + + if '$isobject(testResource.Processor) { + continue + } + if 'testResource.Processor.%IsA("%IPM.ResourceProcessor.Test") { + continue + } + if '$data(byResource(testResource.Name)) { + continue + } + if '$listfind(testResource.Processor.Phase, "test") { + set skipClassName = "" + for { + set skipClassName = $order(byResource(testResource.Name, skipClassName)) + quit:skipClassName="" + write:verbose !, "Skipping verify-scoped test: ", skipClassName, " (use 'verify' to run)" + } + continue + } + kill classInfo + merge classInfo = byResource(testResource.Name) + kill testParams + merge testParams = params + set testParams("DeveloperMode") = 1 + $$$ThrowOnError(testResource.Processor.OnSyncRunTests(.classInfo, .testParams)) + } + } catch e { + set sc = e.AsStatus() + } + quit sc +} + +/// Commit the current module.xml hash to the baseline. +ClassMethod SyncCommitModuleXml( + module As %IPM.Storage.Module, + moduleXmlPath As %String, + moduleXmlRelPath As %String) [ Private ] +{ + kill moduleXmlMod, emptyDel + set moduleXmlMod(moduleXmlRelPath) = $$$lcase(##class(%File).SHA1Hash(moduleXmlPath, 1)) + set commitSC = ##class(%IPM.Storage.FileHash).CommitChanges(module, .moduleXmlMod, .emptyDel, 0) + if $$$ISERR(commitSC) { + write !, "Warning: failed to record module.xml hash: ", $system.Status.GetOneErrorText(commitSC) + } +} + +ClassMethod SyncPrintModuleXmlWarning(moduleName As %String) [ Private ] +{ + write ! + write !, "Warning: module.xml changed and was reloaded." + write !, " Resources may have been added/removed. Run `reload ", moduleName, "` to fully apply" + write !, " manifest-level changes (mappings, dependencies, defaults)." +} + +ClassMethod SyncPrintUnsupportedNote(moduleName As %String, ByRef unsupportedResources) [ Private ] +{ + set count = 0 + set names = "" + set resName = "" + for { + set resName = $order(unsupportedResources(resName)) + quit:resName="" + set count = count + 1 + if count <= 3 { + set names = names _ $select(names="":"", 1:", ") _ resName + } + } + if count > 3 { + set names = names _ ", ... (" _ (count - 3) _ " more)" + } + write !, "[", moduleName, "] ", count, " resource(s) skipped (no sync support): ", names + write !, " Use `reload ", moduleName, "` to apply changes to those resources." +} + +/// Recompile all compilable resources in the module to catch dependents invalidated by +/// changes loaded in SyncDispatchProcessors. Skips CompileFromProject resources — those are +/// deployed code loaded via Studio project files (.prj); compiling them individually would +/// fail because deployed classes have no source in the routine database. +ClassMethod SyncCompile(module As %IPM.Storage.Module, orderedResourceList As %ListOfObjects, verbose As %Boolean = 0, ByRef params) As %Status [ Private ] +{ + set sc = $$$OK + try { + kill compileArray + + set key = "" + for { + set resource = orderedResourceList.GetNext(.key) + quit:key="" + + if '$isobject(resource.Processor) { + continue + } + if 'resource.Processor.%IsA("%IPM.ResourceProcessor.AbstractCompilable") { + continue + } + if 'resource.IsInScope("Compile") { + continue + } + if resource.Processor.CompileFromProject { + continue + } + + kill oneResourceList + set oneResourceList(resource.Name) = "" + set resSC = resource.ResolveChildren(.oneResourceList) + if $$$ISERR(resSC) { + continue + } + + set childKey = "" + for { + set childKey = $order(oneResourceList(childKey)) + quit:childKey="" + set ext = $zconvert($piece(childKey, ".", *), "U") + if ext = "CLS" { + set className = $piece(childKey, ".", 1, *-1) + if '$$$comClassDefined(className) { + continue + } + } + set compileArray(childKey) = "" + } + } + + if $data(compileArray) > 1 { + set flags = $select(verbose:"d", 1:"-d") _ "cku" + set sc = ##class(%IPM.Utils.LegacyCompat).UpdateSuperclassAndCompile(.compileArray, .flags) + } + } catch e { + set sc = e.AsStatus() + } + quit sc +} + Method %Initialize(ByRef pParams) As %Status { set status = $$$OK diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls index 18d9cf03a..dc5179dce 100644 --- a/src/cls/IPM/Main.cls +++ b/src/cls/IPM/Main.cls @@ -2331,7 +2331,7 @@ ClassMethod Sync(ByRef commandInfo) [ Private ] merge params = commandInfo("data") if moduleName '= "" { - $$$ThrowOnError(##class(%IPM.Storage.Module).Sync(moduleName, .params)) + $$$ThrowOnError(##class(%IPM.Storage.Module).ExecutePhases(moduleName, $listbuild("Sync"), 1, .params)) } else { // Sync all dev-mode modules set result = ##class(%SQL.Statement).%ExecDirect(, @@ -2349,7 +2349,7 @@ ClassMethod Sync(ByRef commandInfo) [ Private ] } set found = found + 1 set name = result.%Get("Name") - set syncSC = ##class(%IPM.Storage.Module).Sync(name, .params) + set syncSC = ##class(%IPM.Storage.Module).ExecutePhases(name, $listbuild("Sync"), 1, .params) if $$$ISERR(syncSC) { set failures = failures _ $listbuild(name) do $system.OBJ.DisplayError(syncSC) diff --git a/src/cls/IPM/Storage/FileHash.cls b/src/cls/IPM/Storage/FileHash.cls index 55d33c530..829958349 100644 --- a/src/cls/IPM/Storage/FileHash.cls +++ b/src/cls/IPM/Storage/FileHash.cls @@ -68,7 +68,7 @@ ClassMethod StampModule(module As %IPM.Storage.Module) As %Status set relPath = $order(allFiles(relPath)) quit:relPath="" set ext = $$$lcase($piece(relPath, ".", *)) - if ",cls,inc,mac,int," '[ (","_ext_",") { + if ",cls,inc,mac,int,xml,rtn," '[ (","_ext_",") { continue } set hash = $get(allHashes(relPath)) @@ -428,7 +428,7 @@ SKIP_DIRS = {'.git', '__pycache__', 'node_modules'} dirs = json.loads(dirsJson) root_len = len(relativeToRoot.rstrip(os.sep)) + 1 -# Walk all directories first, collecting (rel, full) pairs. +# Walk all directories, collecting all files. file_list = [] for d in dirs: for dirpath, dirnames, filenames in os.walk(d): @@ -449,8 +449,8 @@ def hash_file(full_path): except (OSError, PermissionError): return "" -# Hash in parallel — SHA1 releases the GIL, so threads overlap I/O wait. -with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: +# Hash all files in parallel — SHA1 releases the GIL, so threads overlap I/O wait. +with concurrent.futures.ThreadPoolExecutor(max_workers=24) as executor: hash_results = list(executor.map(hash_file, [f for _, f in file_list])) return json.dumps([ diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls index c3af949f6..9f0ad18ac 100644 --- a/src/cls/IPM/Storage/Module.cls +++ b/src/cls/IPM/Storage/Module.cls @@ -719,637 +719,6 @@ ClassMethod ExecutePhases( quit tSC } -/// Incrementally sync changed files from disk into the namespace for a dev-mode module. -/// Detects files changed since last load/sync, routes them to resource processors, and recompiles. -ClassMethod Sync(moduleName As %String, ByRef params) As %Status -{ - set sc = $$$OK - try { - set syncStart = $zhorolog - set verbose = $get(params("Verbose"), 0) - set processDeletes = $get(params("ProcessDeletes"), 0) - set runTests = $get(params("RunTests"), 0) - - // params may be reused across modules by Main.Sync's sync-all-dev-mode-modules loop. - // Clear this module's own scratch subtree so a prior module's recorded test-case - // changes can't leak into this module's SyncRunTests dispatch. - kill params("Sync") - - set module = ..NameOpen(moduleName, , .sc) - if '$isobject(module) { - $$$ThrowStatus($$$ERROR($$$GeneralError, $$$FormatText("Module '%1' not found.", moduleName))) - } - $$$ThrowOnError(sc) - - if 'module.DeveloperMode { - $$$ThrowStatus($$$ERROR($$$GeneralError, $$$FormatText("Module '%1' is not in development mode. Sync requires dev mode.", moduleName))) - } - - set root = ##class(%File).NormalizeDirectory(module.Root) - if root = "" { - $$$ThrowStatus($$$ERROR($$$GeneralError, $$$FormatText("Module '%1' has no root directory configured.", moduleName))) - } - - // Step 1: Check if module.xml changed; reload manifest if so - set moduleXmlRelPath = ##class(%IPM.Storage.FileHash).NormalizePath("module.xml") - set moduleXmlPath = root _ "module.xml" - set moduleXmlChanged = ..SyncCheckModuleXml(.module, moduleXmlPath, moduleXmlRelPath) - - if '##class(%IPM.Storage.FileHash).HasBaseline(moduleName) { - // Self-heal: establish baseline for modules loaded before this feature - $$$ThrowOnError(##class(%IPM.Storage.FileHash).StampModule(module)) - write !, "[", moduleName, "] Baseline established. Run sync again to detect changes." - quit - } - - // No StampModule call here after a manifest reload: StampModule would overwrite current - // hashes for ALL files (including ones the user just edited), causing ComputeChanges to - // see current-vs-current and report zero changes for co-edited files. - // Newly-declared resources are handled without a full stamp: SyncBuildReverseIndex - // (step 3) calls ResolveChildren, which adds their derived relPaths to reverseIndex → - // manifestPaths. ComputeChanges Pass 2 finds those paths with no baseline row and - // reports them as modified, so they are loaded in this same sync call. - - // Step 2: Collect scan directories from resource processors, then walk only those. - // Each processor declares its owned directory via GetSyncDirectory(); sync never - // touches files outside declared locations. - set orderedResourceList = module.GetOrderedResourceList() - kill scanDirs - set rlKey = "" - for { - set rlResource = orderedResourceList.GetNext(.rlKey) - quit:rlKey="" - if '$isobject(rlResource.Processor) { - continue - } - if 'rlResource.Processor.SupportsSync() { - continue - } - set syncDir = rlResource.Processor.GetSyncDirectory() - if syncDir '= "" { - set scanDirs(syncDir) = "" - } - } - do ##class(%IPM.Storage.FileHash).DeduplicateScanDirs(.scanDirs) - - set walkStart = $zhorolog - kill allFiles, allHashes, bfsFiles - $$$ThrowOnError(##class(%IPM.Storage.FileHash).WalkAndHashDirs(root, .scanDirs, .allFiles, .allHashes)) - - // module.xml is always tracked — add it to allFiles/allHashes explicitly. - if ##class(%File).Exists(root _ "module.xml") { - set allFiles("module.xml") = root _ "module.xml" - set allHashes("module.xml") = $$$lcase(##class(%File).SHA1Hash(root _ "module.xml", 1)) - } - - // Build compilable-only subset for ComputeChanges Pass 1; count total for verbose. - set relPath = "", fileCount = 0 - for { - set relPath = $order(allFiles(relPath), 1, fullPath) - quit:relPath="" - set fileCount = fileCount + 1 - set ext = $$$lcase($piece(relPath, ".", *)) - if ",cls,inc,mac,int," [ (","_ext_",") { - set bfsFiles(relPath) = fullPath - } - } - if verbose { - set dirCount = 0 - set tmpDir = "" - for { set tmpDir = $order(scanDirs(tmpDir)) quit:tmpDir="" set dirCount = dirCount + 1 } - write !, "[", moduleName, "] Scanned ", fileCount, " file(s) across ", dirCount, " director(ies) in ", $fnumber($zhorolog - walkStart, "", 2), "s" - } - - // Step 3: Build reverse index (relPath -> owning resource + processor). - kill reverseIndex, unsupportedResources - do ..SyncBuildReverseIndex(module, orderedResourceList, .reverseIndex, .unsupportedResources, .allFiles) - - // Collect manifest-derived paths from reverseIndex for ComputeChanges. - kill manifestPaths - set riCount = 0 - set riKey = "" - for { - set riKey = $order(reverseIndex(riKey)) - quit:riKey="" - set manifestPaths(riKey) = "" - set riCount = riCount + 1 - } - if verbose { - write !, "[", moduleName, "] Tracking ", riCount, " path(s) across ", orderedResourceList.Count(), " resource(s)" - } - - // Step 4: Compute disk changes vs baseline - $$$ThrowOnError(##class(%IPM.Storage.FileHash).ComputeChanges(module, .modified, .deleted, .manifestPaths, .bfsFiles, .allFiles, .allHashes)) - - // Exclude module.xml from file routing (handled by SyncCheckModuleXml above) - kill modified(moduleXmlRelPath) - kill deleted(moduleXmlRelPath) - - // Falls through when only deletes exist and processDeletes=1 - if '$data(modified) && ('$data(deleted) || 'processDeletes) { - write !, "[", moduleName, "] Nothing to sync." - if moduleXmlChanged { - do ..SyncCommitModuleXml(module, moduleXmlPath, moduleXmlRelPath) - do ..SyncPrintModuleXmlWarning(moduleName) - } - if verbose && $data(unsupportedResources) { - do ..SyncPrintUnsupportedNote(moduleName, .unsupportedResources) - } - write !, "[", moduleName, "] Sync done in ", $fnumber($zhorolog - syncStart, "", 2), "s" - quit - } - - // Step 5: Partition changes by owning resource - kill syncByResource - do ..SyncRoutePathSet(.modified, .reverseIndex, "modified", .syncByResource) - if processDeletes { - do ..SyncRoutePathSet(.deleted, .reverseIndex, "deleted", .syncByResource) - } - - // Step 6: Dispatch OnSync to each processor; load unhandled compilable files - if verbose { - set resCount = 0 - set resName = "" - for { - set resName = $order(syncByResource(resName)) - quit:resName="" - set resCount = resCount + 1 - } - write !, "[", moduleName, "] Dispatching to ", resCount, " resource(s)" - } - $$$ThrowOnError(..SyncDispatchProcessors(module, root, verbose, .syncByResource, .params, .loadItems)) - - // Step 7: Compile the full resource set with u-flag to pick up dependent recompiles - if loadItems > 0 { - $$$ThrowOnError(..SyncCompile(module, orderedResourceList, verbose, .params)) - } - - // Step 8: Delete server-side documents for removed files, then recompile - if processDeletes && ($data(deleted) > 1) { - do ..SyncApplyDeletes(.deleted, .reverseIndex, .syncByResource, verbose) - $$$ThrowOnError(..SyncCompile(module, orderedResourceList, verbose, .params)) - } - - // Step 9: Commit new hashes on success (skipped on error so next sync re-detects). - // Done before running tests below so a test failure doesn't prevent the file-sync - // outcome from being committed and reported — file sync and test results are independent. - $$$ThrowOnError(##class(%IPM.Storage.FileHash).CommitChanges(module, .modified, .deleted, processDeletes)) - if moduleXmlChanged { - do ..SyncCommitModuleXml(module, moduleXmlPath, moduleXmlRelPath) - } - - set modCount = 0 - set key = "" - for { - set key = $order(modified(key)) - quit:key="" - set modCount = modCount + 1 - write !, " Updated: ", key - } - set delCount = 0 - if processDeletes { - set key = "" - for { - set key = $order(deleted(key)) - quit:key="" - set delCount = delCount + 1 - write !, " Deleted: ", key - } - } - write !, "[", moduleName, "] Sync complete: ", modCount, " file(s) updated" - if delCount > 0 { - write ", ", delCount, " deleted" - } - write "." - - if moduleXmlChanged { - do ..SyncPrintModuleXmlWarning(moduleName) - } - if verbose && $data(unsupportedResources) { - do ..SyncPrintUnsupportedNote(moduleName, .unsupportedResources) - } - - write !, "[", moduleName, "] Sync done in ", $fnumber($zhorolog - syncStart, "", 2), "s" - - // Step 10: Run changed test-phase tests if -test flag is set (after sync is reported) - if runTests { - $$$ThrowOnError(..SyncRunTests(orderedResourceList, verbose, .params)) - } - - } catch e { - set sc = e.AsStatus() - } - quit sc -} - -/// Check if module.xml changed and reload the manifest if so. -/// Returns 1 if module.xml changed and was reloaded, 0 otherwise. -/// On reload, replaces module with the freshly-opened instance. -ClassMethod SyncCheckModuleXml( - ByRef module As %IPM.Storage.Module, - moduleXmlPath As %String, - moduleXmlRelPath As %String) As %Boolean [ Private ] -{ - if '##class(%File).Exists(moduleXmlPath) { - quit 0 - } - set existing = ##class(%IPM.Storage.FileHash).ModulePathIndexOpen(module.Name, moduleXmlRelPath) - if '$isobject(existing) { - quit 0 - } - set newHash = $$$lcase(##class(%File).SHA1Hash(moduleXmlPath, 1)) - if newHash = existing.Hash { - quit 0 - } - $$$ThrowOnError($system.OBJ.Load(moduleXmlPath, "-d")) - set module = ..NameOpen(module.Name, , .openSC) - $$$ThrowOnError(openSC) - quit 1 -} - -/// Build a reverse index: normalizedRelPath -> resource name, Processor, Resource object. -/// Used by SyncRoutePathSet to map changed files back to their owning resource processors. -/// Skips resources whose processor does not support sync — those are collected in -/// unsupportedResources(resourceName)="" for informational display. -/// -/// Step 1 builds docToResource (docName → owner) from ResolveChildren, and prefix-scans -/// allFiles for directory-owned resources (e.g. test dirs) via GetSyncDirectory(). -/// Step 2 maps all stored baseline paths to owners via RelPathToDocName → docToResource. -ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, orderedResourceList As %ListOfObjects, ByRef reverseIndex, Output unsupportedResources, ByRef allFiles) [ Private ] -{ - kill unsupportedResources - // docToResource maps server document names (e.g. "SyncTest.SuperClass.CLS") to the - // resource that owns them. This lets us resolve filesystem paths → owners in O(1) below, - // since RelPathToDocName converts a relPath to a docName deterministically. - kill docToResource - set key = "" - for { - set resource = orderedResourceList.GetNext(.key) - quit:key="" - - if '$isobject(resource.Processor) { - continue - } - - if 'resource.Processor.SupportsSync() { - set unsupportedResources(resource.Name) = "" - continue - } - - // ResolveChildren returns childArr keyed by document name (server-side identifier). - kill childArr - set childSC = resource.ResolveChildren(.childArr) - if $$$ISERR(childSC) { - continue - } - - set childName = "" - for { - set childName = $order(childArr(childName)) - quit:childName="" - set docToResource(childName) = resource.Name - set docToResource(childName, "Processor") = resource.Processor - set docToResource(childName, "Resource") = resource - - // Newly-declared resources have no baseline row yet (StampModule skips uncompiled - // classes), so GetStoredPaths below won't find them. Map their relPath directly. - set relPath = $get(childArr(childName, "RelativePath")) - if relPath = "" { - set relPath = resource.Processor.OnItemRelativePath(childName) - } - if relPath '= "" { - set normalizedRelPath = ##class(%IPM.Storage.FileHash).NormalizePath(relPath) - set reverseIndex(normalizedRelPath) = resource.Name - set reverseIndex(normalizedRelPath, "Processor") = resource.Processor - set reverseIndex(normalizedRelPath, "Resource") = resource - } - } - - // Directory-based resources (e.g. test dirs) own all files under their declared - // directory. Prefix-scan allFiles to map every file under that dir to this resource. - set syncDir = resource.Processor.GetSyncDirectory() - if syncDir '= "" { - set prefix = ##class(%IPM.Storage.FileHash).NormalizePath(syncDir _ "/") - set prefixLen = $length(prefix) - set dirRelPath = prefix - for { - set dirRelPath = $order(allFiles(dirRelPath)) - quit:dirRelPath="" - quit:($extract(dirRelPath, 1, prefixLen) '= prefix) - if '$data(reverseIndex(dirRelPath)) { - set reverseIndex(dirRelPath) = resource.Name - set reverseIndex(dirRelPath, "Processor") = resource.Processor - set reverseIndex(dirRelPath, "Resource") = resource - } - } - } - } - - // Resolve compilable paths: every baseline path not already claimed above gets mapped - // through RelPathToDocName → docToResource. - kill storedPaths - do ##class(%IPM.Storage.FileHash).GetStoredPaths(module.Name, .storedPaths) - set relPath = "" - for { - set relPath = $order(storedPaths(relPath)) - quit:relPath="" - - if $data(reverseIndex(relPath)) { - continue - } - - set docName = ##class(%IPM.Storage.FileHash).RelPathToDocName(relPath) - if docName = "" { - continue - } - if '$data(docToResource(docName)) { - continue - } - - set reverseIndex(relPath) = docToResource(docName) - set reverseIndex(relPath, "Processor") = docToResource(docName, "Processor") - set reverseIndex(relPath, "Resource") = docToResource(docName, "Resource") - } -} - - -/// Route a set of changed paths to their owning resources, categorized by type (modified/deleted). -/// Only sync-supporting resources appear in reverseIndex, so no filtering needed here. -ClassMethod SyncRoutePathSet( - ByRef paths, - ByRef reverseIndex, - category As %String, - ByRef syncByResource) [ Private ] -{ - set relPath = "" - for { - set relPath = $order(paths(relPath)) - quit:relPath="" - - if '$data(reverseIndex(relPath)) { - continue - } - set resName = reverseIndex(relPath) - set syncByResource(resName, category, relPath) = "" - set syncByResource(resName, "Processor") = reverseIndex(relPath, "Processor") - set syncByResource(resName, "Resource") = reverseIndex(relPath, "Resource") - } -} - -/// Call OnSync on each processor, then load any unhandled compilable files. -/// Returns the number of files loaded (used to decide whether SyncCompile is needed). -/// If a processor sets handled=1 in OnSync, it fully owns the sync for that resource -/// (e.g. Test reloads its own way). Otherwise the default path loads+compiles each file. -ClassMethod SyncDispatchProcessors( - module As %IPM.Storage.Module, - root As %String, - verbose As %Boolean, - ByRef syncByResource, - ByRef params, - Output loadItems As %Integer = 0) As %Status [ Private ] -{ - set sc = $$$OK - try { - set resName = "" - for { - set resName = $order(syncByResource(resName)) - quit:resName="" - - if '$data(syncByResource(resName, "Processor")) { - continue - } - set processor = syncByResource(resName, "Processor") - - kill resModified, resDeleted - merge resModified = syncByResource(resName, "modified") - merge resDeleted = syncByResource(resName, "deleted") - - set handled = 0 - $$$ThrowOnError(processor.OnSync(.resModified, .resDeleted, .params, .handled)) - - if handled || 'processor.%IsA("%IPM.ResourceProcessor.AbstractCompilable") { - continue - } - - // Default: load each changed file; SyncCompile will do a full compile with u-flag - set loadRelPath = "" - for { - set loadRelPath = $order(resModified(loadRelPath)) - quit:loadRelPath="" - - set fullPath = ##class(%File).NormalizeFilename(loadRelPath, root) - if ##class(%File).Exists(fullPath) { - set loadFlags = $select(verbose:"d", 1:"-d") - $$$ThrowOnError($system.OBJ.Load(fullPath, loadFlags _ "c")) - set loadItems = loadItems + 1 - } - } - } - } catch e { - set sc = e.AsStatus() - } - quit sc -} - -/// Delete server-side documents for compilable deleted files. -/// Guards skip paths with no owner, no processor, or non-compilable processors — those are -/// handled elsewhere (unsupported warnings) or aren't server-side documents at all. -ClassMethod SyncApplyDeletes( - ByRef deleted, - ByRef reverseIndex, - ByRef syncByResource, - verbose As %Boolean) [ Private ] -{ - set relPath = "" - for { - set relPath = $order(deleted(relPath)) - quit:relPath="" - - if '$data(reverseIndex(relPath)) { - continue - } - set resName = reverseIndex(relPath) - if '$data(syncByResource(resName, "Processor")) { - continue - } - set processor = syncByResource(resName, "Processor") - if 'processor.%IsA("%IPM.ResourceProcessor.AbstractCompilable") { - continue - } - set docName = ##class(%IPM.Storage.FileHash).RelPathToDocName(relPath) - if docName '= "" { - set delFlags = $select(verbose:"d", 1:"-d") - set delSC = $system.OBJ.Delete(docName, delFlags) - // Delete failures are non-fatal: the SyncCompile pass that immediately follows - // will fail to compile any class that still references the deleted doc, surfacing - // the error with full context. Aborting the delete loop here would leave other - // deletions unapplied and make the overall error harder to diagnose. - if $$$ISERR(delSC) { - write !, "Warning: could not delete ", docName, ": ", $system.Status.GetOneErrorText(delSC) - } - } - } -} - -/// Run test-phase tests for changed test case classes recorded in params("Sync","ChangedTestCases"). -/// Groups changed classes by owning resource first, then dispatches one batched RunTest call -/// per resource (via OnSyncRunTests) instead of one call per class — avoids N separate -/// reload/compile/run cycles when several test classes in the same resource changed together. -ClassMethod SyncRunTests( - orderedResourceList As %ListOfObjects, - verbose As %Boolean, - ByRef params) As %Status [ Private ] -{ - set sc = $$$OK - try { - // Step 1: group changed classes by owning resource: byResource(resourceName, className) = "" - kill byResource - set className = "" - for { - set className = $order(params("Sync", "ChangedTestCases", className), 1, owningResource) - quit:className="" - set byResource(owningResource, className) = "" - } - - // Step 2: one OnSyncRunTests call per resource - set testKey = "" - for { - set testResource = orderedResourceList.GetNext(.testKey) - quit:testKey="" - - if '$isobject(testResource.Processor) { - continue - } - if 'testResource.Processor.%IsA("%IPM.ResourceProcessor.Test") { - continue - } - if '$data(byResource(testResource.Name)) { - continue - } - if '$listfind(testResource.Processor.Phase, "test") { - set skipClassName = "" - for { - set skipClassName = $order(byResource(testResource.Name, skipClassName)) - quit:skipClassName="" - write:verbose !, "Skipping verify-scoped test: ", skipClassName, " (use 'verify' to run)" - } - continue - } - kill classInfo - merge classInfo = byResource(testResource.Name) - kill testParams - merge testParams = params - set testParams("DeveloperMode") = 1 - $$$ThrowOnError(testResource.Processor.OnSyncRunTests(.classInfo, .testParams)) - } - } catch e { - set sc = e.AsStatus() - } - quit sc -} - -/// Commit the current module.xml hash to the baseline. -ClassMethod SyncCommitModuleXml( - module As %IPM.Storage.Module, - moduleXmlPath As %String, - moduleXmlRelPath As %String) [ Private ] -{ - kill moduleXmlMod, emptyDel - set moduleXmlMod(moduleXmlRelPath) = $$$lcase(##class(%File).SHA1Hash(moduleXmlPath, 1)) - set commitSC = ##class(%IPM.Storage.FileHash).CommitChanges(module, .moduleXmlMod, .emptyDel, 0) - if $$$ISERR(commitSC) { - write !, "Warning: failed to record module.xml hash: ", $system.Status.GetOneErrorText(commitSC) - } -} - -ClassMethod SyncPrintModuleXmlWarning(moduleName As %String) [ Private ] -{ - write ! - write !, "Warning: module.xml changed and was reloaded." - write !, " Resources may have been added/removed. Run `reload ", moduleName, "` to fully apply" - write !, " manifest-level changes (mappings, dependencies, defaults)." -} - -ClassMethod SyncPrintUnsupportedNote(moduleName As %String, ByRef unsupportedResources) [ Private ] -{ - set count = 0 - set names = "" - set resName = "" - for { - set resName = $order(unsupportedResources(resName)) - quit:resName="" - set count = count + 1 - if count <= 3 { - set names = names _ $select(names="":"", 1:", ") _ resName - } - } - if count > 3 { - set names = names _ ", ... (" _ (count - 3) _ " more)" - } - write !, "[", moduleName, "] ", count, " resource(s) skipped (no sync support): ", names - write !, " Use `reload ", moduleName, "` to apply changes to those resources." -} - -/// Recompile all compilable resources in the module to catch dependents invalidated by -/// changes loaded in SyncDispatchProcessors. Skips CompileFromProject resources — those are -/// deployed code loaded via Studio project files (.prj); compiling them individually would -/// fail because deployed classes have no source in the routine database. -ClassMethod SyncCompile(module As %IPM.Storage.Module, orderedResourceList As %ListOfObjects, verbose As %Boolean = 0, ByRef params) As %Status [ Private ] -{ - set sc = $$$OK - try { - kill compileArray - - set key = "" - for { - set resource = orderedResourceList.GetNext(.key) - quit:key="" - - if '$isobject(resource.Processor) { - continue - } - if 'resource.Processor.%IsA("%IPM.ResourceProcessor.AbstractCompilable") { - continue - } - if 'resource.IsInScope("Compile") { - continue - } - if resource.Processor.CompileFromProject { - continue - } - - kill oneResourceList - set oneResourceList(resource.Name) = "" - set resSC = resource.ResolveChildren(.oneResourceList) - if $$$ISERR(resSC) { - continue - } - - set childKey = "" - for { - set childKey = $order(oneResourceList(childKey)) - quit:childKey="" - set ext = $zconvert($piece(childKey, ".", *), "U") - if ext = "CLS" { - set className = $piece(childKey, ".", 1, *-1) - if '$$$comClassDefined(className) { - continue - } - } - set compileArray(childKey) = "" - } - } - - if $data(compileArray) > 1 { - set flags = $select(verbose:"d", 1:"-d") _ "cku" - set sc = ##class(%IPM.Utils.LegacyCompat).UpdateSuperclassAndCompile(.compileArray, .flags) - } - } catch e { - set sc = e.AsStatus() - } - quit sc -} - /// Uninstalls a named module (pModuleName). /// May optionally force installation (uninstalling even if required by other modules) if pForce is 1. /// May optionally recurse to also uninstall dependencies that are not required by other modules if pRecurse is 1. diff --git a/tests/integration_tests/Test/PM/Integration/Sync.cls b/tests/integration_tests/Test/PM/Integration/Sync.cls index 7bcfad78e..701a46b9d 100644 --- a/tests/integration_tests/Test/PM/Integration/Sync.cls +++ b/tests/integration_tests/Test/PM/Integration/Sync.cls @@ -73,10 +73,8 @@ Method TestNoChangeIsNoOp() { do $$$AssertTrue(##class(%IPM.Storage.FileHash).HasBaseline("sync-test"), "Baseline rows exist after load") - kill params - set params("Verbose") = 1 do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + set sc = ##class(%IPM.Main).Shell("sync sync-test -verbose") do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) do $$$AssertStatusOK(sc, "Sync with no changes succeeds") @@ -95,10 +93,8 @@ Method TestMigrationFromNoBaseline() set filePath = ..TempDir _ "src/cls/SyncTest/SuperClass.cls" do ..ReplaceInFile(filePath, """original""", """modified""") - kill params - set params("Verbose") = 1 do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + set sc = ##class(%IPM.Main).Shell("sync sync-test -verbose") do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) do $$$AssertStatusOK(sc, "First sync after baseline loss succeeds") @@ -107,20 +103,18 @@ Method TestMigrationFromNoBaseline() // The edit made before the self-heal is captured in the new baseline (not retroactively // detected), so a second sync with no further changes reports nothing to do. - kill params2, output2 - set params2("Verbose") = 1 + kill output2 do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie2) - set sc2 = ##class(%IPM.Storage.Module).Sync("sync-test", .params2) + set sc2 = ##class(%IPM.Main).Shell("sync sync-test -verbose") do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie2, .output2) do $$$AssertStatusOK(sc2, "Second sync succeeds") do $$$AssertTrue(..FindInOutput(.output2, "Nothing to sync"), "Second sync finds no changes (baseline already reflects the modified file)") // A genuinely new edit after the baseline is established is detected normally. do ..ReplaceInFile(filePath, """modified""", """modified again""") - kill params3, output3 - set params3("Verbose") = 1 + kill output3 do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie3) - set sc3 = ##class(%IPM.Storage.Module).Sync("sync-test", .params3) + set sc3 = ##class(%IPM.Main).Shell("sync sync-test -verbose") do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie3, .output3) do $$$AssertStatusOK(sc3, "Third sync succeeds") do $$$AssertTrue(..FindInOutput(.output3, "Sync complete"), "Third sync detects the post-baseline edit normally") @@ -132,10 +126,8 @@ Method TestModifiedClassRecompiles() set filePath = ..TempDir _ "src/cls/SyncTest/SuperClass.cls" do ..ReplaceInFile(filePath, """original""", """modified""") - kill params - set params("Verbose") = 1 do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + set sc = ##class(%IPM.Main).Shell("sync sync-test -verbose") do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) do $$$AssertStatusOK(sc, "Sync after class modification succeeds") @@ -150,8 +142,7 @@ Method TestSuperclassEditRecompilesSubclass() set filePath = ..TempDir _ "src/cls/SyncTest/SuperClass.cls" do ..ReplaceInFile(filePath, "Property BaseValue", "Property NewProp As %String;" _ $char(10) _ $char(10) _ "Property BaseValue") - kill params - set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + set sc = ##class(%IPM.Main).Shell("sync sync-test") do $$$AssertStatusOK(sc, "Sync after superclass edit succeeds (subclass recompiles via u-flag)") } @@ -162,8 +153,7 @@ Method TestIncludeEditRecompilesConsumer() set incPath = ..TempDir _ "src/inc/SyncTest.inc" do ..ReplaceInFile(incPath, """original-include""", """modified-include""") - kill params - set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + set sc = ##class(%IPM.Main).Shell("sync sync-test") do $$$AssertStatusOK(sc, "Sync after include edit succeeds") do $$$AssertEquals(##class(SyncTest.Consumer).GetMacroValue(), "modified-include", "Consumer reflects updated macro value after include sync") } @@ -181,10 +171,8 @@ Method TestUntrackedFileIgnored() $$$ThrowOnError(stream.%Save()) set stream = "" - kill params - set params("Verbose") = 1 do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + set sc = ##class(%IPM.Main).Shell("sync sync-test -verbose") do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) do $$$AssertStatusOK(sc, "Sync ignores untracked files") @@ -200,15 +188,12 @@ Method TestDeleteSkippedByDefault() do ##class(%Library.File).Delete(filePath) // Without -delete: class still present - kill params - set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + set sc = ##class(%IPM.Main).Shell("sync sync-test") do $$$AssertStatusOK(sc, "Sync without -delete succeeds") do $$$AssertTrue($$$comClassDefined("SyncTest.Deletable"), "Deletable class still exists without -delete flag") // With -delete: class is removed from server - kill params - set params("ProcessDeletes") = 1 - set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + set sc = ##class(%IPM.Main).Shell("sync sync-test -delete") do $$$AssertStatusOK(sc, "Sync with -delete succeeds") do $$$AssertNotTrue($$$comClassDefined("SyncTest.Deletable"), "Deletable class removed after -delete sync") } @@ -223,9 +208,7 @@ Method TestDeleteRecompilesDependents() do $$$AssertTrue($$$comClassDefined("SyncTest.SubClass"), "SubClass still compiled before delete sync") - kill params - set params("ProcessDeletes") = 1 - set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + set sc = ##class(%IPM.Main).Shell("sync sync-test -delete") do $$$AssertStatusNotOK(sc, "Sync with -delete fails: SubClass now references a deleted superclass") do $$$AssertNotTrue($$$comClassDefined("SyncTest.SuperClass"), "SuperClass was removed from the server") @@ -238,10 +221,8 @@ Method TestModuleXmlChangedWarning() set filePath = ..TempDir _ "module.xml" do ..ReplaceInFile(filePath, "1.0.0", "1.0.1") - kill params - set params("Verbose") = 1 do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + set sc = ##class(%IPM.Main).Shell("sync sync-test -verbose") do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) do $$$AssertStatusOK(sc, "Sync after module.xml change succeeds") @@ -257,10 +238,8 @@ Method TestSyncTestFlag() do ..ReplaceInFile(filePath, "This test always passes.", "This test always passes (modified).") // Without RunTests: loads the changed test but does not execute it - kill params - set params("Verbose") = 1 do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + set sc = ##class(%IPM.Main).Shell("sync sync-test -verbose") do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) do $$$AssertStatusOK(sc, "Sync without RunTests loads but does not run tests") do $$$AssertNotTrue(..FindInOutput(.output, "Test Results"), "No test results without RunTests flag") @@ -269,11 +248,9 @@ Method TestSyncTestFlag() do ..ReplaceInFile(filePath, "(modified).", "(modified again).") // With RunTests: executes the changed test-phase test class - kill params, output - set params("RunTests") = 1 - set params("Verbose") = 1 + kill output do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + set sc = ##class(%IPM.Main).Shell("sync sync-test -test -verbose") do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) do $$$AssertStatusOK(sc, "Sync with RunTests runs changed tests") do $$$AssertTrue(..FindInOutput(.output, "Test Results"), "Test results shown with RunTests flag") @@ -298,11 +275,8 @@ Method TestSyncTestFlagBatchesMultipleChangedClasses() $$$ThrowOnError(stream.%Save()) set stream = "" - kill params - set params("RunTests") = 1 - set params("Verbose") = 1 do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + set sc = ##class(%IPM.Main).Shell("sync sync-test -test -verbose") do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) do $$$AssertStatusOK(sc, "Sync with two changed classes in one resource succeeds") @@ -334,11 +308,8 @@ Method TestSyncTestFlagOnlyRunsOwningResource() set filePath = ..TempDir _ "tests/unit/SyncTest/Tests/Trivial.cls" do ..ReplaceInFile(filePath, "This test always passes.", "This test always passes (modified).") - kill params - set params("RunTests") = 1 - set params("Verbose") = 1 do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + set sc = ##class(%IPM.Main).Shell("sync sync-test -test -verbose") do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) do $$$AssertStatusOK(sc, "Sync with RunTests succeeds") @@ -349,7 +320,6 @@ Method TestSyncTestFlagOnlyRunsOwningResource() /// Running sync with no module name syncs every module currently in development mode. Method TestSyncAllDevModeModules() { - kill params do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) set sc = ##class(%IPM.Main).Shell("sync") do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) @@ -361,8 +331,7 @@ Method TestSyncAllDevModeModules() /// Sync fails with an error when given a module name that isn't installed. Method TestSyncModuleNotFound() { - kill params - set sc = ##class(%IPM.Storage.Module).Sync("this-module-does-not-exist", .params) + set sc = ##class(%IPM.Main).Shell("sync this-module-does-not-exist") do $$$AssertStatusNotOK(sc, "Sync fails for a module that doesn't exist") } @@ -374,8 +343,7 @@ Method TestSyncNonDevModeModule() do ##class(%IPM.Main).Shell("uninstall sync-test") $$$ThrowOnError(##class(%IPM.Main).Shell("load " _ ..TempDir)) - kill params - set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + set sc = ##class(%IPM.Main).Shell("sync sync-test") do $$$AssertStatusNotOK(sc, "Sync fails for a module not in development mode") } @@ -395,10 +363,8 @@ Method TestModuleXmlAddsResourcePicksUpNewFile() do ..ReplaceInFile(moduleXmlPath, "1.0.0", "1.0.1") do ..ReplaceInFile(moduleXmlPath, "", "" _ $char(10) _ " ") - kill params - set params("Verbose") = 1 do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + set sc = ##class(%IPM.Main).Shell("sync sync-test -verbose") do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) do $$$AssertStatusOK(sc, "Sync after module.xml adds a resource succeeds") @@ -418,8 +384,7 @@ Method TestFailedCompileRetries() $$$ThrowOnError(stream.%Save()) set stream = "" - kill params - set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + set sc = ##class(%IPM.Main).Shell("sync sync-test") do $$$AssertStatusNotOK(sc, "Sync fails with syntax error") // Fix and retry — CommitChanges was skipped on failure, so file is still detected as changed @@ -432,8 +397,7 @@ Method TestFailedCompileRetries() $$$ThrowOnError(stream.%Save()) set stream = "" - kill params - set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + set sc = ##class(%IPM.Main).Shell("sync sync-test") do $$$AssertStatusOK(sc, "Sync succeeds after fixing syntax error (retry works)") } @@ -449,10 +413,8 @@ Method TestModuleXmlAndClassEditedTogether() do ..ReplaceInFile(moduleXmlPath, "1.0.0", "1.0.1") do ..ReplaceInFile(clsPath, """original""", """modified-with-xml""") - kill params - set params("Verbose") = 1 do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) - set sc = ##class(%IPM.Storage.Module).Sync("sync-test", .params) + set sc = ##class(%IPM.Main).Shell("sync sync-test -verbose") do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) do $$$AssertStatusOK(sc, "Sync after co-editing module.xml and a class succeeds") From 17d9636038527a44197ff8752dd1dcdc3459ba1f Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Wed, 22 Jul 2026 14:20:05 -0400 Subject: [PATCH 24/39] Fix a few small issues and improve tests --- src/cls/IPM/Lifecycle/Base.cls | 5 ++++- src/cls/IPM/ResourceProcessor/Abstract.cls | 2 ++ .../ResourceProcessor/Default/Document.cls | 3 +++ src/cls/IPM/ResourceProcessor/Test.cls | 20 +++++++++++++++++++ src/cls/IPM/Storage/FileHash.cls | 2 +- .../Test/PM/Integration/Sync.cls | 15 +++++++++++++- 6 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls index 1bd0fdd36..8233dbc8e 100644 --- a/src/cls/IPM/Lifecycle/Base.cls +++ b/src/cls/IPM/Lifecycle/Base.cls @@ -1010,8 +1010,9 @@ ClassMethod SyncApplyDeletes( ByRef deleted, ByRef reverseIndex, ByRef syncByResource, - verbose As %Boolean) [ Private ] + verbose As %Boolean) As %Status [ Private ] { + set sc = $$$OK set relPath = "" for { set relPath = $order(deleted(relPath)) @@ -1038,9 +1039,11 @@ ClassMethod SyncApplyDeletes( // deletions unapplied and make the overall error harder to diagnose. if $$$ISERR(delSC) { write !, "Warning: could not delete ", docName, ": ", $system.Status.GetOneErrorText(delSC) + set sc = $$$ADDSC(sc, delSC) } } } + quit sc } /// Run test-phase tests for changed test case classes recorded in params("Sync","ChangedTestCases"). diff --git a/src/cls/IPM/ResourceProcessor/Abstract.cls b/src/cls/IPM/ResourceProcessor/Abstract.cls index 38bee611a..06586f7a9 100644 --- a/src/cls/IPM/ResourceProcessor/Abstract.cls +++ b/src/cls/IPM/ResourceProcessor/Abstract.cls @@ -213,6 +213,8 @@ Method OnSync(ByRef modifiedPaths, ByRef deletedPaths, ByRef params, Output hand } /// Returns 1 if this processor supports incremental sync. Base returns 0. +/// Stage 2 TODO: FileCopy, WebApplication, PythonWheel — override to 1 + add OnSync. +/// Stage 3 TODO: CPF, Copy, ArtifactoryTarball, LegacyLocalizedMessages, Default.Global. Method SupportsSync() As %Boolean { quit 0 diff --git a/src/cls/IPM/ResourceProcessor/Default/Document.cls b/src/cls/IPM/ResourceProcessor/Default/Document.cls index 663a9eac6..b2f5f4452 100644 --- a/src/cls/IPM/ResourceProcessor/Default/Document.cls +++ b/src/cls/IPM/ResourceProcessor/Default/Document.cls @@ -519,6 +519,9 @@ Method GetSyncDirectory() As %String { if ..LoadAsDirectory { set dir = ##class(%IPM.Storage.FileHash).NormalizePath(..ResourceReference.Name) + if $extract(dir, *) = "/" { + set dir = $extract(dir, 1, *-1) + } } else { set sourcesRoot = ..ResourceReference.Module.SourcesRoot set dir = ..Directory diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls index 8f824a3e7..fd7551ed7 100644 --- a/src/cls/IPM/ResourceProcessor/Test.cls +++ b/src/cls/IPM/ResourceProcessor/Test.cls @@ -431,6 +431,26 @@ Method OnSync(ByRef modifiedPaths, ByRef deletedPaths, ByRef params, Output hand set params("Sync", "ChangedTestCases", className) = ..ResourceReference.Name } } + + // Delete server-side classes for removed test files. SyncApplyDeletes skips Test + // resources (not AbstractCompilable), so OnSync must own the delete for this type. + set relPath = "" + for { + set relPath = $order(deletedPaths(relPath)) + quit:relPath="" + + set packageRelPath = relPath + if $extract(packageRelPath, 1, $length(resourceDir)) = resourceDir { + set packageRelPath = $extract(packageRelPath, $length(resourceDir) + 1, *) + } + set fileName = $piece(packageRelPath, "/", *) + set baseName = $piece(fileName, ".", 1, *-1) + set dirPart = $piece(packageRelPath, "/", 1, *-1) + set className = $select(dirPart '= "": $translate(dirPart, "/", ".") _ "." _ baseName, 1: baseName) + if $zconvert($piece(fileName, ".", *), "U") = "CLS" && $$$comClassDefined(className) { + $$$ThrowOnError($system.OBJ.Delete(className, "-d")) + } + } } catch e { set sc = e.AsStatus() } diff --git a/src/cls/IPM/Storage/FileHash.cls b/src/cls/IPM/Storage/FileHash.cls index 829958349..8c1f2b4fb 100644 --- a/src/cls/IPM/Storage/FileHash.cls +++ b/src/cls/IPM/Storage/FileHash.cls @@ -35,7 +35,7 @@ ClassMethod StampModule(module As %IPM.Storage.Module) As %Status for { set resource = orderedResourceList.GetNext(.key) quit:key="" - if '$isobject(resource.Processor) { + if '$isobject(resource.Processor) || 'resource.Processor.SupportsSync() { continue } set syncDir = resource.Processor.GetSyncDirectory() diff --git a/tests/integration_tests/Test/PM/Integration/Sync.cls b/tests/integration_tests/Test/PM/Integration/Sync.cls index 701a46b9d..cc8fd71f5 100644 --- a/tests/integration_tests/Test/PM/Integration/Sync.cls +++ b/tests/integration_tests/Test/PM/Integration/Sync.cls @@ -198,6 +198,19 @@ Method TestDeleteSkippedByDefault() do $$$AssertNotTrue($$$comClassDefined("SyncTest.Deletable"), "Deletable class removed after -delete sync") } +/// Deleting a test class file with -delete must remove the server-side class. +/// Without this, the %UnitTest.TestCase subclass persists with no on-disk source and no +/// hash record — permanently invisible to future syncs. +Method TestDeleteTestClassRemovesFromServer() +{ + set filePath = ..TempDir _ "tests/unit/SyncTest/Tests/Trivial.cls" + do ##class(%Library.File).Delete(filePath) + + set sc = ##class(%IPM.Main).Shell("sync sync-test -delete") + do $$$AssertStatusOK(sc, "Sync with -delete on a deleted test class succeeds") + do $$$AssertNotTrue($$$comClassDefined("SyncTest.Tests.Trivial"), "Deleted test class removed from server after -delete sync") +} + /// Deleting a superclass (with -delete) leaves its subclass referencing a now-missing class. /// The post-delete recompile pass (SyncApplyDeletes + SyncCompile) must still run and surface /// this as a compile error, rather than silently leaving the subclass in a stale-but-compiled state. @@ -325,7 +338,7 @@ Method TestSyncAllDevModeModules() do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) do $$$AssertStatusOK(sc, "Sync with no module name (sync-all) succeeds") - do $$$AssertTrue(..FindInOutput(.output, "Nothing to sync"), "sync-test reports nothing to sync (no changes since baseline)") + do $$$AssertTrue(..FindInOutput(.output, "[sync-test] Nothing to sync"), "sync-test specifically reports nothing to sync (no changes since baseline)") } /// Sync fails with an error when given a module name that isn't installed. From 71cd87c48fb3cdac2d17c47291f7bef59c47044f Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Wed, 22 Jul 2026 14:42:32 -0400 Subject: [PATCH 25/39] Better UX for deletes --- src/cls/IPM/Lifecycle/Base.cls | 6 ++++++ src/cls/IPM/ResourceProcessor/Test.cls | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls index 8233dbc8e..0e02ca627 100644 --- a/src/cls/IPM/Lifecycle/Base.cls +++ b/src/cls/IPM/Lifecycle/Base.cls @@ -700,6 +700,12 @@ Method %Sync(ByRef params) As %Status // Falls through when only deletes exist and processDeletes=1 if '$data(modified) && ('$data(deleted) || 'processDeletes) { write !, "[", moduleName, "] Nothing to sync." + if '$data(modified) && $data(deleted) && 'processDeletes { + set delCount = 0 + set key = "" + for { set key = $order(deleted(key)) quit:key="" set delCount = delCount + 1 } + write !, " ", delCount, " deleted file(s) detected but not applied. Use -delete to remove from server." + } if moduleXmlChanged { do ..SyncCommitModuleXml(module, moduleXmlPath, moduleXmlRelPath) do ..SyncPrintModuleXmlWarning(moduleName) diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls index fd7551ed7..9d7fa4449 100644 --- a/src/cls/IPM/ResourceProcessor/Test.cls +++ b/src/cls/IPM/ResourceProcessor/Test.cls @@ -448,7 +448,7 @@ Method OnSync(ByRef modifiedPaths, ByRef deletedPaths, ByRef params, Output hand set dirPart = $piece(packageRelPath, "/", 1, *-1) set className = $select(dirPart '= "": $translate(dirPart, "/", ".") _ "." _ baseName, 1: baseName) if $zconvert($piece(fileName, ".", *), "U") = "CLS" && $$$comClassDefined(className) { - $$$ThrowOnError($system.OBJ.Delete(className, "-d")) + $$$ThrowOnError($system.OBJ.Delete(className, $select(verbose:"d",1:"-d"))) } } } catch e { From 901e01fb021ffde21885521778d6e17fffe15daf Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Wed, 22 Jul 2026 14:49:54 -0400 Subject: [PATCH 26/39] Remove extra newline and improve module.xml in test --- src/cls/IPM/ResourceProcessor/FileCopy.cls | 1 - .../Test/PM/Integration/_data/sync-test/module.xml | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/cls/IPM/ResourceProcessor/FileCopy.cls b/src/cls/IPM/ResourceProcessor/FileCopy.cls index fe62659ad..22976ea2a 100644 --- a/src/cls/IPM/ResourceProcessor/FileCopy.cls +++ b/src/cls/IPM/ResourceProcessor/FileCopy.cls @@ -184,7 +184,6 @@ Method DoCopy( quit tSC } - Method OnExportItem( pFullExportPath As %String, pItemName As %String, diff --git a/tests/integration_tests/Test/PM/Integration/_data/sync-test/module.xml b/tests/integration_tests/Test/PM/Integration/_data/sync-test/module.xml index a7671723e..143793e31 100644 --- a/tests/integration_tests/Test/PM/Integration/_data/sync-test/module.xml +++ b/tests/integration_tests/Test/PM/Integration/_data/sync-test/module.xml @@ -1,6 +1,6 @@ - + sync-test @@ -8,9 +8,7 @@ module src - - - + From 4455548106b82eeeadfed3eaae0e8b431f360b8c Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Wed, 22 Jul 2026 15:28:34 -0400 Subject: [PATCH 27/39] Try different container image --- .github/workflows/main.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 780f407ad..145357d96 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -79,6 +79,8 @@ jobs: - name: Run temporary registry timeout-minutes: 15 run: | + echo ${{ secrets.GITHUB_TOKEN }} | docker login docker.pkg.github.com --username ${{ github.actor }} --password-stdin + docker pull containers.intersystems.com/intersystems/iris-community:latest-em docker network create zpm docker build -f tests/registry/Dockerfile -t registry-image . REGISTRY=$(docker run --rm -d \ @@ -86,7 +88,7 @@ jobs: --name registry \ -p 52773:52773 \ --network-alias registry \ - registry-image \ + containers.intersystems.com/intersystems/iris-community:latest-em \ -a "iris session iris -U%SYS '##class(Security.Users).UnExpireUserPasswords(\"*\")'") sleep 5; docker exec $REGISTRY /usr/irissys/dev/Cloud/ICM/waitReady.sh docker exec -i $REGISTRY iris session iris -UUSER << EOF @@ -135,7 +137,7 @@ jobs: --name registry \ -p 52773:52773 \ --network-alias registry \ - registry-image \ + containers.intersystems.com/intersystems/iris-community:latest-em \ -a "iris session iris -U%SYS '##class(Security.Users).UnExpireUserPasswords(\"*\")'" REGISTRY=`docker ps -lq` sleep 5; docker exec $REGISTRY /usr/irissys/dev/Cloud/ICM/waitReady.sh From 895bd77f4d9c7277f060c4203ada0f692a54e255 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Thu, 23 Jul 2026 11:03:09 -0400 Subject: [PATCH 28/39] Revert image change --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 145357d96..7f0bda466 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -80,7 +80,7 @@ jobs: timeout-minutes: 15 run: | echo ${{ secrets.GITHUB_TOKEN }} | docker login docker.pkg.github.com --username ${{ github.actor }} --password-stdin - docker pull containers.intersystems.com/intersystems/iris-community:latest-em + docker pull intersystemsdc/iris-community:latest docker network create zpm docker build -f tests/registry/Dockerfile -t registry-image . REGISTRY=$(docker run --rm -d \ @@ -88,7 +88,7 @@ jobs: --name registry \ -p 52773:52773 \ --network-alias registry \ - containers.intersystems.com/intersystems/iris-community:latest-em \ + intersystemsdc/iris-community:latest \ -a "iris session iris -U%SYS '##class(Security.Users).UnExpireUserPasswords(\"*\")'") sleep 5; docker exec $REGISTRY /usr/irissys/dev/Cloud/ICM/waitReady.sh docker exec -i $REGISTRY iris session iris -UUSER << EOF @@ -137,7 +137,7 @@ jobs: --name registry \ -p 52773:52773 \ --network-alias registry \ - containers.intersystems.com/intersystems/iris-community:latest-em \ + intersystemsdc/iris-community:latest \ -a "iris session iris -U%SYS '##class(Security.Users).UnExpireUserPasswords(\"*\")'" REGISTRY=`docker ps -lq` sleep 5; docker exec $REGISTRY /usr/irissys/dev/Cloud/ICM/waitReady.sh From 239d819644e6bd939b04867b4505ebecde91f794 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Thu, 23 Jul 2026 11:37:15 -0400 Subject: [PATCH 29/39] Try different fix --- .github/workflows/main.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7f0bda466..13bb9a6eb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -88,9 +88,10 @@ jobs: --name registry \ -p 52773:52773 \ --network-alias registry \ + --shm-size=512m \ intersystemsdc/iris-community:latest \ -a "iris session iris -U%SYS '##class(Security.Users).UnExpireUserPasswords(\"*\")'") - sleep 5; docker exec $REGISTRY /usr/irissys/dev/Cloud/ICM/waitReady.sh + sleep 5; docker exec $REGISTRY /usr/irissys/dev/Container/waitReady.sh docker exec -i $REGISTRY iris session iris -UUSER << EOF zpm "install zpm-registry" halt @@ -117,7 +118,7 @@ jobs: -e TEST_REGISTRY_USER=admin \ -e TEST_REGISTRY_PASSWORD=SYS \ zpm ${{ steps.image.outputs.flags }}) - sleep 5; docker exec $CONTAINER /usr/irissys/dev/Cloud/ICM/waitReady.sh + sleep 5; docker exec $CONTAINER /usr/irissys/dev/Container/waitReady.sh docker cp . $CONTAINER:/home/irisowner/zpm/ echo `docker exec -i --user root $CONTAINER chmod -R 777 /home/irisowner/zpm/` echo `docker exec -i --workdir /home/irisowner/zpm/ $CONTAINER ls -rtl` @@ -137,10 +138,11 @@ jobs: --name registry \ -p 52773:52773 \ --network-alias registry \ + --shm-size=512m \ intersystemsdc/iris-community:latest \ -a "iris session iris -U%SYS '##class(Security.Users).UnExpireUserPasswords(\"*\")'" REGISTRY=`docker ps -lq` - sleep 5; docker exec $REGISTRY /usr/irissys/dev/Cloud/ICM/waitReady.sh + sleep 5; docker exec $REGISTRY /usr/irissys/dev/Container/waitReady.sh docker exec -i $REGISTRY iris session iris -UUSER << EOF zpm "install zpm-registry" halt @@ -150,7 +152,7 @@ jobs: timeout-minutes: 15 run: | CONTAINER=$(docker run --network zpm -d --rm zpm ${{ steps.image.outputs.flags }}) - sleep 5; docker exec $CONTAINER /usr/irissys/dev/Cloud/ICM/waitReady.sh + sleep 5; docker exec $CONTAINER /usr/irissys/dev/Container/waitReady.sh docker cp . $CONTAINER:/home/irisowner/zpm/ echo `docker exec -i --user root $CONTAINER chmod -R 777 /home/irisowner/zpm/` docker exec -i $CONTAINER iris session iris -UUSER << EOF @@ -176,7 +178,7 @@ jobs: CONTAINER=$(docker run --network zpm --rm -d -v /tmp/zpm.xml:/home/irisowner/zpm.xml ${{ steps.image.outputs.name }} ${{ steps.image.outputs.flags }}) docker cp . $CONTAINER:/home/irisowner/zpm/ echo `docker exec -i --user root $CONTAINER chmod -R 777 /home/irisowner/zpm/` - sleep 5; docker exec $CONTAINER /usr/irissys/dev/Cloud/ICM/waitReady.sh + sleep 5; docker exec $CONTAINER /usr/irissys/dev/Container/waitReady.sh docker exec -i $CONTAINER iris session iris -UUSER << 'EOF' do $System.OBJ.Load("/home/irisowner/zpm.xml","c") zpm "enable -globally -map -repos -community" @@ -207,7 +209,7 @@ jobs: CONTAINER=$(docker run --network zpm --rm -d ${{ steps.image.outputs.name }} ${{ steps.image.outputs.flags }}) docker cp tests/migration/v0.7-to-v0.9/. $CONTAINER:/tmp/test-package/ docker cp . $CONTAINER:/home/irisowner/zpm/ - sleep 5; docker exec $CONTAINER /usr/irissys/dev/Cloud/ICM/waitReady.sh + sleep 5; docker exec $CONTAINER /usr/irissys/dev/Container/waitReady.sh docker exec -i $CONTAINER iris session iris -UUSER << 'EOF' s version="0.7.4" s r=##class(%Net.HttpRequest).%New(),r.Server="pm.community.intersystems.com",r.SSLConfiguration="ISC.FeatureTracker.SSL.Config" d r.Get("/packages/zpm/"_version_"/installer"),$system.OBJ.LoadStream(r.HttpResponse.Data,"c") zpm "list":1 @@ -233,7 +235,7 @@ jobs: wget http://localhost:52773/registry/packages/zpm/latest/installer -O /tmp/zpm.xml CONTAINER=$(docker run --network zpm --rm -d ${{ steps.image.outputs.name }} ${{ steps.image.outputs.flags }}) docker cp /tmp/zpm.xml $CONTAINER:/home/irisowner/zpm.xml - sleep 5; docker exec $CONTAINER /usr/irissys/dev/Cloud/ICM/waitReady.sh + sleep 5; docker exec $CONTAINER /usr/irissys/dev/Container/waitReady.sh docker exec -i $CONTAINER iris session iris -U%SYS << EOF set sc = ##class(%SYSTEM.OBJ).Load("/home/irisowner/zpm.xml", "ck") if +sc=0 do ##class(%SYSTEM.Process).Terminate(,1) @@ -320,7 +322,7 @@ jobs: sed -i -E "s/(.*)<\/Version>/${VERSION}<\/Version>/" module.xml cat module.xml CONTAINER=$(docker run -d --rm -v $(pwd):/home/irisowner/zpm/ containers.intersystems.com/intersystems/${{ needs.prepare.outputs.main }} --check-caps false) - sleep 5; docker exec $CONTAINER /usr/irissys/dev/Cloud/ICM/waitReady.sh + sleep 5; docker exec $CONTAINER /usr/irissys/dev/Container/waitReady.sh docker exec -i $CONTAINER iris session iris -UUSER << EOF set sc=##class(%SYSTEM.OBJ).Load("/home/irisowner/zpm/preload/cls/IPM/Installer.cls","ck") set sc=##class(IPM.Installer).setup("/home/irisowner/zpm/",3) From 6bbdfedfded0bd7a89cf7976c859a77287092890 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Thu, 23 Jul 2026 15:10:06 -0400 Subject: [PATCH 30/39] Use the new dockerfile in the main workflow too --- .github/workflows/main.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 13bb9a6eb..0c77967e4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -79,8 +79,6 @@ jobs: - name: Run temporary registry timeout-minutes: 15 run: | - echo ${{ secrets.GITHUB_TOKEN }} | docker login docker.pkg.github.com --username ${{ github.actor }} --password-stdin - docker pull intersystemsdc/iris-community:latest docker network create zpm docker build -f tests/registry/Dockerfile -t registry-image . REGISTRY=$(docker run --rm -d \ @@ -88,14 +86,9 @@ jobs: --name registry \ -p 52773:52773 \ --network-alias registry \ - --shm-size=512m \ - intersystemsdc/iris-community:latest \ + registry-image \ -a "iris session iris -U%SYS '##class(Security.Users).UnExpireUserPasswords(\"*\")'") sleep 5; docker exec $REGISTRY /usr/irissys/dev/Container/waitReady.sh - docker exec -i $REGISTRY iris session iris -UUSER << EOF - zpm "install zpm-registry" - halt - EOF docker logs $REGISTRY - name: Run ORAS registry timeout-minutes: 5 @@ -138,15 +131,10 @@ jobs: --name registry \ -p 52773:52773 \ --network-alias registry \ - --shm-size=512m \ - intersystemsdc/iris-community:latest \ + registry-image \ -a "iris session iris -U%SYS '##class(Security.Users).UnExpireUserPasswords(\"*\")'" REGISTRY=`docker ps -lq` sleep 5; docker exec $REGISTRY /usr/irissys/dev/Container/waitReady.sh - docker exec -i $REGISTRY iris session iris -UUSER << EOF - zpm "install zpm-registry" - halt - EOF docker logs $REGISTRY - name: Test and publish to temporary registry timeout-minutes: 15 From 53660a3da803070e30a4d34f66ed06a4259bbe3f Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Thu, 23 Jul 2026 15:49:21 -0400 Subject: [PATCH 31/39] Use the old script location for 2025.3 --- .github/workflows/main.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0c77967e4..b80b8a8e7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -88,7 +88,7 @@ jobs: --network-alias registry \ registry-image \ -a "iris session iris -U%SYS '##class(Security.Users).UnExpireUserPasswords(\"*\")'") - sleep 5; docker exec $REGISTRY /usr/irissys/dev/Container/waitReady.sh + sleep 5; docker exec $REGISTRY /usr/irissys/dev/Cloud/ICM/waitReady.sh docker logs $REGISTRY - name: Run ORAS registry timeout-minutes: 5 @@ -111,7 +111,7 @@ jobs: -e TEST_REGISTRY_USER=admin \ -e TEST_REGISTRY_PASSWORD=SYS \ zpm ${{ steps.image.outputs.flags }}) - sleep 5; docker exec $CONTAINER /usr/irissys/dev/Container/waitReady.sh + sleep 5; docker exec $CONTAINER /usr/irissys/dev/Cloud/ICM/waitReady.sh docker cp . $CONTAINER:/home/irisowner/zpm/ echo `docker exec -i --user root $CONTAINER chmod -R 777 /home/irisowner/zpm/` echo `docker exec -i --workdir /home/irisowner/zpm/ $CONTAINER ls -rtl` @@ -134,13 +134,13 @@ jobs: registry-image \ -a "iris session iris -U%SYS '##class(Security.Users).UnExpireUserPasswords(\"*\")'" REGISTRY=`docker ps -lq` - sleep 5; docker exec $REGISTRY /usr/irissys/dev/Container/waitReady.sh + sleep 5; docker exec $REGISTRY /usr/irissys/dev/Cloud/ICM/waitReady.sh docker logs $REGISTRY - name: Test and publish to temporary registry timeout-minutes: 15 run: | CONTAINER=$(docker run --network zpm -d --rm zpm ${{ steps.image.outputs.flags }}) - sleep 5; docker exec $CONTAINER /usr/irissys/dev/Container/waitReady.sh + sleep 5; docker exec $CONTAINER /usr/irissys/dev/Cloud/ICM/waitReady.sh docker cp . $CONTAINER:/home/irisowner/zpm/ echo `docker exec -i --user root $CONTAINER chmod -R 777 /home/irisowner/zpm/` docker exec -i $CONTAINER iris session iris -UUSER << EOF @@ -166,7 +166,7 @@ jobs: CONTAINER=$(docker run --network zpm --rm -d -v /tmp/zpm.xml:/home/irisowner/zpm.xml ${{ steps.image.outputs.name }} ${{ steps.image.outputs.flags }}) docker cp . $CONTAINER:/home/irisowner/zpm/ echo `docker exec -i --user root $CONTAINER chmod -R 777 /home/irisowner/zpm/` - sleep 5; docker exec $CONTAINER /usr/irissys/dev/Container/waitReady.sh + sleep 5; docker exec $CONTAINER /usr/irissys/dev/Cloud/ICM/waitReady.sh docker exec -i $CONTAINER iris session iris -UUSER << 'EOF' do $System.OBJ.Load("/home/irisowner/zpm.xml","c") zpm "enable -globally -map -repos -community" @@ -197,7 +197,7 @@ jobs: CONTAINER=$(docker run --network zpm --rm -d ${{ steps.image.outputs.name }} ${{ steps.image.outputs.flags }}) docker cp tests/migration/v0.7-to-v0.9/. $CONTAINER:/tmp/test-package/ docker cp . $CONTAINER:/home/irisowner/zpm/ - sleep 5; docker exec $CONTAINER /usr/irissys/dev/Container/waitReady.sh + sleep 5; docker exec $CONTAINER /usr/irissys/dev/Cloud/ICM/waitReady.sh docker exec -i $CONTAINER iris session iris -UUSER << 'EOF' s version="0.7.4" s r=##class(%Net.HttpRequest).%New(),r.Server="pm.community.intersystems.com",r.SSLConfiguration="ISC.FeatureTracker.SSL.Config" d r.Get("/packages/zpm/"_version_"/installer"),$system.OBJ.LoadStream(r.HttpResponse.Data,"c") zpm "list":1 @@ -223,7 +223,7 @@ jobs: wget http://localhost:52773/registry/packages/zpm/latest/installer -O /tmp/zpm.xml CONTAINER=$(docker run --network zpm --rm -d ${{ steps.image.outputs.name }} ${{ steps.image.outputs.flags }}) docker cp /tmp/zpm.xml $CONTAINER:/home/irisowner/zpm.xml - sleep 5; docker exec $CONTAINER /usr/irissys/dev/Container/waitReady.sh + sleep 5; docker exec $CONTAINER /usr/irissys/dev/Cloud/ICM/waitReady.sh docker exec -i $CONTAINER iris session iris -U%SYS << EOF set sc = ##class(%SYSTEM.OBJ).Load("/home/irisowner/zpm.xml", "ck") if +sc=0 do ##class(%SYSTEM.Process).Terminate(,1) @@ -310,7 +310,7 @@ jobs: sed -i -E "s/(.*)<\/Version>/${VERSION}<\/Version>/" module.xml cat module.xml CONTAINER=$(docker run -d --rm -v $(pwd):/home/irisowner/zpm/ containers.intersystems.com/intersystems/${{ needs.prepare.outputs.main }} --check-caps false) - sleep 5; docker exec $CONTAINER /usr/irissys/dev/Container/waitReady.sh + sleep 5; docker exec $CONTAINER /usr/irissys/dev/Cloud/ICM/waitReady.sh docker exec -i $CONTAINER iris session iris -UUSER << EOF set sc=##class(%SYSTEM.OBJ).Load("/home/irisowner/zpm/preload/cls/IPM/Installer.cls","ck") set sc=##class(IPM.Installer).setup("/home/irisowner/zpm/",3) From 609a74907c4e08d013824ec14b86af0d37466814 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Fri, 24 Jul 2026 13:37:34 -0400 Subject: [PATCH 32/39] Add test case for xml classes --- .../integration_tests/Test/PM/Integration/Sync.cls | 13 +++++++++++++ .../Test/PM/Integration/_data/sync-test/module.xml | 1 + .../_data/sync-test/src/cls/SyncTest/XmlClass.xml | 14 ++++++++++++++ 3 files changed, 28 insertions(+) create mode 100644 tests/integration_tests/Test/PM/Integration/_data/sync-test/src/cls/SyncTest/XmlClass.xml diff --git a/tests/integration_tests/Test/PM/Integration/Sync.cls b/tests/integration_tests/Test/PM/Integration/Sync.cls index cc8fd71f5..a44534f96 100644 --- a/tests/integration_tests/Test/PM/Integration/Sync.cls +++ b/tests/integration_tests/Test/PM/Integration/Sync.cls @@ -134,6 +134,19 @@ Method TestModifiedClassRecompiles() do $$$AssertTrue(..FindInOutput(.output, "Sync complete"), "Reports sync complete") } +/// A modified XML-format class file is detected and recompiled by sync. +/// Verifies that Format="XML" resources participate in the sync pipeline identically +/// to UDL resources, and that the reloaded class reflects the edited content. +Method TestModifiedXmlClassRecompiles() +{ + set filePath = ..TempDir _ "src/cls/SyncTest/XmlClass.xml" + do ..ReplaceInFile(filePath, """xml-original""", """xml-modified""") + + set sc = ##class(%IPM.Main).Shell("sync sync-test") + do $$$AssertStatusOK(sc, "Sync after XML class modification succeeds") + do $$$AssertEquals(##class(SyncTest.XmlClass).GetValue(), "xml-modified", "Class reflects updated value after XML sync") +} + /// Editing a superclass recompiles its subclasses via the u-flag, even though the /// subclass's own file on disk never changed. Method TestSuperclassEditRecompilesSubclass() diff --git a/tests/integration_tests/Test/PM/Integration/_data/sync-test/module.xml b/tests/integration_tests/Test/PM/Integration/_data/sync-test/module.xml index 143793e31..f43503d7e 100644 --- a/tests/integration_tests/Test/PM/Integration/_data/sync-test/module.xml +++ b/tests/integration_tests/Test/PM/Integration/_data/sync-test/module.xml @@ -8,6 +8,7 @@ module src + diff --git a/tests/integration_tests/Test/PM/Integration/_data/sync-test/src/cls/SyncTest/XmlClass.xml b/tests/integration_tests/Test/PM/Integration/_data/sync-test/src/cls/SyncTest/XmlClass.xml new file mode 100644 index 000000000..bc4098ff5 --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/sync-test/src/cls/SyncTest/XmlClass.xml @@ -0,0 +1,14 @@ + + + + +XML-format fixture class for sync integration tests. + + +1 +%String + + + + From 0f7d14ebd44e08bdcfaf9fa8e8f139f900b5ba27 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Tue, 28 Jul 2026 11:10:50 -0400 Subject: [PATCH 33/39] Add better dependency handling and test case --- src/cls/IPM/Main.cls | 103 ++++++++++++++--- .../Test/PM/Integration/Sync.cls | 105 ++++++++++++++++++ .../Integration/_data/sync-dep/a/module.xml | 13 +++ .../_data/sync-dep/a/src/cls/SyncDepA/Top.cls | 9 ++ .../Integration/_data/sync-dep/b/module.xml | 13 +++ .../sync-dep/b/src/cls/SyncDepB/Middle.cls | 9 ++ .../Integration/_data/sync-dep/c/module.xml | 13 +++ .../sync-dep/c/src/cls/SyncDepC/Base.cls | 9 ++ 8 files changed, 259 insertions(+), 15 deletions(-) create mode 100644 tests/integration_tests/Test/PM/Integration/_data/sync-dep/a/module.xml create mode 100644 tests/integration_tests/Test/PM/Integration/_data/sync-dep/a/src/cls/SyncDepA/Top.cls create mode 100644 tests/integration_tests/Test/PM/Integration/_data/sync-dep/b/module.xml create mode 100644 tests/integration_tests/Test/PM/Integration/_data/sync-dep/b/src/cls/SyncDepB/Middle.cls create mode 100644 tests/integration_tests/Test/PM/Integration/_data/sync-dep/c/module.xml create mode 100644 tests/integration_tests/Test/PM/Integration/_data/sync-dep/c/src/cls/SyncDepC/Base.cls diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls index dc5179dce..4364460c1 100644 --- a/src/cls/IPM/Main.cls +++ b/src/cls/IPM/Main.cls @@ -2333,22 +2333,13 @@ ClassMethod Sync(ByRef commandInfo) [ Private ] if moduleName '= "" { $$$ThrowOnError(##class(%IPM.Storage.Module).ExecutePhases(moduleName, $listbuild("Sync"), 1, .params)) } else { - // Sync all dev-mode modules - set result = ##class(%SQL.Statement).%ExecDirect(, - "select Name from %IPM_Storage.ModuleItem where DeveloperMode = 1") - if (result.%SQLCODE < 0) { - throw ##class(%Exception.SQL).CreateFromSQLCODE(result.%SQLCODE, result.%Message) - } - set found = 0 + // Sync all dev-mode modules in dependency order (least dependent first) so each module + // recompiles against dependencies that have already been synced this run. + set orderedNames = ..GetDevModeModulesInDependencyOrder() + set found = $listlength(orderedNames) set failures = "" - for { - set hasData = result.%Next(.sc) - $$$ThrowOnError(sc) - if 'hasData { - quit - } - set found = found + 1 - set name = result.%Get("Name") + for i = 1:1:found { + set name = $list(orderedNames, i) set syncSC = ##class(%IPM.Storage.Module).ExecutePhases(name, $listbuild("Sync"), 1, .params) if $$$ISERR(syncSC) { set failures = failures _ $listbuild(name) @@ -2363,6 +2354,88 @@ ClassMethod Sync(ByRef commandInfo) [ Private ] } } +/// Returns a $list of dev-mode module names ordered least-dependent-first: a module's +/// dependencies appear before it. Sync-all uses this so each module recompiles against +/// dependencies already synced in the same run. Only dependency edges among the dev-mode set +/// are considered; modules unconnected by any such edge keep their natural (row) order. +ClassMethod GetDevModeModulesInDependencyOrder() As %List [ Private ] +{ + // Dev-mode modules in row order (stable tiebreaker for modules with no ordering constraint). + set result = ##class(%SQL.Statement).%ExecDirect(, + "select Name from %IPM_Storage.ModuleItem where DeveloperMode = 1") + if (result.%SQLCODE < 0) { + throw ##class(%Exception.SQL).CreateFromSQLCODE(result.%SQLCODE, result.%Message) + } + kill devMode + set rowOrder = "" + while result.%Next(.sc) { + $$$ThrowOnError(sc) + set name = result.%Get("Name") + set devMode(name) = "" + set rowOrder = rowOrder _ $listbuild(name) + } + $$$ThrowOnError(sc) + + // Dependency edges restricted to the dev-mode set: dependsOn(modName, depName) = "". + set depRes = ##class(%SQL.Statement).%ExecDirect(, + "select ModuleItem->Name ModName, Dependencies_Name DepName from %IPM_Storage.ModuleItem_Dependencies") + if (depRes.%SQLCODE < 0) { + throw ##class(%Exception.SQL).CreateFromSQLCODE(depRes.%SQLCODE, depRes.%Message) + } + kill dependsOn + while depRes.%Next(.sc) { + $$$ThrowOnError(sc) + set modName = depRes.%Get("ModName") + set depName = depRes.%Get("DepName") + if $data(devMode(modName)) && $data(devMode(depName)) { + set dependsOn(modName, depName) = "" + } + } + $$$ThrowOnError(sc) + + // Emit a module only once all its (dev-mode) dependencies are already emitted; repeat until + // every module is placed. Each pass scans in row order so independent modules keep their + // natural sequence. + set ordered = "" + kill emitted + set remaining = $listlength(rowOrder) + while remaining > 0 { + set progress = 0 + set ptr = 0 + while $listnext(rowOrder, ptr, name) { + continue:$data(emitted(name)) + set ready = 1 + set depName = "" + for { + set depName = $order(dependsOn(name, depName)) + quit:depName="" + if '$data(emitted(depName)) { + set ready = 0 + quit + } + } + if ready { + set ordered = ordered _ $listbuild(name) + set emitted(name) = "" + set remaining = remaining - 1 + set progress = 1 + } + } + if 'progress { + // A dependency cycle within the dev-mode set would stall the loop. Install rejects + // real cycles, so this is a safety net: emit the rest in row order and stop. + set ptr = 0 + while $listnext(rowOrder, ptr, name) { + continue:$data(emitted(name)) + set ordered = ordered _ $listbuild(name) + set emitted(name) = "" + } + set remaining = 0 + } + } + quit ordered +} + ClassMethod Load( ByRef pCommandInfo, pLog As %IPM.General.AbstractHistory = "") [ Internal ] diff --git a/tests/integration_tests/Test/PM/Integration/Sync.cls b/tests/integration_tests/Test/PM/Integration/Sync.cls index a44534f96..8879dd2dd 100644 --- a/tests/integration_tests/Test/PM/Integration/Sync.cls +++ b/tests/integration_tests/Test/PM/Integration/Sync.cls @@ -354,6 +354,78 @@ Method TestSyncAllDevModeModules() do $$$AssertTrue(..FindInOutput(.output, "[sync-test] Nothing to sync"), "sync-test specifically reports nothing to sync (no changes since baseline)") } +/// Naked `sync` (no module name) must process modules most-dependent last: a dependency is +/// synced before the module that depends on it. This matters because sync recompiles each +/// module's changed files against the currently-installed version of its dependencies — if a +/// dependent is synced before its dependency, it recompiles against stale code. +/// +/// The bug this guards against: Main.Sync selects dev-mode modules with no ORDER BY, so it +/// iterates in row-ID (first-install) order. A clean install is coincidentally in dependency +/// order (the installer forces deps-first), so the divergence only appears once a dependency +/// edge is added between two already-installed modules. This test reproduces that: install +/// a, b, c standalone (row IDs a < b < c), then add edges a->b->c so dependency order (c, b, a) +/// is the reverse of row order. Reloading to register the edges preserves row IDs. +Method TestSyncAllProcessesDependenciesFirst() +{ + // Fixture source dirs (resolve before any work — GetModuleDir needs ^UnitTestRoot). + set srcA = ..GetModuleDir("sync-dep", "a") + set srcB = ..GetModuleDir("sync-dep", "b") + set srcC = ..GetModuleDir("sync-dep", "c") + + set depRoot = ##class(%File).NormalizeDirectory(##class(%File).ManagerDirectory() _ "sync-dep-work-" _ $job) + set workA = ##class(%File).NormalizeDirectory(depRoot _ "a") + set workB = ##class(%File).NormalizeDirectory(depRoot _ "b") + set workC = ##class(%File).NormalizeDirectory(depRoot _ "c") + + try { + $$$ThrowOnError(##class(%IPM.Utils.File).CreateDirectoryChain(depRoot)) + do $$$AssertTrue(##class(%Library.File).CopyDir(srcA, workA, 1), "Copied sync-dep-a fixture") + do $$$AssertTrue(##class(%Library.File).CopyDir(srcB, workB, 1), "Copied sync-dep-b fixture") + do $$$AssertTrue(##class(%Library.File).CopyDir(srcC, workC, 1), "Copied sync-dep-c fixture") + + // Install standalone (no deps yet) in a, b, c order so row IDs are a < b < c. + $$$ThrowOnError(##class(%IPM.Main).Shell("load " _ workA _ " -dev")) + $$$ThrowOnError(##class(%IPM.Main).Shell("load " _ workB _ " -dev")) + $$$ThrowOnError(##class(%IPM.Main).Shell("load " _ workC _ " -dev")) + + // Add dependency edges: a depends on b, b depends on c. Dependency order is now c, b, a + // (the reverse of row order). Reloading registers the edges without changing row IDs. + do ..AddDependency(workA _ "module.xml", "SyncDepA.Top.CLS", "sync-dep-b") + do ..AddDependency(workB _ "module.xml", "SyncDepB.Middle.CLS", "sync-dep-c") + $$$ThrowOnError(##class(%IPM.Main).Shell("load " _ workA _ " -dev")) + $$$ThrowOnError(##class(%IPM.Main).Shell("load " _ workB _ " -dev")) + + // Modify a file in every module so each one actually syncs (and prints its marker). + do ..ReplaceInFile(workA _ "src/cls/SyncDepA/Top.cls", """a-original""", """a-modified""") + do ..ReplaceInFile(workB _ "src/cls/SyncDepB/Middle.cls", """b-original""", """b-modified""") + do ..ReplaceInFile(workC _ "src/cls/SyncDepC/Base.cls", """c-original""", """c-modified""") + + do ##class(%IPM.Utils.Module).BeginCaptureOutput(.cookie) + set sc = ##class(%IPM.Main).Shell("sync") + do ##class(%IPM.Utils.Module).EndCaptureOutput(cookie, .output) + do $$$AssertStatusOK(sc, "Naked sync succeeds") + + set idxA = ..FirstLineContaining(.output, "sync-dep-a") + set idxB = ..FirstLineContaining(.output, "sync-dep-b") + set idxC = ..FirstLineContaining(.output, "sync-dep-c") + do $$$AssertTrue(idxA > 0, "sync-dep-a appears in output") + do $$$AssertTrue(idxB > 0, "sync-dep-b appears in output") + do $$$AssertTrue(idxC > 0, "sync-dep-c appears in output") + + // c (least dependent) before b before a (most dependent). + do $$$AssertTrue(idxC < idxB, "Dependency sync-dep-c is processed before sync-dep-b") + do $$$AssertTrue(idxB < idxA, "Dependency sync-dep-b is processed before sync-dep-a") + } catch e { + do $$$AssertStatusOK(e.AsStatus(), "No unexpected exception during dependency-order sync") + } + + // Cleanup: uninstall dependent-first (a needs b needs c) and remove the work tree. + do ##class(%IPM.Main).Shell("uninstall sync-dep-a") + do ##class(%IPM.Main).Shell("uninstall sync-dep-b") + do ##class(%IPM.Main).Shell("uninstall sync-dep-c") + do ##class(%Library.File).RemoveDirectoryTree(depRoot) +} + /// Sync fails with an error when given a module name that isn't installed. Method TestSyncModuleNotFound() { @@ -488,4 +560,37 @@ ClassMethod FindInOutput(ByRef output, searchString As %String) As %Boolean return 0 } +/// Returns the 1-based line index of the first captured output line containing searchString, +/// or 0 if not found. Used to assert relative ordering of per-module output in sync-all. +ClassMethod FirstLineContaining(ByRef output, searchString As %String) As %Integer +{ + set sub = "" + for { + set sub = $order(output(sub), 1, line) + quit:sub="" + if line [ searchString { + return sub + } + } + return 0 +} + +/// Rewrites a module.xml to insert a single-dependency block after the given +/// line. Assumes the resource line is present and no block exists yet. +ClassMethod AddDependency( + moduleXmlPath As %String, + afterResource As %String, + dependencyName As %String) +{ + set resourceLine = "" + set depBlock = resourceLine _ $char(10) + _ " " _ $char(10) + _ " " _ $char(10) + _ " " _ dependencyName _ "" _ $char(10) + _ " 1.0.0" _ $char(10) + _ " " _ $char(10) + _ " " + do ..ReplaceInFile(moduleXmlPath, resourceLine, depBlock) +} + } diff --git a/tests/integration_tests/Test/PM/Integration/_data/sync-dep/a/module.xml b/tests/integration_tests/Test/PM/Integration/_data/sync-dep/a/module.xml new file mode 100644 index 000000000..2601c8ef0 --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/sync-dep/a/module.xml @@ -0,0 +1,13 @@ + + + + + + sync-dep-a + 1.0.0 + module + src + + + + diff --git a/tests/integration_tests/Test/PM/Integration/_data/sync-dep/a/src/cls/SyncDepA/Top.cls b/tests/integration_tests/Test/PM/Integration/_data/sync-dep/a/src/cls/SyncDepA/Top.cls new file mode 100644 index 000000000..954492e03 --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/sync-dep/a/src/cls/SyncDepA/Top.cls @@ -0,0 +1,9 @@ +Class SyncDepA.Top +{ + +ClassMethod Version() As %String +{ + quit "a-original" +} + +} diff --git a/tests/integration_tests/Test/PM/Integration/_data/sync-dep/b/module.xml b/tests/integration_tests/Test/PM/Integration/_data/sync-dep/b/module.xml new file mode 100644 index 000000000..f65fbf0fa --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/sync-dep/b/module.xml @@ -0,0 +1,13 @@ + + + + + + sync-dep-b + 1.0.0 + module + src + + + + diff --git a/tests/integration_tests/Test/PM/Integration/_data/sync-dep/b/src/cls/SyncDepB/Middle.cls b/tests/integration_tests/Test/PM/Integration/_data/sync-dep/b/src/cls/SyncDepB/Middle.cls new file mode 100644 index 000000000..6cbb6556a --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/sync-dep/b/src/cls/SyncDepB/Middle.cls @@ -0,0 +1,9 @@ +Class SyncDepB.Middle +{ + +ClassMethod Version() As %String +{ + quit "b-original" +} + +} diff --git a/tests/integration_tests/Test/PM/Integration/_data/sync-dep/c/module.xml b/tests/integration_tests/Test/PM/Integration/_data/sync-dep/c/module.xml new file mode 100644 index 000000000..36305d642 --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/sync-dep/c/module.xml @@ -0,0 +1,13 @@ + + + + + + sync-dep-c + 1.0.0 + module + src + + + + diff --git a/tests/integration_tests/Test/PM/Integration/_data/sync-dep/c/src/cls/SyncDepC/Base.cls b/tests/integration_tests/Test/PM/Integration/_data/sync-dep/c/src/cls/SyncDepC/Base.cls new file mode 100644 index 000000000..759e6ed5c --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/sync-dep/c/src/cls/SyncDepC/Base.cls @@ -0,0 +1,9 @@ +Class SyncDepC.Base +{ + +ClassMethod Version() As %String +{ + quit "c-original" +} + +} From 35104526a67801a8e2d63df9b321071471c30b9b Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Tue, 28 Jul 2026 13:55:11 -0400 Subject: [PATCH 34/39] Add summary section to sync --- src/cls/IPM/General/Sync/Summary.cls | 195 +++++++++++++++++++++++++ src/cls/IPM/Lifecycle/Base.cls | 10 ++ src/cls/IPM/Main.cls | 30 ++-- src/cls/IPM/ResourceProcessor/Test.cls | 42 ++++++ 4 files changed, 267 insertions(+), 10 deletions(-) create mode 100644 src/cls/IPM/General/Sync/Summary.cls diff --git a/src/cls/IPM/General/Sync/Summary.cls b/src/cls/IPM/General/Sync/Summary.cls new file mode 100644 index 000000000..ea2e5fe59 --- /dev/null +++ b/src/cls/IPM/General/Sync/Summary.cls @@ -0,0 +1,195 @@ +/// Naked-sync summary accumulator. Main.Sync ("sync" with no module named) opens the summary with +/// Begin, runs its per-module loop, then prints with Report. While open, each module's %Sync +/// records its own outcome into the process-private global ^||IPM.Sync.Summary so the reader can +/// aggregate without threading data through ExecutePhases' by-ref params. Shape: +/// ^||IPM.Sync.Summary("Active") = 1 while a sync-all is in progress +/// ^||IPM.Sync.Summary("Module",name,"File",relPath) = "Updated" | "Deleted" +/// ^||IPM.Sync.Summary("Module",name,"SyncOK") = 1 once file sync committed (pre-tests) +/// ^||IPM.Sync.Summary("Module",name,"TestsPassed") = running count of passed test methods +/// ^||IPM.Sync.Summary("Module",name,"TestsFailed") = running count of failed test methods +/// ^||IPM.Sync.Summary("Module",name,"Skipped",seq) = verify-scoped test class not run +/// ^||IPM.Sync.Summary("Module",name,"Warning",seq) = warning text +/// Every recorder is a no-op unless "Active" is set, so a single-module sync (which never calls +/// Begin) records nothing and prints no summary. +/// +/// SyncOK separates a genuine sync failure from a test failure: %Sync runs tests with +/// FailuresAreFatal, so a failing test makes %Sync return an error status just like a compile +/// failure would. %Sync sets SyncOK once file sync is committed (before tests run), so the +/// reader can tell the two apart — a module in the failures list with SyncOK set had its files +/// synced fine and only a test failed; without SyncOK the sync itself failed. +Class %IPM.General.Sync.Summary [ Abstract ] +{ + +ClassMethod Begin() [ Internal ] +{ + kill ^||IPM.Sync.Summary + set ^||IPM.Sync.Summary("Active") = 1 +} + +ClassMethod End() [ Internal ] +{ + kill ^||IPM.Sync.Summary +} + +/// Record one synced file for the module. action is "Updated" or "Deleted". +ClassMethod RecordFile( + moduleName As %String, + relPath As %String, + action As %String) [ Internal ] +{ + if '$get(^||IPM.Sync.Summary("Active")) { + quit + } + set ^||IPM.Sync.Summary("Module", moduleName, "File", relPath) = action +} + +/// Mark that the module's file sync committed successfully. Called by %Sync just before it runs +/// tests, so a later test failure (which returns an error status) is distinguishable from a +/// sync/compile failure that never reached this point. +ClassMethod MarkSyncOK(moduleName As %String) [ Internal ] +{ + if '$get(^||IPM.Sync.Summary("Active")) { + quit + } + set ^||IPM.Sync.Summary("Module", moduleName, "SyncOK") = 1 +} + +/// Record a verify-scoped test class that sync detected as changed but did not run (needs +/// `verify`). Called once per skipped class by SyncRunTests. +ClassMethod AddSkipped( + moduleName As %String, + className As %String) [ Internal ] +{ + if '$get(^||IPM.Sync.Summary("Active")) { + quit + } + set seq = $increment(^||IPM.Sync.Summary("Module", moduleName, "SkippedCount")) + set ^||IPM.Sync.Summary("Module", moduleName, "Skipped", seq) = className +} + +/// Record one advisory warning for the module (e.g. module.xml reloaded, deletes not applied). +ClassMethod AddWarning( + moduleName As %String, + text As %String) [ Internal ] +{ + if '$get(^||IPM.Sync.Summary("Active")) { + quit + } + set seq = $increment(^||IPM.Sync.Summary("Module", moduleName, "WarningCount")) + set ^||IPM.Sync.Summary("Module", moduleName, "Warning", seq) = text +} + +/// Add to the module's running test-method pass/fail tally. Called once per test resource by +/// OnSyncRunTests. +ClassMethod AddTests( + moduleName As %String, + passed As %Integer, + failed As %Integer) [ Internal ] +{ + if '$get(^||IPM.Sync.Summary("Active")) { + quit + } + set ^||IPM.Sync.Summary("Module", moduleName, "TestsPassed") = $get(^||IPM.Sync.Summary("Module", moduleName, "TestsPassed"), 0) + passed + set ^||IPM.Sync.Summary("Module", moduleName, "TestsFailed") = $get(^||IPM.Sync.Summary("Module", moduleName, "TestsFailed"), 0) + failed +} + +/// Print the bordered sync-all summary from the accumulated ^||IPM.Sync.Summary data. +/// orderedNames is the full dependency-ordered dev-mode module list (the "checked" set); +/// failures is the $list of module names whose Sync phase returned an error status. A module in +/// failures that still recorded SyncOK synced its files fine and only had a failing test — that +/// is reflected in its test tally, not the Errors row (see SyncOK). A per-module line is shown +/// only for modules that were updated, had a real sync error, or ran/skipped tests — unchanged +/// modules are implied by the "checked" count. +ClassMethod Report( + orderedNames As %List, + failures As %List = "") [ Internal ] +{ + if '$get(^||IPM.Sync.Summary("Active")) { + quit + } + set border = $translate($justify("", 64), " ", "=") + set checkedCount = $listlength(orderedNames) + + // Header count: modules that actually had at least one file change. + set updatedCount = 0 + for i = 1:1:checkedCount { + set name = $list(orderedNames, i) + if $data(^||IPM.Sync.Summary("Module", name, "File")) { + set updatedCount = updatedCount + 1 + } + } + + write !!, border + write !, "Sync Summary" + write !, border + write !, "Modules checked: ", checkedCount, " (", $listtostring(orderedNames, ", "), ")" + write !, "Modules updated: ", updatedCount + + set warningCount = 0, errorNames = "" + kill warnings + for i = 1:1:checkedCount { + set name = $list(orderedNames, i) + + set fileCount = 0 + set relPath = "" + for { + set relPath = $order(^||IPM.Sync.Summary("Module", name, "File", relPath)) + quit:relPath="" + set fileCount = fileCount + 1 + } + // A real sync error: the Sync phase failed AND file sync never committed. If SyncOK is + // set, the phase error came from a fatal test failure, not from sync itself. + set isError = $listfind(failures, name) && '$get(^||IPM.Sync.Summary("Module", name, "SyncOK")) + if isError { + set errorNames = errorNames _ $listbuild(name) + } + set passed = $get(^||IPM.Sync.Summary("Module", name, "TestsPassed")) + set failed = $get(^||IPM.Sync.Summary("Module", name, "TestsFailed")) + set hasTests = (passed '= "") || (failed '= "") + set skippedCount = $get(^||IPM.Sync.Summary("Module", name, "SkippedCount"), 0) + + if fileCount || isError || hasTests || skippedCount { + set line = " " _ name + if fileCount { + set line = line _ " " _ fileCount _ " file" _ $select(fileCount = 1:"", 1:"s") + } + if hasTests { + // "passed/total" already conveys the failure count; no redundant "N failed". + set line = line _ " tests " _ (+passed) _ "/" _ ((+passed) + (+failed)) + } + if isError { + set line = line _ " ERROR" + } + write !, line + // Enumerate verify-scoped test classes that were detected but not run. + set sseq = "" + for { + set sseq = $order(^||IPM.Sync.Summary("Module", name, "Skipped", sseq), 1, skippedClass) + quit:sseq="" + write !, " skipped (verify-scoped): ", skippedClass + } + } + + // Collect this module's warnings for the aggregated section below. + set wseq = "" + for { + set wseq = $order(^||IPM.Sync.Summary("Module", name, "Warning", wseq), 1, wtext) + quit:wseq="" + set warningCount = warningCount + 1 + set warnings(warningCount) = " [" _ name _ "] " _ wtext + } + } + + if errorNames '= "" { + write !, "Errors: ", $listtostring(errorNames, ", ") + } + if warningCount { + write !, "Warnings: ", warningCount + for i = 1:1:warningCount { + write !, warnings(i) + } + } + write !, border +} + +} diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls index 0e02ca627..ec6f28060 100644 --- a/src/cls/IPM/Lifecycle/Base.cls +++ b/src/cls/IPM/Lifecycle/Base.cls @@ -705,6 +705,7 @@ Method %Sync(ByRef params) As %Status set key = "" for { set key = $order(deleted(key)) quit:key="" set delCount = delCount + 1 } write !, " ", delCount, " deleted file(s) detected but not applied. Use -delete to remove from server." + do ##class(%IPM.General.Sync.Summary).AddWarning(moduleName, delCount _ " deleted file(s) detected but not applied; use -delete to remove from server.") } if moduleXmlChanged { do ..SyncCommitModuleXml(module, moduleXmlPath, moduleXmlRelPath) @@ -763,6 +764,7 @@ Method %Sync(ByRef params) As %Status quit:key="" set modCount = modCount + 1 write !, " Updated: ", key + do ##class(%IPM.General.Sync.Summary).RecordFile(moduleName, key, "Updated") } set delCount = 0 if processDeletes { @@ -772,6 +774,7 @@ Method %Sync(ByRef params) As %Status quit:key="" set delCount = delCount + 1 write !, " Deleted: ", key + do ##class(%IPM.General.Sync.Summary).RecordFile(moduleName, key, "Deleted") } } write !, "[", moduleName, "] Sync complete: ", modCount, " file(s) updated" @@ -789,6 +792,10 @@ Method %Sync(ByRef params) As %Status write !, "[", moduleName, "] Sync done in ", $fnumber($zhorolog - syncStart, "", 2), "s" + // File sync is committed and reported. Mark success before running tests so a fatal + // test failure below is reported as a test failure, not a sync error, in the summary. + do ##class(%IPM.General.Sync.Summary).MarkSyncOK(moduleName) + // Step 10: Run changed test-phase tests if -test flag is set (after sync is reported) if runTests { $$$ThrowOnError(..SyncRunTests(orderedResourceList, verbose, .params)) @@ -1093,6 +1100,7 @@ ClassMethod SyncRunTests( set skipClassName = $order(byResource(testResource.Name, skipClassName)) quit:skipClassName="" write:verbose !, "Skipping verify-scoped test: ", skipClassName, " (use 'verify' to run)" + do ##class(%IPM.General.Sync.Summary).AddSkipped(testResource.Module.Name, skipClassName) } continue } @@ -1129,6 +1137,7 @@ ClassMethod SyncPrintModuleXmlWarning(moduleName As %String) [ Private ] write !, "Warning: module.xml changed and was reloaded." write !, " Resources may have been added/removed. Run `reload ", moduleName, "` to fully apply" write !, " manifest-level changes (mappings, dependencies, defaults)." + do ##class(%IPM.General.Sync.Summary).AddWarning(moduleName, "module.xml changed and was reloaded; run `reload " _ moduleName _ "` to fully apply.") } ClassMethod SyncPrintUnsupportedNote(moduleName As %String, ByRef unsupportedResources) [ Private ] @@ -1149,6 +1158,7 @@ ClassMethod SyncPrintUnsupportedNote(moduleName As %String, ByRef unsupportedRes } write !, "[", moduleName, "] ", count, " resource(s) skipped (no sync support): ", names write !, " Use `reload ", moduleName, "` to apply changes to those resources." + do ##class(%IPM.General.Sync.Summary).AddWarning(moduleName, count _ " resource(s) skipped (no sync support): " _ names _ ".") } /// Recompile all compilable resources in the module to catch dependents invalidated by diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls index 4364460c1..104af88a0 100644 --- a/src/cls/IPM/Main.cls +++ b/src/cls/IPM/Main.cls @@ -2338,18 +2338,28 @@ ClassMethod Sync(ByRef commandInfo) [ Private ] set orderedNames = ..GetDevModeModulesInDependencyOrder() set found = $listlength(orderedNames) set failures = "" - for i = 1:1:found { - set name = $list(orderedNames, i) - set syncSC = ##class(%IPM.Storage.Module).ExecutePhases(name, $listbuild("Sync"), 1, .params) - if $$$ISERR(syncSC) { - set failures = failures _ $listbuild(name) - do $system.OBJ.DisplayError(syncSC) - } - } if 'found { write !, "No modules in development mode." - } elseif failures '= "" { - write !, "Sync completed with errors in: ", $listtostring(failures, ", ") + } else { + // Turn on per-module summary recording for the duration of the loop, then print the + // aggregated bordered summary. Kept in a try/finally so a mid-loop error still clears + // the process-private accumulator. + do ##class(%IPM.General.Sync.Summary).Begin() + try { + for i = 1:1:found { + set name = $list(orderedNames, i) + set syncSC = ##class(%IPM.Storage.Module).ExecutePhases(name, $listbuild("Sync"), 1, .params) + if $$$ISERR(syncSC) { + set failures = failures _ $listbuild(name) + do $system.OBJ.DisplayError(syncSC) + } + } + do ##class(%IPM.General.Sync.Summary).Report(orderedNames, failures) + } catch e { + do ##class(%IPM.General.Sync.Summary).End() + throw e + } + do ##class(%IPM.General.Sync.Summary).End() } } } diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls index 9d7fa4449..74f0ee941 100644 --- a/src/cls/IPM/ResourceProcessor/Test.cls +++ b/src/cls/IPM/ResourceProcessor/Test.cls @@ -527,6 +527,9 @@ Method OnSyncRunTests(ByRef classInfo, ByRef params) As %Status $$$ThrowOnError(sc) set suppressor = "" + // Tally this resource's per-method pass/fail into the naked-sync summary before + // ReportTestResults runs — at top level it consumes (kills) the result accumulator. + do ..SyncSummaryTallyTests(..ResourceReference.Module.Name, phaseStartIndex) $$$ThrowOnError(..ReportTestResults(phaseStartIndex, verbose, .params)) } catch e { set sc = e.AsStatus() @@ -535,6 +538,45 @@ Method OnSyncRunTests(ByRef classInfo, ByRef params) As %Status quit sc } +/// Count the test methods run since phaseStartIndex and forward their pass/fail tally to the +/// naked-sync summary. A method counts as failed if it errored or has any failed assertion, +/// mirroring how GetAllTestsStatus/OutputFailures classify a failure; otherwise it passed. +/// No-op unless a sync-all is in progress (%IPM.General.Sync.Summary gates on that). +ClassMethod SyncSummaryTallyTests( + moduleName As %String, + phaseStartIndex As %Integer) [ Private ] +{ + set passed = 0, failed = 0 + set testCount = $get(^||%UnitTest.Manager.AllResultsCount, 0) + for i = (phaseStartIndex + 1):1:testCount { + set logIndex = $get(^||%UnitTest.Manager.AllResults(i)) + continue:logIndex="" + set tree = ##class(%IPM.Test.Abstract).BuildResultTree(logIndex) + set suiteIter = tree.suites.%GetIterator() + while suiteIter.%GetNext(, .suiteObj) { + set caseIter = suiteObj.cases.%GetIterator() + while caseIter.%GetNext(, .caseObj) { + set methodIter = caseObj.methods.%GetIterator() + while methodIter.%GetNext(, .methodObj) { + set methodFailed = (methodObj.error '= "") + set assertIter = methodObj.asserts.%GetIterator() + while assertIter.%GetNext(, .assertObj) { + if assertObj.status = "failed" { + set methodFailed = 1 + } + } + if methodFailed { + set failed = failed + 1 + } else { + set passed = passed + 1 + } + } + } + } + } + do ##class(%IPM.General.Sync.Summary).AddTests(moduleName, passed, failed) +} + Method %OnValidateObject() As %Status [ Private, ServerOnly = 1 ] { if ((..Package = "") && (..Class = "")) || ((..Package '= "") && (..Class '= "")) { From 8945e2ab079724177798851e2dd8abd6652c6580 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Wed, 29 Jul 2026 14:16:01 -0400 Subject: [PATCH 35/39] Refactor sync into its own file and fix minor issues --- src/cls/IPM/General/Sync/Pipeline.cls | 667 +++++++++++++++++++++++++ src/cls/IPM/General/Sync/Summary.cls | 16 +- src/cls/IPM/Lifecycle/Base.cls | 648 +----------------------- src/cls/IPM/ResourceProcessor/Test.cls | 50 +- 4 files changed, 712 insertions(+), 669 deletions(-) create mode 100644 src/cls/IPM/General/Sync/Pipeline.cls diff --git a/src/cls/IPM/General/Sync/Pipeline.cls b/src/cls/IPM/General/Sync/Pipeline.cls new file mode 100644 index 000000000..f9bc60af9 --- /dev/null +++ b/src/cls/IPM/General/Sync/Pipeline.cls @@ -0,0 +1,667 @@ +Include (%occErrors, %IPM.Common) + +/// The incremental-sync pipeline for a dev-mode module. %IPM.Lifecycle.Base's %Sync phase is a +/// thin delegator to Run below; all the sync orchestration and its helpers live here so Base.cls +/// stays a lifecycle hook surface rather than the sync implementation. +Class %IPM.General.Sync.Pipeline [ Abstract ] +{ + +/// Incrementally sync changed files from disk into the namespace for a dev-mode module. +/// Detects files changed since last load/sync, routes them to resource processors, and recompiles. +/// module is the lifecycle's already-opened, validated module instance; SyncCheckModuleXml may +/// replace it with a freshly-reloaded instance if module.xml changed. +ClassMethod Run( + module As %IPM.Storage.Module, + ByRef params) As %Status +{ + set sc = $$$OK + try { + set syncStart = $zhorolog + set verbose = $get(params("Verbose"), 0) + set processDeletes = $get(params("ProcessDeletes"), 0) + set runTests = $get(params("RunTests"), 0) + + // Intentional ByRef mutation of caller's array: params("Sync") is the per-module scratch + // subtree for test-case tracking. Main.Sync reuses the same params array across modules, + // so without this kill a prior module's ChangedTestCases leak into this module's SyncRunTests. + kill params("Sync") + + set moduleName = module.Name + + if 'module.DeveloperMode { + $$$ThrowStatus($$$ERROR($$$GeneralError, $$$FormatText("Module '%1' is not in development mode. Sync requires dev mode.", moduleName))) + } + + set root = ##class(%File).NormalizeDirectory(module.Root) + if root = "" { + $$$ThrowStatus($$$ERROR($$$GeneralError, $$$FormatText("Module '%1' has no root directory configured.", moduleName))) + } + + // Step 1: Check if module.xml changed; reload manifest if so + set moduleXmlRelPath = ##class(%IPM.Storage.FileHash).NormalizePath("module.xml") + set moduleXmlPath = root _ "module.xml" + set moduleXmlChanged = ..SyncCheckModuleXml(.module, moduleXmlPath, moduleXmlRelPath) + + if '##class(%IPM.Storage.FileHash).HasBaseline(moduleName) { + // Self-heal: establish baseline for modules loaded before this feature + $$$ThrowOnError(##class(%IPM.Storage.FileHash).StampModule(module)) + write !, "[", moduleName, "] Baseline established. Run sync again to detect changes." + quit + } + + // No StampModule call here after a manifest reload: StampModule would overwrite current + // hashes for ALL files (including ones the user just edited), causing ComputeChanges to + // see current-vs-current and report zero changes for co-edited files. + // Newly-declared resources are handled without a full stamp: SyncBuildReverseIndex + // (step 3) calls ResolveChildren, which adds their derived relPaths to reverseIndex → + // manifestPaths. ComputeChanges Pass 2 finds those paths with no baseline row and + // reports them as modified, so they are loaded in this same sync call. + + // Step 2: Collect scan directories from resource processors, then walk only those. + // Each processor declares its owned directory via GetSyncDirectory(); sync never + // touches files outside declared locations. + set orderedResourceList = module.GetOrderedResourceList() + kill scanDirs + set rlKey = "" + for { + set rlResource = orderedResourceList.GetNext(.rlKey) + quit:rlKey="" + if '$isobject(rlResource.Processor) { + continue + } + if 'rlResource.Processor.SupportsSync() { + continue + } + set syncDir = rlResource.Processor.GetSyncDirectory() + if syncDir '= "" { + set scanDirs(syncDir) = "" + } + } + do ##class(%IPM.Storage.FileHash).DeduplicateScanDirs(.scanDirs) + + set walkStart = $zhorolog + kill allFiles, allHashes, bfsFiles + $$$ThrowOnError(##class(%IPM.Storage.FileHash).WalkAndHashDirs(root, .scanDirs, .allFiles, .allHashes)) + + // module.xml is always tracked — add it to allFiles/allHashes explicitly. + if ##class(%File).Exists(root _ "module.xml") { + set allFiles("module.xml") = root _ "module.xml" + set allHashes("module.xml") = $$$lcase(##class(%File).SHA1Hash(root _ "module.xml", 1)) + } + + // Build compilable-only subset for ComputeChanges Pass 1; count total for verbose. + set relPath = "", fileCount = 0 + for { + set relPath = $order(allFiles(relPath), 1, fullPath) + quit:relPath="" + set fileCount = fileCount + 1 + set ext = $$$lcase($piece(relPath, ".", *)) + if ",cls,inc,mac,int," [ (","_ext_",") { + set bfsFiles(relPath) = fullPath + } + } + if verbose { + set dirCount = 0 + set tmpDir = "" + for { set tmpDir = $order(scanDirs(tmpDir)) quit:tmpDir="" set dirCount = dirCount + 1 } + write !, "[", moduleName, "] Scanned ", fileCount, " file(s) across ", dirCount, " director(ies) in ", $fnumber($zhorolog - walkStart, "", 2), "s" + } + + // Step 3: Build reverse index (relPath -> owning resource + processor). + kill reverseIndex, unsupportedResources + do ..SyncBuildReverseIndex(module, orderedResourceList, .reverseIndex, .unsupportedResources, .allFiles) + + // Collect manifest-derived paths from reverseIndex for ComputeChanges. + kill manifestPaths + set riCount = 0 + set riKey = "" + for { + set riKey = $order(reverseIndex(riKey)) + quit:riKey="" + set manifestPaths(riKey) = "" + set riCount = riCount + 1 + } + if verbose { + write !, "[", moduleName, "] Tracking ", riCount, " path(s) across ", orderedResourceList.Count(), " resource(s)" + } + + // Step 4: Compute disk changes vs baseline + $$$ThrowOnError(##class(%IPM.Storage.FileHash).ComputeChanges(module, .modified, .deleted, .manifestPaths, .bfsFiles, .allFiles, .allHashes)) + + // Exclude module.xml from file routing (handled by SyncCheckModuleXml above) + kill modified(moduleXmlRelPath) + kill deleted(moduleXmlRelPath) + + // Falls through when only deletes exist and processDeletes=1 + if '$data(modified) && ('$data(deleted) || 'processDeletes) { + write !, "[", moduleName, "] Nothing to sync." + if '$data(modified) && $data(deleted) && 'processDeletes { + set delCount = 0 + set key = "" + for { set key = $order(deleted(key)) quit:key="" set delCount = delCount + 1 } + write !, " ", delCount, " deleted file(s) detected but not applied. Use -delete to remove from server." + do ##class(%IPM.General.Sync.Summary).AddWarning(moduleName, delCount _ " deleted file(s) detected but not applied; use -delete to remove from server.") + } + if moduleXmlChanged { + do ..SyncCommitModuleXml(module, moduleXmlPath, moduleXmlRelPath) + do ..SyncPrintModuleXmlWarning(moduleName) + } + if $data(unsupportedResources) { + do ..SyncPrintUnsupportedNote(moduleName, verbose, .unsupportedResources) + } + write !, "[", moduleName, "] Sync done in ", $fnumber($zhorolog - syncStart, "", 2), "s" + quit + } + + // Step 5: Partition changes by owning resource + kill syncByResource + do ..SyncRoutePathSet(.modified, .reverseIndex, "modified", .syncByResource) + if processDeletes { + do ..SyncRoutePathSet(.deleted, .reverseIndex, "deleted", .syncByResource) + } + + // Step 6: Dispatch OnSync to each processor; load unhandled compilable files + if verbose { + set resCount = 0 + set resName = "" + for { + set resName = $order(syncByResource(resName)) + quit:resName="" + set resCount = resCount + 1 + } + write !, "[", moduleName, "] Dispatching to ", resCount, " resource(s)" + } + $$$ThrowOnError(..SyncDispatchProcessors(module, root, verbose, .syncByResource, .params, .loadItems)) + + // Step 7: Compile the full resource set with u-flag to pick up dependent recompiles + if loadItems > 0 { + $$$ThrowOnError(..SyncCompile(module, orderedResourceList, verbose, .params)) + } + + // Step 8: Delete server-side documents for removed files, then recompile + if processDeletes && ($data(deleted) > 1) { + do ..SyncApplyDeletes(moduleName, .deleted, .reverseIndex, .syncByResource, verbose) + $$$ThrowOnError(..SyncCompile(module, orderedResourceList, verbose, .params)) + } + + // Step 9: Commit new hashes on success (skipped on error so next sync re-detects). + // Done before running tests below so a test failure doesn't prevent the file-sync + // outcome from being committed and reported — file sync and test results are independent. + $$$ThrowOnError(##class(%IPM.Storage.FileHash).CommitChanges(module, .modified, .deleted, processDeletes)) + if moduleXmlChanged { + do ..SyncCommitModuleXml(module, moduleXmlPath, moduleXmlRelPath) + } + + set modCount = 0 + set key = "" + for { + set key = $order(modified(key)) + quit:key="" + set modCount = modCount + 1 + write !, " Updated: ", key + do ##class(%IPM.General.Sync.Summary).RecordFile(moduleName, key, "Updated") + } + set delCount = 0 + if processDeletes { + set key = "" + for { + set key = $order(deleted(key)) + quit:key="" + set delCount = delCount + 1 + write !, " Deleted: ", key + do ##class(%IPM.General.Sync.Summary).RecordFile(moduleName, key, "Deleted") + } + } + write !, "[", moduleName, "] Sync complete: ", modCount, " file(s) updated" + if delCount > 0 { + write ", ", delCount, " deleted" + } + write "." + + if moduleXmlChanged { + do ..SyncPrintModuleXmlWarning(moduleName) + } + if $data(unsupportedResources) { + do ..SyncPrintUnsupportedNote(moduleName, verbose, .unsupportedResources) + } + + write !, "[", moduleName, "] Sync done in ", $fnumber($zhorolog - syncStart, "", 2), "s" + + // File sync is committed and reported. Mark success before running tests so a fatal + // test failure below is reported as a test failure, not a sync error, in the summary. + do ##class(%IPM.General.Sync.Summary).MarkSyncOK(moduleName) + + // Step 10: Run changed test-phase tests if -test flag is set (after sync is reported) + if runTests { + $$$ThrowOnError(..SyncRunTests(orderedResourceList, verbose, .params)) + } + + } catch e { + set sc = e.AsStatus() + } + quit sc +} + +/// Check if module.xml changed and reload the manifest if so. +/// Returns 1 if module.xml changed and was reloaded, 0 otherwise. +/// On reload, replaces module with the freshly-opened instance. +ClassMethod SyncCheckModuleXml( + ByRef module As %IPM.Storage.Module, + moduleXmlPath As %String, + moduleXmlRelPath As %String) As %Boolean [ Private ] +{ + if '##class(%File).Exists(moduleXmlPath) { + quit 0 + } + set existing = ##class(%IPM.Storage.FileHash).ModulePathIndexOpen(module.Name, moduleXmlRelPath) + if '$isobject(existing) { + quit 0 + } + set newHash = $$$lcase(##class(%File).SHA1Hash(moduleXmlPath, 1)) + if newHash = existing.Hash { + quit 0 + } + $$$ThrowOnError($system.OBJ.Load(moduleXmlPath, "-d")) + set module = ##class(%IPM.Storage.Module).NameOpen(module.Name, , .openSC) + $$$ThrowOnError(openSC) + quit 1 +} + +/// Build a reverse index: normalizedRelPath -> resource name, Processor, Resource object. +/// Used by SyncRoutePathSet to map changed files back to their owning resource processors. +/// Skips resources whose processor does not support sync — those are collected in +/// unsupportedResources(resourceName)="" for informational display. +/// +/// Step 1 builds docToResource (docName → owner) from ResolveChildren, and prefix-scans +/// allFiles for directory-owned resources (e.g. test dirs) via GetSyncDirectory(). +/// Step 2 maps all stored baseline paths to owners via RelPathToDocName → docToResource. +ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, orderedResourceList As %ListOfObjects, ByRef reverseIndex, Output unsupportedResources, ByRef allFiles) [ Private ] +{ + kill unsupportedResources + // docToResource maps server document names (e.g. "SyncTest.SuperClass.CLS") to the + // resource that owns them. This lets us resolve filesystem paths → owners in O(1) below, + // since RelPathToDocName converts a relPath to a docName deterministically. + kill docToResource + set key = "" + for { + set resource = orderedResourceList.GetNext(.key) + quit:key="" + + if '$isobject(resource.Processor) { + continue + } + + if 'resource.Processor.SupportsSync() { + set unsupportedResources(resource.Name) = "" + continue + } + + // ResolveChildren returns childArr keyed by document name (server-side identifier). + kill childArr + set childSC = resource.ResolveChildren(.childArr) + if $$$ISERR(childSC) { + continue + } + + set childName = "" + for { + set childName = $order(childArr(childName)) + quit:childName="" + set docToResource(childName) = resource.Name + set docToResource(childName, "Processor") = resource.Processor + set docToResource(childName, "Resource") = resource + + // Newly-declared resources have no baseline row yet (StampModule skips uncompiled + // classes), so GetStoredPaths below won't find them. Map their relPath directly. + set relPath = $get(childArr(childName, "RelativePath")) + if relPath = "" { + set relPath = resource.Processor.OnItemRelativePath(childName) + } + if relPath '= "" { + set normalizedRelPath = ##class(%IPM.Storage.FileHash).NormalizePath(relPath) + set reverseIndex(normalizedRelPath) = resource.Name + set reverseIndex(normalizedRelPath, "Processor") = resource.Processor + set reverseIndex(normalizedRelPath, "Resource") = resource + } + } + + // Directory-based resources (e.g. test dirs) own all files under their declared + // directory. Prefix-scan allFiles to map every file under that dir to this resource. + set syncDir = resource.Processor.GetSyncDirectory() + if syncDir '= "" { + set prefix = ##class(%IPM.Storage.FileHash).NormalizePath(syncDir _ "/") + set prefixLen = $length(prefix) + set dirRelPath = prefix + for { + set dirRelPath = $order(allFiles(dirRelPath)) + quit:dirRelPath="" + quit:($extract(dirRelPath, 1, prefixLen) '= prefix) + if '$data(reverseIndex(dirRelPath)) { + set reverseIndex(dirRelPath) = resource.Name + set reverseIndex(dirRelPath, "Processor") = resource.Processor + set reverseIndex(dirRelPath, "Resource") = resource + } + } + } + } + + // Resolve compilable paths: every baseline path not already claimed above gets mapped + // through RelPathToDocName → docToResource. + kill storedPaths + do ##class(%IPM.Storage.FileHash).GetStoredPaths(module.Name, .storedPaths) + set relPath = "" + for { + set relPath = $order(storedPaths(relPath)) + quit:relPath="" + + if $data(reverseIndex(relPath)) { + continue + } + + set docName = ##class(%IPM.Storage.FileHash).RelPathToDocName(relPath) + if docName = "" { + continue + } + if '$data(docToResource(docName)) { + continue + } + + set reverseIndex(relPath) = docToResource(docName) + set reverseIndex(relPath, "Processor") = docToResource(docName, "Processor") + set reverseIndex(relPath, "Resource") = docToResource(docName, "Resource") + } +} + +/// Route a set of changed paths to their owning resources, categorized by type (modified/deleted). +/// Only sync-supporting resources appear in reverseIndex, so no filtering needed here. +ClassMethod SyncRoutePathSet( + ByRef paths, + ByRef reverseIndex, + category As %String, + ByRef syncByResource) [ Private ] +{ + set relPath = "" + for { + set relPath = $order(paths(relPath)) + quit:relPath="" + + if '$data(reverseIndex(relPath)) { + continue + } + set resName = reverseIndex(relPath) + set syncByResource(resName, category, relPath) = "" + set syncByResource(resName, "Processor") = reverseIndex(relPath, "Processor") + set syncByResource(resName, "Resource") = reverseIndex(relPath, "Resource") + } +} + +/// Call OnSync on each processor, then load any unhandled compilable files. +/// Returns the number of files loaded (used to decide whether SyncCompile is needed). +/// If a processor sets handled=1 in OnSync, it fully owns the sync for that resource +/// (e.g. Test reloads its own way). Otherwise the default path loads+compiles each file. +ClassMethod SyncDispatchProcessors( + module As %IPM.Storage.Module, + root As %String, + verbose As %Boolean, + ByRef syncByResource, + ByRef params, + Output loadItems As %Integer = 0) As %Status [ Private ] +{ + set sc = $$$OK + try { + set resName = "" + for { + set resName = $order(syncByResource(resName)) + quit:resName="" + + if '$data(syncByResource(resName, "Processor")) { + continue + } + set processor = syncByResource(resName, "Processor") + + kill resModified, resDeleted + merge resModified = syncByResource(resName, "modified") + merge resDeleted = syncByResource(resName, "deleted") + + set handled = 0 + $$$ThrowOnError(processor.OnSync(.resModified, .resDeleted, .params, .handled)) + + if handled || 'processor.%IsA("%IPM.ResourceProcessor.AbstractCompilable") { + continue + } + + // Default: load each changed file; SyncCompile will do a full compile with u-flag + set loadRelPath = "" + for { + set loadRelPath = $order(resModified(loadRelPath)) + quit:loadRelPath="" + + set fullPath = ##class(%File).NormalizeFilename(loadRelPath, root) + if ##class(%File).Exists(fullPath) { + set loadFlags = $select(verbose:"d", 1:"-d") + $$$ThrowOnError($system.OBJ.Load(fullPath, loadFlags _ "c")) + set loadItems = loadItems + 1 + } + } + } + } catch e { + set sc = e.AsStatus() + } + quit sc +} + +/// Delete server-side documents for compilable deleted files. +/// Guards skip paths with no owner, no processor, or non-compilable processors — those are +/// handled elsewhere (unsupported warnings) or aren't server-side documents at all. +ClassMethod SyncApplyDeletes( + moduleName As %String, + ByRef deleted, + ByRef reverseIndex, + ByRef syncByResource, + verbose As %Boolean) As %Status [ Private ] +{ + set sc = $$$OK + set relPath = "" + for { + set relPath = $order(deleted(relPath)) + quit:relPath="" + + if '$data(reverseIndex(relPath)) { + continue + } + set resName = reverseIndex(relPath) + if '$data(syncByResource(resName, "Processor")) { + continue + } + set processor = syncByResource(resName, "Processor") + if 'processor.%IsA("%IPM.ResourceProcessor.AbstractCompilable") { + continue + } + set docName = ##class(%IPM.Storage.FileHash).RelPathToDocName(relPath) + if docName '= "" { + set delFlags = $select(verbose:"d", 1:"-d") + set delSC = $system.OBJ.Delete(docName, delFlags) + // Delete failures are non-fatal: the SyncCompile pass that immediately follows + // will fail to compile any class that still references the deleted doc, surfacing + // the error with full context. Aborting the delete loop here would leave other + // deletions unapplied and make the overall error harder to diagnose. + // Non-fatal: the SyncCompile pass that follows will surface any compilation errors + // caused by the still-present document, giving the user full context. Aborting here + // would leave remaining deletions unapplied and make the overall error harder to diagnose. + if $$$ISERR(delSC) { + set errText = $system.Status.GetOneErrorText(delSC) + write !, "Warning: could not delete ", docName, ": ", errText + do ##class(%IPM.General.Sync.Summary).AddWarning(moduleName, "Could not delete " _ docName _ ": " _ errText) + set sc = $$$ADDSC(sc, delSC) + } + } + } + quit sc +} + +/// Run test-phase tests for changed test case classes recorded in params("Sync","ChangedTestCases"). +/// Groups changed classes by owning resource first, then dispatches one batched RunTest call +/// per resource (via OnSyncRunTests) instead of one call per class — avoids N separate +/// reload/compile/run cycles when several test classes in the same resource changed together. +ClassMethod SyncRunTests( + orderedResourceList As %ListOfObjects, + verbose As %Boolean, + ByRef params) As %Status [ Private ] +{ + set sc = $$$OK + try { + // Step 1: group changed classes by owning resource: byResource(resourceName, className) = "" + kill byResource + set className = "" + for { + set className = $order(params("Sync", "ChangedTestCases", className), 1, owningResource) + quit:className="" + set byResource(owningResource, className) = "" + } + + // Step 2: one OnSyncRunTests call per resource + set testKey = "" + for { + set testResource = orderedResourceList.GetNext(.testKey) + quit:testKey="" + + if '$isobject(testResource.Processor) { + continue + } + if 'testResource.Processor.%IsA("%IPM.ResourceProcessor.Test") { + continue + } + if '$data(byResource(testResource.Name)) { + continue + } + if '$listfind(testResource.Processor.Phase, "test") { + set skipClassName = "" + for { + set skipClassName = $order(byResource(testResource.Name, skipClassName)) + quit:skipClassName="" + write:verbose !, "Skipping verify-scoped test: ", skipClassName, " (use 'verify' to run)" + do ##class(%IPM.General.Sync.Summary).AddSkipped(testResource.Module.Name, skipClassName) + } + continue + } + kill classInfo + merge classInfo = byResource(testResource.Name) + kill testParams + merge testParams = params + set testParams("DeveloperMode") = 1 + $$$ThrowOnError(testResource.Processor.OnSyncRunTests(.classInfo, .testParams)) + } + } catch e { + set sc = e.AsStatus() + } + quit sc +} + +/// Commit the current module.xml hash to the baseline. +ClassMethod SyncCommitModuleXml( + module As %IPM.Storage.Module, + moduleXmlPath As %String, + moduleXmlRelPath As %String) [ Private ] +{ + kill moduleXmlMod, emptyDel + set moduleXmlMod(moduleXmlRelPath) = $$$lcase(##class(%File).SHA1Hash(moduleXmlPath, 1)) + set commitSC = ##class(%IPM.Storage.FileHash).CommitChanges(module, .moduleXmlMod, .emptyDel, 0) + if $$$ISERR(commitSC) { + // Non-fatal: the next sync will re-detect module.xml as changed and reload the manifest again. + write !, "Warning: failed to record module.xml hash: ", $system.Status.GetOneErrorText(commitSC) + } +} + +ClassMethod SyncPrintModuleXmlWarning(moduleName As %String) [ Private ] +{ + write ! + write !, "Warning: module.xml changed and was reloaded." + write !, " Resources may have been added/removed. Run `reload ", moduleName, "` to fully apply" + write !, " manifest-level changes (mappings, dependencies, defaults)." + do ##class(%IPM.General.Sync.Summary).AddWarning(moduleName, "module.xml changed and was reloaded; run `reload " _ moduleName _ "` to fully apply.") +} + +ClassMethod SyncPrintUnsupportedNote(moduleName As %String, verbose As %Boolean, ByRef unsupportedResources) [ Private ] +{ + set count = 0 + set names = "" + set resName = "" + for { + set resName = $order(unsupportedResources(resName)) + quit:resName="" + set count = count + 1 + if count <= 3 { + set names = names _ $select(names="":"", 1:", ") _ resName + } + } + if count > 3 { + set names = names _ ", ... (" _ (count - 3) _ " more)" + } + if verbose { + write !, "[", moduleName, "] ", count, " resource(s) skipped (no sync support): ", names + write !, " Use `reload ", moduleName, "` to apply changes to those resources." + } + do ##class(%IPM.General.Sync.Summary).AddWarning(moduleName, count _ " resource(s) skipped (no sync support): " _ names _ ".") +} + +/// Recompile all compilable resources in the module to catch dependents invalidated by +/// changes loaded in SyncDispatchProcessors. Skips CompileFromProject resources — those are +/// deployed code loaded via Studio project files (.prj); compiling them individually would +/// fail because deployed classes have no source in the routine database. +ClassMethod SyncCompile(module As %IPM.Storage.Module, orderedResourceList As %ListOfObjects, verbose As %Boolean = 0, ByRef params) As %Status [ Private ] +{ + set sc = $$$OK + try { + kill compileArray + + set key = "" + for { + set resource = orderedResourceList.GetNext(.key) + quit:key="" + + if '$isobject(resource.Processor) { + continue + } + if 'resource.Processor.%IsA("%IPM.ResourceProcessor.AbstractCompilable") { + continue + } + if 'resource.IsInScope("Compile") { + continue + } + if resource.Processor.CompileFromProject { + continue + } + + kill oneResourceList + set oneResourceList(resource.Name) = "" + set resSC = resource.ResolveChildren(.oneResourceList) + if $$$ISERR(resSC) { + continue + } + + set childKey = "" + for { + set childKey = $order(oneResourceList(childKey)) + quit:childKey="" + set ext = $zconvert($piece(childKey, ".", *), "U") + if ext = "CLS" { + set className = $piece(childKey, ".", 1, *-1) + if '$$$comClassDefined(className) { + continue + } + } + set compileArray(childKey) = "" + } + } + + if $data(compileArray) > 1 { + set flags = $select(verbose:"d", 1:"-d") _ "cku" + set sc = ##class(%IPM.Utils.LegacyCompat).UpdateSuperclassAndCompile(.compileArray, .flags) + } + } catch e { + set sc = e.AsStatus() + } + quit sc +} + +} diff --git a/src/cls/IPM/General/Sync/Summary.cls b/src/cls/IPM/General/Sync/Summary.cls index ea2e5fe59..db19dbce1 100644 --- a/src/cls/IPM/General/Sync/Summary.cls +++ b/src/cls/IPM/General/Sync/Summary.cls @@ -122,7 +122,21 @@ ClassMethod Report( write !!, border write !, "Sync Summary" write !, border - write !, "Modules checked: ", checkedCount, " (", $listtostring(orderedNames, ", "), ")" + + // Write module name list, wrapping at border width with aligned continuation lines. + set linePrefix = "Modules checked: " _ checkedCount _ " (" + set contPrefix = $justify("", $length(linePrefix)) + set nameLine = linePrefix + for i = 1:1:checkedCount { + set name = $list(orderedNames, i) + set sep = $select(i < checkedCount: ", ", 1: ")") + if ($length(nameLine) + $length(name) + $length(sep)) > $length(border) && (nameLine '= linePrefix) { + write !, nameLine + set nameLine = contPrefix + } + set nameLine = nameLine _ name _ sep + } + write !, nameLine write !, "Modules updated: ", updatedCount set warningCount = 0, errorNames = "" diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls index ec6f28060..1c7c5c52f 100644 --- a/src/cls/IPM/Lifecycle/Base.cls +++ b/src/cls/IPM/Lifecycle/Base.cls @@ -573,652 +573,12 @@ Method %Unconfigure(ByRef pParams) As %Status } /// Incrementally sync changed files from disk into the namespace for a dev-mode module. -/// Detects files changed since last load/sync, routes them to resource processors, and recompiles. +/// Thin lifecycle hook: ExecutePhases dispatches phases via $method on the lifecycle instance, +/// so this must be an instance method here. The sync orchestration itself lives in +/// %IPM.General.Sync.Pipeline. Method %Sync(ByRef params) As %Status { - set sc = $$$OK - try { - set syncStart = $zhorolog - set verbose = $get(params("Verbose"), 0) - set processDeletes = $get(params("ProcessDeletes"), 0) - set runTests = $get(params("RunTests"), 0) - - // params may be reused across modules by Main.Sync's sync-all-dev-mode-modules loop. - // Clear this module's own scratch subtree so a prior module's recorded test-case - // changes can't leak into this module's SyncRunTests dispatch. - kill params("Sync") - - // The lifecycle already opened and validated this module; use it directly. - // SyncCheckModuleXml may replace this local with a freshly-reloaded instance. - set module = ..Module - set moduleName = module.Name - - if 'module.DeveloperMode { - $$$ThrowStatus($$$ERROR($$$GeneralError, $$$FormatText("Module '%1' is not in development mode. Sync requires dev mode.", moduleName))) - } - - set root = ##class(%File).NormalizeDirectory(module.Root) - if root = "" { - $$$ThrowStatus($$$ERROR($$$GeneralError, $$$FormatText("Module '%1' has no root directory configured.", moduleName))) - } - - // Step 1: Check if module.xml changed; reload manifest if so - set moduleXmlRelPath = ##class(%IPM.Storage.FileHash).NormalizePath("module.xml") - set moduleXmlPath = root _ "module.xml" - set moduleXmlChanged = ..SyncCheckModuleXml(.module, moduleXmlPath, moduleXmlRelPath) - - if '##class(%IPM.Storage.FileHash).HasBaseline(moduleName) { - // Self-heal: establish baseline for modules loaded before this feature - $$$ThrowOnError(##class(%IPM.Storage.FileHash).StampModule(module)) - write !, "[", moduleName, "] Baseline established. Run sync again to detect changes." - quit - } - - // No StampModule call here after a manifest reload: StampModule would overwrite current - // hashes for ALL files (including ones the user just edited), causing ComputeChanges to - // see current-vs-current and report zero changes for co-edited files. - // Newly-declared resources are handled without a full stamp: SyncBuildReverseIndex - // (step 3) calls ResolveChildren, which adds their derived relPaths to reverseIndex → - // manifestPaths. ComputeChanges Pass 2 finds those paths with no baseline row and - // reports them as modified, so they are loaded in this same sync call. - - // Step 2: Collect scan directories from resource processors, then walk only those. - // Each processor declares its owned directory via GetSyncDirectory(); sync never - // touches files outside declared locations. - set orderedResourceList = module.GetOrderedResourceList() - kill scanDirs - set rlKey = "" - for { - set rlResource = orderedResourceList.GetNext(.rlKey) - quit:rlKey="" - if '$isobject(rlResource.Processor) { - continue - } - if 'rlResource.Processor.SupportsSync() { - continue - } - set syncDir = rlResource.Processor.GetSyncDirectory() - if syncDir '= "" { - set scanDirs(syncDir) = "" - } - } - do ##class(%IPM.Storage.FileHash).DeduplicateScanDirs(.scanDirs) - - set walkStart = $zhorolog - kill allFiles, allHashes, bfsFiles - $$$ThrowOnError(##class(%IPM.Storage.FileHash).WalkAndHashDirs(root, .scanDirs, .allFiles, .allHashes)) - - // module.xml is always tracked — add it to allFiles/allHashes explicitly. - if ##class(%File).Exists(root _ "module.xml") { - set allFiles("module.xml") = root _ "module.xml" - set allHashes("module.xml") = $$$lcase(##class(%File).SHA1Hash(root _ "module.xml", 1)) - } - - // Build compilable-only subset for ComputeChanges Pass 1; count total for verbose. - set relPath = "", fileCount = 0 - for { - set relPath = $order(allFiles(relPath), 1, fullPath) - quit:relPath="" - set fileCount = fileCount + 1 - set ext = $$$lcase($piece(relPath, ".", *)) - if ",cls,inc,mac,int," [ (","_ext_",") { - set bfsFiles(relPath) = fullPath - } - } - if verbose { - set dirCount = 0 - set tmpDir = "" - for { set tmpDir = $order(scanDirs(tmpDir)) quit:tmpDir="" set dirCount = dirCount + 1 } - write !, "[", moduleName, "] Scanned ", fileCount, " file(s) across ", dirCount, " director(ies) in ", $fnumber($zhorolog - walkStart, "", 2), "s" - } - - // Step 3: Build reverse index (relPath -> owning resource + processor). - kill reverseIndex, unsupportedResources - do ..SyncBuildReverseIndex(module, orderedResourceList, .reverseIndex, .unsupportedResources, .allFiles) - - // Collect manifest-derived paths from reverseIndex for ComputeChanges. - kill manifestPaths - set riCount = 0 - set riKey = "" - for { - set riKey = $order(reverseIndex(riKey)) - quit:riKey="" - set manifestPaths(riKey) = "" - set riCount = riCount + 1 - } - if verbose { - write !, "[", moduleName, "] Tracking ", riCount, " path(s) across ", orderedResourceList.Count(), " resource(s)" - } - - // Step 4: Compute disk changes vs baseline - $$$ThrowOnError(##class(%IPM.Storage.FileHash).ComputeChanges(module, .modified, .deleted, .manifestPaths, .bfsFiles, .allFiles, .allHashes)) - - // Exclude module.xml from file routing (handled by SyncCheckModuleXml above) - kill modified(moduleXmlRelPath) - kill deleted(moduleXmlRelPath) - - // Falls through when only deletes exist and processDeletes=1 - if '$data(modified) && ('$data(deleted) || 'processDeletes) { - write !, "[", moduleName, "] Nothing to sync." - if '$data(modified) && $data(deleted) && 'processDeletes { - set delCount = 0 - set key = "" - for { set key = $order(deleted(key)) quit:key="" set delCount = delCount + 1 } - write !, " ", delCount, " deleted file(s) detected but not applied. Use -delete to remove from server." - do ##class(%IPM.General.Sync.Summary).AddWarning(moduleName, delCount _ " deleted file(s) detected but not applied; use -delete to remove from server.") - } - if moduleXmlChanged { - do ..SyncCommitModuleXml(module, moduleXmlPath, moduleXmlRelPath) - do ..SyncPrintModuleXmlWarning(moduleName) - } - if verbose && $data(unsupportedResources) { - do ..SyncPrintUnsupportedNote(moduleName, .unsupportedResources) - } - write !, "[", moduleName, "] Sync done in ", $fnumber($zhorolog - syncStart, "", 2), "s" - quit - } - - // Step 5: Partition changes by owning resource - kill syncByResource - do ..SyncRoutePathSet(.modified, .reverseIndex, "modified", .syncByResource) - if processDeletes { - do ..SyncRoutePathSet(.deleted, .reverseIndex, "deleted", .syncByResource) - } - - // Step 6: Dispatch OnSync to each processor; load unhandled compilable files - if verbose { - set resCount = 0 - set resName = "" - for { - set resName = $order(syncByResource(resName)) - quit:resName="" - set resCount = resCount + 1 - } - write !, "[", moduleName, "] Dispatching to ", resCount, " resource(s)" - } - $$$ThrowOnError(..SyncDispatchProcessors(module, root, verbose, .syncByResource, .params, .loadItems)) - - // Step 7: Compile the full resource set with u-flag to pick up dependent recompiles - if loadItems > 0 { - $$$ThrowOnError(..SyncCompile(module, orderedResourceList, verbose, .params)) - } - - // Step 8: Delete server-side documents for removed files, then recompile - if processDeletes && ($data(deleted) > 1) { - do ..SyncApplyDeletes(.deleted, .reverseIndex, .syncByResource, verbose) - $$$ThrowOnError(..SyncCompile(module, orderedResourceList, verbose, .params)) - } - - // Step 9: Commit new hashes on success (skipped on error so next sync re-detects). - // Done before running tests below so a test failure doesn't prevent the file-sync - // outcome from being committed and reported — file sync and test results are independent. - $$$ThrowOnError(##class(%IPM.Storage.FileHash).CommitChanges(module, .modified, .deleted, processDeletes)) - if moduleXmlChanged { - do ..SyncCommitModuleXml(module, moduleXmlPath, moduleXmlRelPath) - } - - set modCount = 0 - set key = "" - for { - set key = $order(modified(key)) - quit:key="" - set modCount = modCount + 1 - write !, " Updated: ", key - do ##class(%IPM.General.Sync.Summary).RecordFile(moduleName, key, "Updated") - } - set delCount = 0 - if processDeletes { - set key = "" - for { - set key = $order(deleted(key)) - quit:key="" - set delCount = delCount + 1 - write !, " Deleted: ", key - do ##class(%IPM.General.Sync.Summary).RecordFile(moduleName, key, "Deleted") - } - } - write !, "[", moduleName, "] Sync complete: ", modCount, " file(s) updated" - if delCount > 0 { - write ", ", delCount, " deleted" - } - write "." - - if moduleXmlChanged { - do ..SyncPrintModuleXmlWarning(moduleName) - } - if verbose && $data(unsupportedResources) { - do ..SyncPrintUnsupportedNote(moduleName, .unsupportedResources) - } - - write !, "[", moduleName, "] Sync done in ", $fnumber($zhorolog - syncStart, "", 2), "s" - - // File sync is committed and reported. Mark success before running tests so a fatal - // test failure below is reported as a test failure, not a sync error, in the summary. - do ##class(%IPM.General.Sync.Summary).MarkSyncOK(moduleName) - - // Step 10: Run changed test-phase tests if -test flag is set (after sync is reported) - if runTests { - $$$ThrowOnError(..SyncRunTests(orderedResourceList, verbose, .params)) - } - - } catch e { - set sc = e.AsStatus() - } - quit sc -} - -/// Check if module.xml changed and reload the manifest if so. -/// Returns 1 if module.xml changed and was reloaded, 0 otherwise. -/// On reload, replaces module with the freshly-opened instance. -ClassMethod SyncCheckModuleXml( - ByRef module As %IPM.Storage.Module, - moduleXmlPath As %String, - moduleXmlRelPath As %String) As %Boolean [ Private ] -{ - if '##class(%File).Exists(moduleXmlPath) { - quit 0 - } - set existing = ##class(%IPM.Storage.FileHash).ModulePathIndexOpen(module.Name, moduleXmlRelPath) - if '$isobject(existing) { - quit 0 - } - set newHash = $$$lcase(##class(%File).SHA1Hash(moduleXmlPath, 1)) - if newHash = existing.Hash { - quit 0 - } - $$$ThrowOnError($system.OBJ.Load(moduleXmlPath, "-d")) - set module = ##class(%IPM.Storage.Module).NameOpen(module.Name, , .openSC) - $$$ThrowOnError(openSC) - quit 1 -} - -/// Build a reverse index: normalizedRelPath -> resource name, Processor, Resource object. -/// Used by SyncRoutePathSet to map changed files back to their owning resource processors. -/// Skips resources whose processor does not support sync — those are collected in -/// unsupportedResources(resourceName)="" for informational display. -/// -/// Step 1 builds docToResource (docName → owner) from ResolveChildren, and prefix-scans -/// allFiles for directory-owned resources (e.g. test dirs) via GetSyncDirectory(). -/// Step 2 maps all stored baseline paths to owners via RelPathToDocName → docToResource. -ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, orderedResourceList As %ListOfObjects, ByRef reverseIndex, Output unsupportedResources, ByRef allFiles) [ Private ] -{ - kill unsupportedResources - // docToResource maps server document names (e.g. "SyncTest.SuperClass.CLS") to the - // resource that owns them. This lets us resolve filesystem paths → owners in O(1) below, - // since RelPathToDocName converts a relPath to a docName deterministically. - kill docToResource - set key = "" - for { - set resource = orderedResourceList.GetNext(.key) - quit:key="" - - if '$isobject(resource.Processor) { - continue - } - - if 'resource.Processor.SupportsSync() { - set unsupportedResources(resource.Name) = "" - continue - } - - // ResolveChildren returns childArr keyed by document name (server-side identifier). - kill childArr - set childSC = resource.ResolveChildren(.childArr) - if $$$ISERR(childSC) { - continue - } - - set childName = "" - for { - set childName = $order(childArr(childName)) - quit:childName="" - set docToResource(childName) = resource.Name - set docToResource(childName, "Processor") = resource.Processor - set docToResource(childName, "Resource") = resource - - // Newly-declared resources have no baseline row yet (StampModule skips uncompiled - // classes), so GetStoredPaths below won't find them. Map their relPath directly. - set relPath = $get(childArr(childName, "RelativePath")) - if relPath = "" { - set relPath = resource.Processor.OnItemRelativePath(childName) - } - if relPath '= "" { - set normalizedRelPath = ##class(%IPM.Storage.FileHash).NormalizePath(relPath) - set reverseIndex(normalizedRelPath) = resource.Name - set reverseIndex(normalizedRelPath, "Processor") = resource.Processor - set reverseIndex(normalizedRelPath, "Resource") = resource - } - } - - // Directory-based resources (e.g. test dirs) own all files under their declared - // directory. Prefix-scan allFiles to map every file under that dir to this resource. - set syncDir = resource.Processor.GetSyncDirectory() - if syncDir '= "" { - set prefix = ##class(%IPM.Storage.FileHash).NormalizePath(syncDir _ "/") - set prefixLen = $length(prefix) - set dirRelPath = prefix - for { - set dirRelPath = $order(allFiles(dirRelPath)) - quit:dirRelPath="" - quit:($extract(dirRelPath, 1, prefixLen) '= prefix) - if '$data(reverseIndex(dirRelPath)) { - set reverseIndex(dirRelPath) = resource.Name - set reverseIndex(dirRelPath, "Processor") = resource.Processor - set reverseIndex(dirRelPath, "Resource") = resource - } - } - } - } - - // Resolve compilable paths: every baseline path not already claimed above gets mapped - // through RelPathToDocName → docToResource. - kill storedPaths - do ##class(%IPM.Storage.FileHash).GetStoredPaths(module.Name, .storedPaths) - set relPath = "" - for { - set relPath = $order(storedPaths(relPath)) - quit:relPath="" - - if $data(reverseIndex(relPath)) { - continue - } - - set docName = ##class(%IPM.Storage.FileHash).RelPathToDocName(relPath) - if docName = "" { - continue - } - if '$data(docToResource(docName)) { - continue - } - - set reverseIndex(relPath) = docToResource(docName) - set reverseIndex(relPath, "Processor") = docToResource(docName, "Processor") - set reverseIndex(relPath, "Resource") = docToResource(docName, "Resource") - } -} - - -/// Route a set of changed paths to their owning resources, categorized by type (modified/deleted). -/// Only sync-supporting resources appear in reverseIndex, so no filtering needed here. -ClassMethod SyncRoutePathSet( - ByRef paths, - ByRef reverseIndex, - category As %String, - ByRef syncByResource) [ Private ] -{ - set relPath = "" - for { - set relPath = $order(paths(relPath)) - quit:relPath="" - - if '$data(reverseIndex(relPath)) { - continue - } - set resName = reverseIndex(relPath) - set syncByResource(resName, category, relPath) = "" - set syncByResource(resName, "Processor") = reverseIndex(relPath, "Processor") - set syncByResource(resName, "Resource") = reverseIndex(relPath, "Resource") - } -} - -/// Call OnSync on each processor, then load any unhandled compilable files. -/// Returns the number of files loaded (used to decide whether SyncCompile is needed). -/// If a processor sets handled=1 in OnSync, it fully owns the sync for that resource -/// (e.g. Test reloads its own way). Otherwise the default path loads+compiles each file. -ClassMethod SyncDispatchProcessors( - module As %IPM.Storage.Module, - root As %String, - verbose As %Boolean, - ByRef syncByResource, - ByRef params, - Output loadItems As %Integer = 0) As %Status [ Private ] -{ - set sc = $$$OK - try { - set resName = "" - for { - set resName = $order(syncByResource(resName)) - quit:resName="" - - if '$data(syncByResource(resName, "Processor")) { - continue - } - set processor = syncByResource(resName, "Processor") - - kill resModified, resDeleted - merge resModified = syncByResource(resName, "modified") - merge resDeleted = syncByResource(resName, "deleted") - - set handled = 0 - $$$ThrowOnError(processor.OnSync(.resModified, .resDeleted, .params, .handled)) - - if handled || 'processor.%IsA("%IPM.ResourceProcessor.AbstractCompilable") { - continue - } - - // Default: load each changed file; SyncCompile will do a full compile with u-flag - set loadRelPath = "" - for { - set loadRelPath = $order(resModified(loadRelPath)) - quit:loadRelPath="" - - set fullPath = ##class(%File).NormalizeFilename(loadRelPath, root) - if ##class(%File).Exists(fullPath) { - set loadFlags = $select(verbose:"d", 1:"-d") - $$$ThrowOnError($system.OBJ.Load(fullPath, loadFlags _ "c")) - set loadItems = loadItems + 1 - } - } - } - } catch e { - set sc = e.AsStatus() - } - quit sc -} - -/// Delete server-side documents for compilable deleted files. -/// Guards skip paths with no owner, no processor, or non-compilable processors — those are -/// handled elsewhere (unsupported warnings) or aren't server-side documents at all. -ClassMethod SyncApplyDeletes( - ByRef deleted, - ByRef reverseIndex, - ByRef syncByResource, - verbose As %Boolean) As %Status [ Private ] -{ - set sc = $$$OK - set relPath = "" - for { - set relPath = $order(deleted(relPath)) - quit:relPath="" - - if '$data(reverseIndex(relPath)) { - continue - } - set resName = reverseIndex(relPath) - if '$data(syncByResource(resName, "Processor")) { - continue - } - set processor = syncByResource(resName, "Processor") - if 'processor.%IsA("%IPM.ResourceProcessor.AbstractCompilable") { - continue - } - set docName = ##class(%IPM.Storage.FileHash).RelPathToDocName(relPath) - if docName '= "" { - set delFlags = $select(verbose:"d", 1:"-d") - set delSC = $system.OBJ.Delete(docName, delFlags) - // Delete failures are non-fatal: the SyncCompile pass that immediately follows - // will fail to compile any class that still references the deleted doc, surfacing - // the error with full context. Aborting the delete loop here would leave other - // deletions unapplied and make the overall error harder to diagnose. - if $$$ISERR(delSC) { - write !, "Warning: could not delete ", docName, ": ", $system.Status.GetOneErrorText(delSC) - set sc = $$$ADDSC(sc, delSC) - } - } - } - quit sc -} - -/// Run test-phase tests for changed test case classes recorded in params("Sync","ChangedTestCases"). -/// Groups changed classes by owning resource first, then dispatches one batched RunTest call -/// per resource (via OnSyncRunTests) instead of one call per class — avoids N separate -/// reload/compile/run cycles when several test classes in the same resource changed together. -ClassMethod SyncRunTests( - orderedResourceList As %ListOfObjects, - verbose As %Boolean, - ByRef params) As %Status [ Private ] -{ - set sc = $$$OK - try { - // Step 1: group changed classes by owning resource: byResource(resourceName, className) = "" - kill byResource - set className = "" - for { - set className = $order(params("Sync", "ChangedTestCases", className), 1, owningResource) - quit:className="" - set byResource(owningResource, className) = "" - } - - // Step 2: one OnSyncRunTests call per resource - set testKey = "" - for { - set testResource = orderedResourceList.GetNext(.testKey) - quit:testKey="" - - if '$isobject(testResource.Processor) { - continue - } - if 'testResource.Processor.%IsA("%IPM.ResourceProcessor.Test") { - continue - } - if '$data(byResource(testResource.Name)) { - continue - } - if '$listfind(testResource.Processor.Phase, "test") { - set skipClassName = "" - for { - set skipClassName = $order(byResource(testResource.Name, skipClassName)) - quit:skipClassName="" - write:verbose !, "Skipping verify-scoped test: ", skipClassName, " (use 'verify' to run)" - do ##class(%IPM.General.Sync.Summary).AddSkipped(testResource.Module.Name, skipClassName) - } - continue - } - kill classInfo - merge classInfo = byResource(testResource.Name) - kill testParams - merge testParams = params - set testParams("DeveloperMode") = 1 - $$$ThrowOnError(testResource.Processor.OnSyncRunTests(.classInfo, .testParams)) - } - } catch e { - set sc = e.AsStatus() - } - quit sc -} - -/// Commit the current module.xml hash to the baseline. -ClassMethod SyncCommitModuleXml( - module As %IPM.Storage.Module, - moduleXmlPath As %String, - moduleXmlRelPath As %String) [ Private ] -{ - kill moduleXmlMod, emptyDel - set moduleXmlMod(moduleXmlRelPath) = $$$lcase(##class(%File).SHA1Hash(moduleXmlPath, 1)) - set commitSC = ##class(%IPM.Storage.FileHash).CommitChanges(module, .moduleXmlMod, .emptyDel, 0) - if $$$ISERR(commitSC) { - write !, "Warning: failed to record module.xml hash: ", $system.Status.GetOneErrorText(commitSC) - } -} - -ClassMethod SyncPrintModuleXmlWarning(moduleName As %String) [ Private ] -{ - write ! - write !, "Warning: module.xml changed and was reloaded." - write !, " Resources may have been added/removed. Run `reload ", moduleName, "` to fully apply" - write !, " manifest-level changes (mappings, dependencies, defaults)." - do ##class(%IPM.General.Sync.Summary).AddWarning(moduleName, "module.xml changed and was reloaded; run `reload " _ moduleName _ "` to fully apply.") -} - -ClassMethod SyncPrintUnsupportedNote(moduleName As %String, ByRef unsupportedResources) [ Private ] -{ - set count = 0 - set names = "" - set resName = "" - for { - set resName = $order(unsupportedResources(resName)) - quit:resName="" - set count = count + 1 - if count <= 3 { - set names = names _ $select(names="":"", 1:", ") _ resName - } - } - if count > 3 { - set names = names _ ", ... (" _ (count - 3) _ " more)" - } - write !, "[", moduleName, "] ", count, " resource(s) skipped (no sync support): ", names - write !, " Use `reload ", moduleName, "` to apply changes to those resources." - do ##class(%IPM.General.Sync.Summary).AddWarning(moduleName, count _ " resource(s) skipped (no sync support): " _ names _ ".") -} - -/// Recompile all compilable resources in the module to catch dependents invalidated by -/// changes loaded in SyncDispatchProcessors. Skips CompileFromProject resources — those are -/// deployed code loaded via Studio project files (.prj); compiling them individually would -/// fail because deployed classes have no source in the routine database. -ClassMethod SyncCompile(module As %IPM.Storage.Module, orderedResourceList As %ListOfObjects, verbose As %Boolean = 0, ByRef params) As %Status [ Private ] -{ - set sc = $$$OK - try { - kill compileArray - - set key = "" - for { - set resource = orderedResourceList.GetNext(.key) - quit:key="" - - if '$isobject(resource.Processor) { - continue - } - if 'resource.Processor.%IsA("%IPM.ResourceProcessor.AbstractCompilable") { - continue - } - if 'resource.IsInScope("Compile") { - continue - } - if resource.Processor.CompileFromProject { - continue - } - - kill oneResourceList - set oneResourceList(resource.Name) = "" - set resSC = resource.ResolveChildren(.oneResourceList) - if $$$ISERR(resSC) { - continue - } - - set childKey = "" - for { - set childKey = $order(oneResourceList(childKey)) - quit:childKey="" - set ext = $zconvert($piece(childKey, ".", *), "U") - if ext = "CLS" { - set className = $piece(childKey, ".", 1, *-1) - if '$$$comClassDefined(className) { - continue - } - } - set compileArray(childKey) = "" - } - } - - if $data(compileArray) > 1 { - set flags = $select(verbose:"d", 1:"-d") _ "cku" - set sc = ##class(%IPM.Utils.LegacyCompat).UpdateSuperclassAndCompile(.compileArray, .flags) - } - } catch e { - set sc = e.AsStatus() - } - quit sc + return ##class(%IPM.General.Sync.Pipeline).Run(..Module, .params) } Method %Initialize(ByRef pParams) As %Status diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls index 74f0ee941..2806507dc 100644 --- a/src/cls/IPM/ResourceProcessor/Test.cls +++ b/src/cls/IPM/ResourceProcessor/Test.cls @@ -408,25 +408,13 @@ Method OnSync(ByRef modifiedPaths, ByRef deletedPaths, ByRef params, Output hand // Record changed TestCase subclasses for SyncRunTests. // relPath is relative to module root (e.g. "tests/unit/SyncTest/Tests/Trivial.cls"). - // Strip the resource directory prefix (e.g. "tests/unit/") to get the package-relative path. set resourceDir = ##class(%IPM.Storage.FileHash).NormalizePath(..ResourceReference.Name) set relPath = "" for { set relPath = $order(modifiedPaths(relPath)) quit:relPath="" - - // Strip resource directory prefix to get path relative to the test dir - set packageRelPath = relPath - if $extract(packageRelPath, 1, $length(resourceDir)) = resourceDir { - set packageRelPath = $extract(packageRelPath, $length(resourceDir) + 1, *) - } - // Convert path to class name: "SyncTest/Tests/Trivial.cls" -> "SyncTest.Tests.Trivial" - set fileName = $piece(packageRelPath, "/", *) - set baseName = $piece(fileName, ".", 1, *-1) - set dirPart = $piece(packageRelPath, "/", 1, *-1) - set className = $select(dirPart '= "": $translate(dirPart, "/", ".") _ "." _ baseName, 1: baseName) - if $zconvert($piece(fileName, ".", *), "U") = "CLS" - && $$$comClassDefined(className) + set className = ..RelPathToClassName(relPath, resourceDir) + if className '= "" && $$$comClassDefined(className) && $classmethod(className, "%Extends", "%UnitTest.TestCase") { set params("Sync", "ChangedTestCases", className) = ..ResourceReference.Name } @@ -438,16 +426,8 @@ Method OnSync(ByRef modifiedPaths, ByRef deletedPaths, ByRef params, Output hand for { set relPath = $order(deletedPaths(relPath)) quit:relPath="" - - set packageRelPath = relPath - if $extract(packageRelPath, 1, $length(resourceDir)) = resourceDir { - set packageRelPath = $extract(packageRelPath, $length(resourceDir) + 1, *) - } - set fileName = $piece(packageRelPath, "/", *) - set baseName = $piece(fileName, ".", 1, *-1) - set dirPart = $piece(packageRelPath, "/", 1, *-1) - set className = $select(dirPart '= "": $translate(dirPart, "/", ".") _ "." _ baseName, 1: baseName) - if $zconvert($piece(fileName, ".", *), "U") = "CLS" && $$$comClassDefined(className) { + set className = ..RelPathToClassName(relPath, resourceDir) + if className '= "" && $$$comClassDefined(className) { $$$ThrowOnError($system.OBJ.Delete(className, $select(verbose:"d",1:"-d"))) } } @@ -577,6 +557,28 @@ ClassMethod SyncSummaryTallyTests( do ##class(%IPM.General.Sync.Summary).AddTests(moduleName, passed, failed) } +/// Convert a module-root-relative relPath to a class name within this test resource. +/// relPath e.g. "tests/unit/SyncTest/Tests/Trivial.cls", resourceDir e.g. "tests/unit". +/// Returns "" for non-.cls files or paths outside resourceDir. +ClassMethod RelPathToClassName(relPath As %String, resourceDir As %String) As %String [ Private ] +{ + set packageRelPath = relPath + if $extract(packageRelPath, 1, $length(resourceDir)) = resourceDir { + set packageRelPath = $extract(packageRelPath, $length(resourceDir) + 1, *) + } + // Strip leading slash left after prefix removal (e.g. "/SyncTest/...") + if $extract(packageRelPath) = "/" { + set packageRelPath = $extract(packageRelPath, 2, *) + } + set fileName = $piece(packageRelPath, "/", *) + if $zconvert($piece(fileName, ".", *), "U") '= "CLS" { + quit "" + } + set baseName = $piece(fileName, ".", 1, *-1) + set dirPart = $piece(packageRelPath, "/", 1, *-1) + quit $select(dirPart '= "": $translate(dirPart, "/", ".") _ "." _ baseName, 1: baseName) +} + Method %OnValidateObject() As %Status [ Private, ServerOnly = 1 ] { if ((..Package = "") && (..Class = "")) || ((..Package '= "") && (..Class '= "")) { From 4326b46519cc66dd29d49724be27d0ec54cfed72 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Wed, 29 Jul 2026 14:21:39 -0400 Subject: [PATCH 36/39] Fix bad rebase --- .github/workflows/main.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b80b8a8e7..780f407ad 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -89,6 +89,10 @@ jobs: registry-image \ -a "iris session iris -U%SYS '##class(Security.Users).UnExpireUserPasswords(\"*\")'") sleep 5; docker exec $REGISTRY /usr/irissys/dev/Cloud/ICM/waitReady.sh + docker exec -i $REGISTRY iris session iris -UUSER << EOF + zpm "install zpm-registry" + halt + EOF docker logs $REGISTRY - name: Run ORAS registry timeout-minutes: 5 @@ -135,6 +139,10 @@ jobs: -a "iris session iris -U%SYS '##class(Security.Users).UnExpireUserPasswords(\"*\")'" REGISTRY=`docker ps -lq` sleep 5; docker exec $REGISTRY /usr/irissys/dev/Cloud/ICM/waitReady.sh + docker exec -i $REGISTRY iris session iris -UUSER << EOF + zpm "install zpm-registry" + halt + EOF docker logs $REGISTRY - name: Test and publish to temporary registry timeout-minutes: 15 From 7a97be253df3001d2f20099671534beb3a6136ff Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Wed, 29 Jul 2026 15:02:17 -0400 Subject: [PATCH 37/39] Refactor and minor fix --- src/cls/IPM/General/Sync/Summary.cls | 16 +++--- src/cls/IPM/Main.cls | 83 +--------------------------- src/cls/IPM/Utils/Module.cls | 82 +++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 90 deletions(-) diff --git a/src/cls/IPM/General/Sync/Summary.cls b/src/cls/IPM/General/Sync/Summary.cls index db19dbce1..54d179d1b 100644 --- a/src/cls/IPM/General/Sync/Summary.cls +++ b/src/cls/IPM/General/Sync/Summary.cls @@ -168,7 +168,7 @@ ClassMethod Report( set line = line _ " " _ fileCount _ " file" _ $select(fileCount = 1:"", 1:"s") } if hasTests { - // "passed/total" already conveys the failure count; no redundant "N failed". + // Report how many tests passed/ran set line = line _ " tests " _ (+passed) _ "/" _ ((+passed) + (+failed)) } if isError { @@ -176,21 +176,21 @@ ClassMethod Report( } write !, line // Enumerate verify-scoped test classes that were detected but not run. - set sseq = "" + set skippedSeq = "" for { - set sseq = $order(^||IPM.Sync.Summary("Module", name, "Skipped", sseq), 1, skippedClass) - quit:sseq="" + set skippedSeq = $order(^||IPM.Sync.Summary("Module", name, "Skipped", skippedSeq), 1, skippedClass) + quit:skippedSeq="" write !, " skipped (verify-scoped): ", skippedClass } } // Collect this module's warnings for the aggregated section below. - set wseq = "" + set warningSeq = "" for { - set wseq = $order(^||IPM.Sync.Summary("Module", name, "Warning", wseq), 1, wtext) - quit:wseq="" + set warningSeq = $order(^||IPM.Sync.Summary("Module", name, "Warning", warningSeq), 1, warningText) + quit:warningSeq="" set warningCount = warningCount + 1 - set warnings(warningCount) = " [" _ name _ "] " _ wtext + set warnings(warningCount) = " [" _ name _ "] " _ warningText } } diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls index 104af88a0..480a4182a 100644 --- a/src/cls/IPM/Main.cls +++ b/src/cls/IPM/Main.cls @@ -2335,7 +2335,7 @@ ClassMethod Sync(ByRef commandInfo) [ Private ] } else { // Sync all dev-mode modules in dependency order (least dependent first) so each module // recompiles against dependencies that have already been synced this run. - set orderedNames = ..GetDevModeModulesInDependencyOrder() + set orderedNames = ##class(%IPM.Utils.Module).GetDevModeModulesInDependencyOrder() set found = $listlength(orderedNames) set failures = "" if 'found { @@ -2364,87 +2364,6 @@ ClassMethod Sync(ByRef commandInfo) [ Private ] } } -/// Returns a $list of dev-mode module names ordered least-dependent-first: a module's -/// dependencies appear before it. Sync-all uses this so each module recompiles against -/// dependencies already synced in the same run. Only dependency edges among the dev-mode set -/// are considered; modules unconnected by any such edge keep their natural (row) order. -ClassMethod GetDevModeModulesInDependencyOrder() As %List [ Private ] -{ - // Dev-mode modules in row order (stable tiebreaker for modules with no ordering constraint). - set result = ##class(%SQL.Statement).%ExecDirect(, - "select Name from %IPM_Storage.ModuleItem where DeveloperMode = 1") - if (result.%SQLCODE < 0) { - throw ##class(%Exception.SQL).CreateFromSQLCODE(result.%SQLCODE, result.%Message) - } - kill devMode - set rowOrder = "" - while result.%Next(.sc) { - $$$ThrowOnError(sc) - set name = result.%Get("Name") - set devMode(name) = "" - set rowOrder = rowOrder _ $listbuild(name) - } - $$$ThrowOnError(sc) - - // Dependency edges restricted to the dev-mode set: dependsOn(modName, depName) = "". - set depRes = ##class(%SQL.Statement).%ExecDirect(, - "select ModuleItem->Name ModName, Dependencies_Name DepName from %IPM_Storage.ModuleItem_Dependencies") - if (depRes.%SQLCODE < 0) { - throw ##class(%Exception.SQL).CreateFromSQLCODE(depRes.%SQLCODE, depRes.%Message) - } - kill dependsOn - while depRes.%Next(.sc) { - $$$ThrowOnError(sc) - set modName = depRes.%Get("ModName") - set depName = depRes.%Get("DepName") - if $data(devMode(modName)) && $data(devMode(depName)) { - set dependsOn(modName, depName) = "" - } - } - $$$ThrowOnError(sc) - - // Emit a module only once all its (dev-mode) dependencies are already emitted; repeat until - // every module is placed. Each pass scans in row order so independent modules keep their - // natural sequence. - set ordered = "" - kill emitted - set remaining = $listlength(rowOrder) - while remaining > 0 { - set progress = 0 - set ptr = 0 - while $listnext(rowOrder, ptr, name) { - continue:$data(emitted(name)) - set ready = 1 - set depName = "" - for { - set depName = $order(dependsOn(name, depName)) - quit:depName="" - if '$data(emitted(depName)) { - set ready = 0 - quit - } - } - if ready { - set ordered = ordered _ $listbuild(name) - set emitted(name) = "" - set remaining = remaining - 1 - set progress = 1 - } - } - if 'progress { - // A dependency cycle within the dev-mode set would stall the loop. Install rejects - // real cycles, so this is a safety net: emit the rest in row order and stop. - set ptr = 0 - while $listnext(rowOrder, ptr, name) { - continue:$data(emitted(name)) - set ordered = ordered _ $listbuild(name) - set emitted(name) = "" - } - set remaining = 0 - } - } - quit ordered -} ClassMethod Load( ByRef pCommandInfo, diff --git a/src/cls/IPM/Utils/Module.cls b/src/cls/IPM/Utils/Module.cls index 19b83a087..3c0a8766c 100644 --- a/src/cls/IPM/Utils/Module.cls +++ b/src/cls/IPM/Utils/Module.cls @@ -1448,6 +1448,88 @@ ClassMethod GetFlatDependencyListFromInvertedDependencyGraph(ByRef pInvertedDepe return orderedDependencyList } +/// Returns a $list of dev-mode module names ordered least-dependent-first: a module's +/// dependencies appear before it. Only dependency edges among the dev-mode set are +/// considered; unconnected modules keep their natural row order. Used by sync-all so each +/// module recompiles against dependencies already synced in the same run. +ClassMethod GetDevModeModulesInDependencyOrder() As %List +{ + // Dev-mode modules in row order (stable tiebreaker for modules with no ordering constraint). + set result = ##class(%SQL.Statement).%ExecDirect(, + "select Name from %IPM_Storage.ModuleItem where DeveloperMode = 1") + if (result.%SQLCODE < 0) { + throw ##class(%Exception.SQL).CreateFromSQLCODE(result.%SQLCODE, result.%Message) + } + kill devMode + set rowOrder = "" + while result.%Next(.sc) { + $$$ThrowOnError(sc) + set name = result.%Get("Name") + set devMode(name) = "" + set rowOrder = rowOrder _ $listbuild(name) + } + $$$ThrowOnError(sc) + + // Dependency edges restricted to the dev-mode set: dependsOn(modName, depName) = "". + set depRes = ##class(%SQL.Statement).%ExecDirect(, + "select ModuleItem->Name ModName, Dependencies_Name DepName from %IPM_Storage.ModuleItem_Dependencies") + if (depRes.%SQLCODE < 0) { + throw ##class(%Exception.SQL).CreateFromSQLCODE(depRes.%SQLCODE, depRes.%Message) + } + kill dependsOn + while depRes.%Next(.sc) { + $$$ThrowOnError(sc) + set modName = depRes.%Get("ModName") + set depName = depRes.%Get("DepName") + if $data(devMode(modName)) && $data(devMode(depName)) { + set dependsOn(modName, depName) = "" + } + } + $$$ThrowOnError(sc) + + // Emit a module only once all its (dev-mode) dependencies are already emitted; repeat until + // every module is placed. Each pass scans in row order so independent modules keep their + // natural sequence. + set ordered = "" + kill emitted + set remaining = $listlength(rowOrder) + while remaining > 0 { + set progress = 0 + set ptr = 0 + while $listnext(rowOrder, ptr, name) { + continue:$data(emitted(name)) + set ready = 1 + set depName = "" + for { + set depName = $order(dependsOn(name, depName)) + quit:depName="" + if '$data(emitted(depName)) { + set ready = 0 + quit + } + } + if ready { + set ordered = ordered _ $listbuild(name) + set emitted(name) = "" + set remaining = remaining - 1 + set progress = 1 + } + } + if 'progress { + // A dependency cycle within the dev-mode set would stall the loop. Install rejects + // real cycles, so this is a safety net: emit the rest in row order and stop. + set ptr = 0 + while $listnext(rowOrder, ptr, name) { + continue:$data(emitted(name)) + set ordered = ordered _ $listbuild(name) + set emitted(name) = "" + } + set remaining = 0 + } + } + quit ordered +} + ClassMethod CheckLicenseKey() As %Boolean { // Limit the number of attempts in case this somehow ends up running in a non-interactive process. From 072b056806dc4f0c12d0d84cac79c556077dee10 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Wed, 29 Jul 2026 15:48:12 -0400 Subject: [PATCH 38/39] Improve comments --- src/cls/IPM/General/Sync/Pipeline.cls | 20 +++++--------------- src/cls/IPM/ResourceProcessor/Test.cls | 4 +--- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/src/cls/IPM/General/Sync/Pipeline.cls b/src/cls/IPM/General/Sync/Pipeline.cls index f9bc60af9..9b00da99f 100644 --- a/src/cls/IPM/General/Sync/Pipeline.cls +++ b/src/cls/IPM/General/Sync/Pipeline.cls @@ -173,7 +173,7 @@ ClassMethod Run( } $$$ThrowOnError(..SyncDispatchProcessors(module, root, verbose, .syncByResource, .params, .loadItems)) - // Step 7: Compile the full resource set with u-flag to pick up dependent recompiles + // Step 7: Compile the full resource set to pick up dependent recompiles if loadItems > 0 { $$$ThrowOnError(..SyncCompile(module, orderedResourceList, verbose, .params)) } @@ -271,16 +271,11 @@ ClassMethod SyncCheckModuleXml( /// Used by SyncRoutePathSet to map changed files back to their owning resource processors. /// Skips resources whose processor does not support sync — those are collected in /// unsupportedResources(resourceName)="" for informational display. -/// -/// Step 1 builds docToResource (docName → owner) from ResolveChildren, and prefix-scans -/// allFiles for directory-owned resources (e.g. test dirs) via GetSyncDirectory(). -/// Step 2 maps all stored baseline paths to owners via RelPathToDocName → docToResource. ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, orderedResourceList As %ListOfObjects, ByRef reverseIndex, Output unsupportedResources, ByRef allFiles) [ Private ] { kill unsupportedResources // docToResource maps server document names (e.g. "SyncTest.SuperClass.CLS") to the - // resource that owns them. This lets us resolve filesystem paths → owners in O(1) below, - // since RelPathToDocName converts a relPath to a docName deterministically. + // resource that owns them. This lets us map relPath → owner in O(1) below. kill docToResource set key = "" for { @@ -345,8 +340,7 @@ ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, orderedResource } } - // Resolve compilable paths: every baseline path not already claimed above gets mapped - // through RelPathToDocName → docToResource. + // Catch compilable files in the baseline not reachable via ResolveChildren. kill storedPaths do ##class(%IPM.Storage.FileHash).GetStoredPaths(module.Name, .storedPaths) set relPath = "" @@ -481,13 +475,9 @@ ClassMethod SyncApplyDeletes( if docName '= "" { set delFlags = $select(verbose:"d", 1:"-d") set delSC = $system.OBJ.Delete(docName, delFlags) - // Delete failures are non-fatal: the SyncCompile pass that immediately follows - // will fail to compile any class that still references the deleted doc, surfacing - // the error with full context. Aborting the delete loop here would leave other - // deletions unapplied and make the overall error harder to diagnose. // Non-fatal: the SyncCompile pass that follows will surface any compilation errors - // caused by the still-present document, giving the user full context. Aborting here - // would leave remaining deletions unapplied and make the overall error harder to diagnose. + // caused by the still-present document. Aborting here would leave remaining deletions + // unapplied and make the overall error harder to diagnose. if $$$ISERR(delSC) { set errText = $system.Status.GetOneErrorText(delSC) write !, "Warning: could not delete ", docName, ": ", errText diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls index 2806507dc..e8f6d0523 100644 --- a/src/cls/IPM/ResourceProcessor/Test.cls +++ b/src/cls/IPM/ResourceProcessor/Test.cls @@ -397,8 +397,6 @@ Method OnSync(ByRef modifiedPaths, ByRef deletedPaths, ByRef params, Output hand // Reload all test files in the directory, then compile so %Extends checks are valid below. // New test classes may not yet be in ^oddDEF, so we can't limit to changed files here. - // Test resources are not AbstractCompilable, so SyncCompile never touches them — - // OnSync owns the full load+compile cycle for this resource type. $$$ThrowOnError(##class(%IPM.Test.Manager).LoadTestDirectory(unitTestDir, verbose, .loadedList, ..Format)) if ..Package '= "" { $$$ThrowOnError($system.OBJ.CompilePackage(..Package, "ck"_$select(verbose:"d",1:"-d"))) @@ -558,7 +556,7 @@ ClassMethod SyncSummaryTallyTests( } /// Convert a module-root-relative relPath to a class name within this test resource. -/// relPath e.g. "tests/unit/SyncTest/Tests/Trivial.cls", resourceDir e.g. "tests/unit". +/// e.g. "tests/unit/SyncTest/Tests/Trivial.cls" → "SyncTest.Tests.Trivial". /// Returns "" for non-.cls files or paths outside resourceDir. ClassMethod RelPathToClassName(relPath As %String, resourceDir As %String) As %String [ Private ] { From 53228669fdf5116c38960d6ef74bfe2ab62103d1 Mon Sep 17 00:00:00 2001 From: isc-dchui Date: Thu, 30 Jul 2026 16:26:47 -0400 Subject: [PATCH 39/39] Remove redundant code and reorganize --- src/cls/IPM/General/Sync/Output.cls | 79 +++++++++++++ src/cls/IPM/General/Sync/Pipeline.cls | 156 +++++-------------------- src/cls/IPM/General/Sync/Summary.cls | 6 +- src/cls/IPM/ResourceProcessor/Test.cls | 8 +- src/cls/IPM/Storage/FileHash.cls | 79 +++++++------ src/cls/IPM/Utils/Module.cls | 2 +- 6 files changed, 161 insertions(+), 169 deletions(-) create mode 100644 src/cls/IPM/General/Sync/Output.cls diff --git a/src/cls/IPM/General/Sync/Output.cls b/src/cls/IPM/General/Sync/Output.cls new file mode 100644 index 000000000..1ba8a465e --- /dev/null +++ b/src/cls/IPM/General/Sync/Output.cls @@ -0,0 +1,79 @@ +/// Sync output that always writes to the current device, regardless of whether a whole-namespace +/// sync (`sync` with no module named) is in progress. Contrast %IPM.General.Sync.Summary, whose +/// recorders are gated on such a run being active. +Class %IPM.General.Sync.Output [ Abstract ] +{ + +/// Write a module-scoped status line: "[moduleName] text". Single place for the bracketed +/// per-module prefix so the many status writes in the pipeline stay consistent. +ClassMethod Log(moduleName As %String, text As %String) [ Internal ] +{ + write !, "[", moduleName, "] ", text +} + +/// Count the top-level subscripts of an array. Used for the verbose "N file(s)"/"N path(s)" +/// status lines, which report only top-level keys and ignore any child nodes. +ClassMethod CountNodes(ByRef array) As %Integer [ Internal ] +{ + set count = 0 + set key = "" + for { + set key = $order(array(key)) + quit:key="" + set count = count + 1 + } + quit count +} + +/// Shared closing output for both pipeline exit paths (nothing-to-sync and full sync): the +/// module.xml reload warning, the unsupported-resource note, and the total-elapsed line. Kept in +/// one place so the two branches can't drift apart. +ClassMethod PrintTail( + moduleName As %String, + verbose As %Boolean, + moduleXmlChanged As %Boolean, + syncStart As %Numeric, + ByRef unsupportedResources) [ Internal ] +{ + if moduleXmlChanged { + do ..PrintModuleXmlWarning(moduleName) + } + if $data(unsupportedResources) { + do ..PrintUnsupportedNote(moduleName, verbose, .unsupportedResources) + } + do ..Log(moduleName, "Sync done in "_$fnumber($zhorolog - syncStart, "", 2)_"s") +} + +ClassMethod PrintModuleXmlWarning(moduleName As %String) [ Internal ] +{ + write ! + write !, "Warning: module.xml changed and was reloaded." + write !, " Resources may have been added/removed. Run `reload ", moduleName, "` to fully apply" + write !, " manifest-level changes (mappings, dependencies, defaults)." + do ##class(%IPM.General.Sync.Summary).AddWarning(moduleName, "module.xml changed and was reloaded; run `reload " _ moduleName _ "` to fully apply.") +} + +ClassMethod PrintUnsupportedNote(moduleName As %String, verbose As %Boolean, ByRef unsupportedResources) [ Internal ] +{ + set count = 0 + set names = "" + set resName = "" + for { + set resName = $order(unsupportedResources(resName)) + quit:resName="" + set count = count + 1 + if count <= 3 { + set names = names _ $select(names="":"", 1:", ") _ resName + } + } + if count > 3 { + set names = names _ ", ... (" _ (count - 3) _ " more)" + } + if verbose { + do ..Log(moduleName, count_" resource(s) skipped (no sync support): "_names) + write !, " Use `reload ", moduleName, "` to apply changes to those resources." + } + do ##class(%IPM.General.Sync.Summary).AddWarning(moduleName, count _ " resource(s) skipped (no sync support): " _ names _ ".") +} + +} diff --git a/src/cls/IPM/General/Sync/Pipeline.cls b/src/cls/IPM/General/Sync/Pipeline.cls index 9b00da99f..6dc5bf698 100644 --- a/src/cls/IPM/General/Sync/Pipeline.cls +++ b/src/cls/IPM/General/Sync/Pipeline.cls @@ -45,111 +45,61 @@ ClassMethod Run( if '##class(%IPM.Storage.FileHash).HasBaseline(moduleName) { // Self-heal: establish baseline for modules loaded before this feature $$$ThrowOnError(##class(%IPM.Storage.FileHash).StampModule(module)) - write !, "[", moduleName, "] Baseline established. Run sync again to detect changes." + do ##class(%IPM.General.Sync.Output).Log(moduleName, "Baseline established. Run sync again to detect changes.") quit } - // No StampModule call here after a manifest reload: StampModule would overwrite current - // hashes for ALL files (including ones the user just edited), causing ComputeChanges to - // see current-vs-current and report zero changes for co-edited files. - // Newly-declared resources are handled without a full stamp: SyncBuildReverseIndex - // (step 3) calls ResolveChildren, which adds their derived relPaths to reverseIndex → - // manifestPaths. ComputeChanges Pass 2 finds those paths with no baseline row and - // reports them as modified, so they are loaded in this same sync call. + // Deliberately no StampModule here after a manifest reload: it would overwrite every + // file's hash (including files the user just edited), so ComputeChanges would see + // current-vs-current and report no changes. Newly-declared resources still get picked up: + // SyncBuildReverseIndex adds their relPaths to reverseIndex, and ComputeChanges Pass 2 + // reports any with no baseline row as modified. // Step 2: Collect scan directories from resource processors, then walk only those. // Each processor declares its owned directory via GetSyncDirectory(); sync never // touches files outside declared locations. set orderedResourceList = module.GetOrderedResourceList() - kill scanDirs - set rlKey = "" - for { - set rlResource = orderedResourceList.GetNext(.rlKey) - quit:rlKey="" - if '$isobject(rlResource.Processor) { - continue - } - if 'rlResource.Processor.SupportsSync() { - continue - } - set syncDir = rlResource.Processor.GetSyncDirectory() - if syncDir '= "" { - set scanDirs(syncDir) = "" - } - } - do ##class(%IPM.Storage.FileHash).DeduplicateScanDirs(.scanDirs) + do ##class(%IPM.Storage.FileHash).CollectScanDirs(module, .scanDirs) set walkStart = $zhorolog - kill allFiles, allHashes, bfsFiles + kill allFiles, allHashes $$$ThrowOnError(##class(%IPM.Storage.FileHash).WalkAndHashDirs(root, .scanDirs, .allFiles, .allHashes)) - // module.xml is always tracked — add it to allFiles/allHashes explicitly. - if ##class(%File).Exists(root _ "module.xml") { - set allFiles("module.xml") = root _ "module.xml" - set allHashes("module.xml") = $$$lcase(##class(%File).SHA1Hash(root _ "module.xml", 1)) - } - - // Build compilable-only subset for ComputeChanges Pass 1; count total for verbose. - set relPath = "", fileCount = 0 - for { - set relPath = $order(allFiles(relPath), 1, fullPath) - quit:relPath="" - set fileCount = fileCount + 1 - set ext = $$$lcase($piece(relPath, ".", *)) - if ",cls,inc,mac,int," [ (","_ext_",") { - set bfsFiles(relPath) = fullPath - } - } if verbose { - set dirCount = 0 - set tmpDir = "" - for { set tmpDir = $order(scanDirs(tmpDir)) quit:tmpDir="" set dirCount = dirCount + 1 } - write !, "[", moduleName, "] Scanned ", fileCount, " file(s) across ", dirCount, " director(ies) in ", $fnumber($zhorolog - walkStart, "", 2), "s" + set fileCount = ##class(%IPM.General.Sync.Output).CountNodes(.allFiles) + set dirCount = ##class(%IPM.General.Sync.Output).CountNodes(.scanDirs) + do ##class(%IPM.General.Sync.Output).Log(moduleName, "Scanned "_fileCount_" file(s) across "_dirCount_" director(ies) in "_$fnumber($zhorolog - walkStart, "", 2)_"s") } // Step 3: Build reverse index (relPath -> owning resource + processor). kill reverseIndex, unsupportedResources do ..SyncBuildReverseIndex(module, orderedResourceList, .reverseIndex, .unsupportedResources, .allFiles) - // Collect manifest-derived paths from reverseIndex for ComputeChanges. - kill manifestPaths - set riCount = 0 - set riKey = "" - for { - set riKey = $order(reverseIndex(riKey)) - quit:riKey="" - set manifestPaths(riKey) = "" - set riCount = riCount + 1 - } if verbose { - write !, "[", moduleName, "] Tracking ", riCount, " path(s) across ", orderedResourceList.Count(), " resource(s)" + // Top-level keys of reverseIndex are the tracked relPaths (subnodes are children). + set riCount = ##class(%IPM.General.Sync.Output).CountNodes(.reverseIndex) + do ##class(%IPM.General.Sync.Output).Log(moduleName, "Tracking "_riCount_" path(s) across "_orderedResourceList.Count()_" resource(s)") } - // Step 4: Compute disk changes vs baseline - $$$ThrowOnError(##class(%IPM.Storage.FileHash).ComputeChanges(module, .modified, .deleted, .manifestPaths, .bfsFiles, .allFiles, .allHashes)) - - // Exclude module.xml from file routing (handled by SyncCheckModuleXml above) - kill modified(moduleXmlRelPath) - kill deleted(moduleXmlRelPath) + // Step 4: Compute disk changes vs baseline. module.xml is owned by SyncCheckModuleXml + // above and is deliberately excluded by ComputeChanges, so it never appears in + // modified/deleted here. + $$$ThrowOnError(##class(%IPM.Storage.FileHash).ComputeChanges(module, .modified, .deleted, .reverseIndex, .allFiles, .allHashes)) // Falls through when only deletes exist and processDeletes=1 if '$data(modified) && ('$data(deleted) || 'processDeletes) { - write !, "[", moduleName, "] Nothing to sync." + do ##class(%IPM.General.Sync.Output).Log(moduleName, "Nothing to sync.") if '$data(modified) && $data(deleted) && 'processDeletes { - set delCount = 0 - set key = "" - for { set key = $order(deleted(key)) quit:key="" set delCount = delCount + 1 } + set delCount = ##class(%IPM.General.Sync.Output).CountNodes(.deleted) write !, " ", delCount, " deleted file(s) detected but not applied. Use -delete to remove from server." do ##class(%IPM.General.Sync.Summary).AddWarning(moduleName, delCount _ " deleted file(s) detected but not applied; use -delete to remove from server.") } + // module.xml can change without producing any routed file change; commit its new hash + // here so the next sync doesn't re-detect it. PrintTail emits the reload warning. if moduleXmlChanged { do ..SyncCommitModuleXml(module, moduleXmlPath, moduleXmlRelPath) - do ..SyncPrintModuleXmlWarning(moduleName) } - if $data(unsupportedResources) { - do ..SyncPrintUnsupportedNote(moduleName, verbose, .unsupportedResources) - } - write !, "[", moduleName, "] Sync done in ", $fnumber($zhorolog - syncStart, "", 2), "s" + do ##class(%IPM.General.Sync.Output).PrintTail(moduleName, verbose, moduleXmlChanged, syncStart, .unsupportedResources) quit } @@ -162,14 +112,7 @@ ClassMethod Run( // Step 6: Dispatch OnSync to each processor; load unhandled compilable files if verbose { - set resCount = 0 - set resName = "" - for { - set resName = $order(syncByResource(resName)) - quit:resName="" - set resCount = resCount + 1 - } - write !, "[", moduleName, "] Dispatching to ", resCount, " resource(s)" + do ##class(%IPM.General.Sync.Output).Log(moduleName, "Dispatching to "_##class(%IPM.General.Sync.Output).CountNodes(.syncByResource)_" resource(s)") } $$$ThrowOnError(..SyncDispatchProcessors(module, root, verbose, .syncByResource, .params, .loadItems)) @@ -212,20 +155,9 @@ ClassMethod Run( do ##class(%IPM.General.Sync.Summary).RecordFile(moduleName, key, "Deleted") } } - write !, "[", moduleName, "] Sync complete: ", modCount, " file(s) updated" - if delCount > 0 { - write ", ", delCount, " deleted" - } - write "." + do ##class(%IPM.General.Sync.Output).Log(moduleName, "Sync complete: "_modCount_" file(s) updated"_$select(delCount>0:", "_delCount_" deleted", 1:"")_".") - if moduleXmlChanged { - do ..SyncPrintModuleXmlWarning(moduleName) - } - if $data(unsupportedResources) { - do ..SyncPrintUnsupportedNote(moduleName, verbose, .unsupportedResources) - } - - write !, "[", moduleName, "] Sync done in ", $fnumber($zhorolog - syncStart, "", 2), "s" + do ##class(%IPM.General.Sync.Output).PrintTail(moduleName, verbose, moduleXmlChanged, syncStart, .unsupportedResources) // File sync is committed and reported. Mark success before running tests so a fatal // test failure below is reported as a test failure, not a sync error, in the summary. @@ -306,8 +238,8 @@ ClassMethod SyncBuildReverseIndex(module As %IPM.Storage.Module, orderedResource set docToResource(childName, "Processor") = resource.Processor set docToResource(childName, "Resource") = resource - // Newly-declared resources have no baseline row yet (StampModule skips uncompiled - // classes), so GetStoredPaths below won't find them. Map their relPath directly. + // A resource added since the last %Compile has no baseline row yet, so GetStoredPaths + // below won't find it. Map its relPath directly. set relPath = $get(childArr(childName, "RelativePath")) if relPath = "" { set relPath = resource.Processor.OnItemRelativePath(childName) @@ -562,38 +494,6 @@ ClassMethod SyncCommitModuleXml( } } -ClassMethod SyncPrintModuleXmlWarning(moduleName As %String) [ Private ] -{ - write ! - write !, "Warning: module.xml changed and was reloaded." - write !, " Resources may have been added/removed. Run `reload ", moduleName, "` to fully apply" - write !, " manifest-level changes (mappings, dependencies, defaults)." - do ##class(%IPM.General.Sync.Summary).AddWarning(moduleName, "module.xml changed and was reloaded; run `reload " _ moduleName _ "` to fully apply.") -} - -ClassMethod SyncPrintUnsupportedNote(moduleName As %String, verbose As %Boolean, ByRef unsupportedResources) [ Private ] -{ - set count = 0 - set names = "" - set resName = "" - for { - set resName = $order(unsupportedResources(resName)) - quit:resName="" - set count = count + 1 - if count <= 3 { - set names = names _ $select(names="":"", 1:", ") _ resName - } - } - if count > 3 { - set names = names _ ", ... (" _ (count - 3) _ " more)" - } - if verbose { - write !, "[", moduleName, "] ", count, " resource(s) skipped (no sync support): ", names - write !, " Use `reload ", moduleName, "` to apply changes to those resources." - } - do ##class(%IPM.General.Sync.Summary).AddWarning(moduleName, count _ " resource(s) skipped (no sync support): " _ names _ ".") -} - /// Recompile all compilable resources in the module to catch dependents invalidated by /// changes loaded in SyncDispatchProcessors. Skips CompileFromProject resources — those are /// deployed code loaded via Studio project files (.prj); compiling them individually would diff --git a/src/cls/IPM/General/Sync/Summary.cls b/src/cls/IPM/General/Sync/Summary.cls index 54d179d1b..60617a4e0 100644 --- a/src/cls/IPM/General/Sync/Summary.cls +++ b/src/cls/IPM/General/Sync/Summary.cls @@ -1,8 +1,8 @@ -/// Naked-sync summary accumulator. Main.Sync ("sync" with no module named) opens the summary with +/// Whole-namespace-sync summary accumulator. Main.Sync ("sync" with no module named) opens the summary with /// Begin, runs its per-module loop, then prints with Report. While open, each module's %Sync /// records its own outcome into the process-private global ^||IPM.Sync.Summary so the reader can /// aggregate without threading data through ExecutePhases' by-ref params. Shape: -/// ^||IPM.Sync.Summary("Active") = 1 while a sync-all is in progress +/// ^||IPM.Sync.Summary("Active") = 1 while a whole-namespace sync is in progress /// ^||IPM.Sync.Summary("Module",name,"File",relPath) = "Updated" | "Deleted" /// ^||IPM.Sync.Summary("Module",name,"SyncOK") = 1 once file sync committed (pre-tests) /// ^||IPM.Sync.Summary("Module",name,"TestsPassed") = running count of passed test methods @@ -93,7 +93,7 @@ ClassMethod AddTests( set ^||IPM.Sync.Summary("Module", moduleName, "TestsFailed") = $get(^||IPM.Sync.Summary("Module", moduleName, "TestsFailed"), 0) + failed } -/// Print the bordered sync-all summary from the accumulated ^||IPM.Sync.Summary data. +/// Print the bordered whole-namespace-sync summary from the accumulated ^||IPM.Sync.Summary data. /// orderedNames is the full dependency-ordered dev-mode module list (the "checked" set); /// failures is the $list of module names whose Sync phase returned an error status. A module in /// failures that still recorded SyncOK synced its files fine and only had a failing test — that diff --git a/src/cls/IPM/ResourceProcessor/Test.cls b/src/cls/IPM/ResourceProcessor/Test.cls index e8f6d0523..ac4bc86b2 100644 --- a/src/cls/IPM/ResourceProcessor/Test.cls +++ b/src/cls/IPM/ResourceProcessor/Test.cls @@ -505,7 +505,7 @@ Method OnSyncRunTests(ByRef classInfo, ByRef params) As %Status $$$ThrowOnError(sc) set suppressor = "" - // Tally this resource's per-method pass/fail into the naked-sync summary before + // Tally this resource's per-method pass/fail into the whole-namespace-sync summary before // ReportTestResults runs — at top level it consumes (kills) the result accumulator. do ..SyncSummaryTallyTests(..ResourceReference.Module.Name, phaseStartIndex) $$$ThrowOnError(..ReportTestResults(phaseStartIndex, verbose, .params)) @@ -517,9 +517,9 @@ Method OnSyncRunTests(ByRef classInfo, ByRef params) As %Status } /// Count the test methods run since phaseStartIndex and forward their pass/fail tally to the -/// naked-sync summary. A method counts as failed if it errored or has any failed assertion, -/// mirroring how GetAllTestsStatus/OutputFailures classify a failure; otherwise it passed. -/// No-op unless a sync-all is in progress (%IPM.General.Sync.Summary gates on that). +/// whole-namespace-sync summary. A method counts as failed if it errored or has any failed +/// assertion, mirroring how GetAllTestsStatus/OutputFailures classify a failure; otherwise passed. +/// No-op unless a whole-namespace sync is in progress (%IPM.General.Sync.Summary gates on that). ClassMethod SyncSummaryTallyTests( moduleName As %String, phaseStartIndex As %Integer) [ Private ] diff --git a/src/cls/IPM/Storage/FileHash.cls b/src/cls/IPM/Storage/FileHash.cls index 8c1f2b4fb..2a6a814f8 100644 --- a/src/cls/IPM/Storage/FileHash.cls +++ b/src/cls/IPM/Storage/FileHash.cls @@ -18,10 +18,33 @@ Index ModuleNameIndex On ModuleName; ForeignKey ModuleNameFK(ModuleName) References %IPM.Storage.Module(Name) [ OnDelete = cascade ]; +/// Collect the deduplicated set of directories sync should walk for a module. +/// Each sync-supporting resource processor declares its owned directory via GetSyncDirectory(); +/// this gathers them into scanDirs(relDir)="" and removes parent/child overlaps. Shared by +/// StampModule and the sync pipeline so both walk exactly the same directory set. +ClassMethod CollectScanDirs(module As %IPM.Storage.Module, Output scanDirs) +{ + kill scanDirs + set orderedResourceList = module.GetOrderedResourceList() + set key = "" + for { + set resource = orderedResourceList.GetNext(.key) + quit:key="" + if '$isobject(resource.Processor) || 'resource.Processor.SupportsSync() { + continue + } + set syncDir = resource.Processor.GetSyncDirectory() + if syncDir '= "" { + set scanDirs(syncDir) = "" + } + } + do ..DeduplicateScanDirs(.scanDirs) +} + /// Stamp all tracked files for a module. /// Collects scan directories from each resource processor via GetSyncDirectory(), deduplicates, -/// walks only those directories, then stamps compilable files present in the namespace. -/// Also stamps module.xml from the module root. +/// walks only those directories, then stamps every compilable file found there (no namespace +/// check — see the note at the stamping loop below). Also stamps module.xml from the module root. ClassMethod StampModule(module As %IPM.Storage.Module) As %Status { set sc = $$$OK @@ -29,21 +52,7 @@ ClassMethod StampModule(module As %IPM.Storage.Module) As %Status set root = ##class(%File).NormalizeDirectory(module.Root) // Collect and deduplicate scan directories from resource processors. - kill scanDirs - set orderedResourceList = module.GetOrderedResourceList() - set key = "" - for { - set resource = orderedResourceList.GetNext(.key) - quit:key="" - if '$isobject(resource.Processor) || 'resource.Processor.SupportsSync() { - continue - } - set syncDir = resource.Processor.GetSyncDirectory() - if syncDir '= "" { - set scanDirs(syncDir) = "" - } - } - do ..DeduplicateScanDirs(.scanDirs) + do ..CollectScanDirs(module, .scanDirs) // Walk each declared directory. kill allFiles, allHashes @@ -143,15 +152,16 @@ ClassMethod RelPathToDocName(relPath As %String) As %String } /// Compute which files changed on disk vs stored baseline. -/// Pass 1: filter pre-walked BFS data for compilable files present in the namespace. -/// Pass 2: check manifest-derived paths (from reverseIndex) for files not yet compiled +/// Pass 1: compilable files present in the namespace, from the pre-walked file set. +/// Pass 2: manifest-derived paths (from reverseIndex) for files not yet compiled /// or non-compilable tracked files. /// Pass 3: iterate stored rows to detect deletions. /// allFiles(relPath)=fullPath is the full pre-walked module root (all extensions). /// allHashes(relPath)=sha1hex are pre-computed hashes from the same walk. -/// bfsFiles(relPath)=fullPath is the compilable subset (cls,inc,mac,int). +/// module.xml is deliberately excluded from all three passes — it is owned by the pipeline's +/// SyncCheckModuleXml/SyncCommitModuleXml, never routed as a normal file. /// Returns modified(relPath)=newHash and deleted(relPath)="" arrays. -ClassMethod ComputeChanges(module As %IPM.Storage.Module, Output modified, Output deleted, ByRef manifestPaths, ByRef bfsFiles, ByRef allFiles, ByRef allHashes) As %Status +ClassMethod ComputeChanges(module As %IPM.Storage.Module, Output modified, Output deleted, ByRef reverseIndex, ByRef allFiles, ByRef allHashes) As %Status { set sc = $$$OK kill modified, deleted @@ -159,16 +169,11 @@ ClassMethod ComputeChanges(module As %IPM.Storage.Module, Output modified, Outpu set moduleName = module.Name kill seen - // module.xml - set moduleXmlRel = "module.xml" - if $data(allHashes(moduleXmlRel)) { - do ..CompareOneFile(moduleName, moduleXmlRel, $get(allHashes(moduleXmlRel)), .modified, .seen) - } - - // Pass 1: filter pre-walked compilable files by namespace presence. + // Pass 1: compilable files present in the namespace. RelPathToDocName returns "" for + // non-compilable extensions, so iterating allFiles and skipping empty docNames suffices. set relPath = "" for { - set relPath = $order(bfsFiles(relPath)) + set relPath = $order(allFiles(relPath)) quit:relPath="" set docName = ..RelPathToDocName(relPath) if docName = "" { @@ -188,11 +193,12 @@ ClassMethod ComputeChanges(module As %IPM.Storage.Module, Output modified, Outpu do ..CompareOneFile(moduleName, relPath, $get(allHashes(relPath)), .modified, .seen) } - // Pass 2: check manifest-derived paths for files BFS missed (uncompiled new files, - // non-compilable tracked files). + // Pass 2: check manifest-derived paths for files Pass 1 missed (uncompiled new files, + // non-compilable tracked files). Top-level $order over reverseIndex yields relPaths only; + // the per-path "Processor"/"Resource" subnodes are children and are skipped. set relPath = "" for { - set relPath = $order(manifestPaths(relPath)) + set relPath = $order(reverseIndex(relPath)) quit:relPath="" if '$data(allFiles(relPath)) { continue @@ -200,12 +206,17 @@ ClassMethod ComputeChanges(module As %IPM.Storage.Module, Output modified, Outpu do ..CompareOneFile(moduleName, relPath, $get(allHashes(relPath)), .modified, .seen) } - // Pass 3: iterate all stored rows — file not in allFiles → deleted. + // Pass 3: iterate all stored rows — file not in allFiles → deleted. module.xml is skipped: + // it is always a stored row but is not part of the walked file set, so it would otherwise + // be misreported as deleted every sync. set result = ##class(%SQL.Statement).%ExecDirect(, "SELECT RelativePath FROM %IPM_Storage.FileHash WHERE ModuleName = ?", moduleName) while result.%Next() { set storedRelPath = result.%Get("RelativePath") + if storedRelPath = "module.xml" { + continue + } if '$data(allFiles(storedRelPath)) { set deleted(storedRelPath) = "" } @@ -217,6 +228,8 @@ ClassMethod ComputeChanges(module As %IPM.Storage.Module, Output modified, Outpu } /// Compare a file's pre-computed hash against stored baseline. No filesystem I/O. +/// Mutates the caller's by-ref arrays: sets modified(relPath)=newHash when the file is new or +/// changed, and marks seen(relPath) so a later pass won't re-compare the same path. ClassMethod CompareOneFile(moduleName As %String, relPath As %String, newHash As %String, ByRef modified, ByRef seen) [ Private ] { if $data(seen(relPath)) { diff --git a/src/cls/IPM/Utils/Module.cls b/src/cls/IPM/Utils/Module.cls index 3c0a8766c..87e3208a2 100644 --- a/src/cls/IPM/Utils/Module.cls +++ b/src/cls/IPM/Utils/Module.cls @@ -1450,7 +1450,7 @@ ClassMethod GetFlatDependencyListFromInvertedDependencyGraph(ByRef pInvertedDepe /// Returns a $list of dev-mode module names ordered least-dependent-first: a module's /// dependencies appear before it. Only dependency edges among the dev-mode set are -/// considered; unconnected modules keep their natural row order. Used by sync-all so each +/// considered; unconnected modules keep their natural row order. Used by whole-namespace sync so each /// module recompiles against dependencies already synced in the same run. ClassMethod GetDevModeModulesInDependencyOrder() As %List {