From 61b472e10c413776d7558844b45a981b2d574532 Mon Sep 17 00:00:00 2001 From: Matee ullah Malik Date: Tue, 11 Aug 2026 09:54:56 +0000 Subject: [PATCH] fix(devnet): make NM processedPath a sibling of srcPath, not a child e357c6e7 fixed the scanner block to emit correctly-typed inline tables, but derived the destination as "${dir%/}/processed" -- i.e. INSIDE the directory being scanned. scanner/scanner.go walks srcPath with filepath.Walk (recursive) and dedups on (dir, name). A nested processed dir is therefore re-scanned on the next tick, and because the move changed `dir` the DB lookup misses, so the file is registered a SECOND time. Every upload costs two cascade registrations (~15120ulume each) and double supernode load. Observed on the shared devnet: 13 `create-metadata start` calls whose source path was already under .../processed/. Deriving the sibling "${dir%/}-processed" restores the uploader's own convention -- its shipped config.toml pairs ~/.lumera-uploader/drop with the sibling ~/.lumera-uploader/processed, never a child. Verified on the shared devnet (5 validators, chain v1.20.2-rc1): after the change, 210 uploads with 0 re-registrations sourced from the processed dir, and processed_files advanced 1242 -> 1276. Note for existing devnets: files already sitting in a nested /processed must be moved out of the scan path before restarting, otherwise they are re-registered once on the next pass. --- devnet/scripts/lumera-uploader-setup.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/devnet/scripts/lumera-uploader-setup.sh b/devnet/scripts/lumera-uploader-setup.sh index 0df9f8f8..92bc0c9f 100755 --- a/devnet/scripts/lumera-uploader-setup.sh +++ b/devnet/scripts/lumera-uploader-setup.sh @@ -308,7 +308,13 @@ add_dir_to_scanner() { return 0 fi - local processed="${dir%/}/processed" + # processedPath MUST be a SIBLING of srcPath, not nested inside it. + # scanner/scanner.go walks srcPath with filepath.Walk (RECURSIVE) and dedups + # on (dir, name). A nested "/processed" therefore gets re-scanned, + # and because the move changes `dir` the file looks brand new -> every file + # is registered a SECOND time, doubling cascade fees and load. + # Observed on devnet: 13 create-metadata calls sourced from .../processed/. + local processed="${dir%/}-processed" local tmp="${cfg}.tmp.$$" # Collect existing srcPath entries so repeated calls accumulate rather than @@ -323,7 +329,7 @@ add_dir_to_scanner() { for e in ${existing}; do [ -n "$e" ] || continue printf ',\n { srcPath = "%s", processedPath = "%s", isPublic = "random" }' \ - "$e" "${e%/}/processed" + "$e" "${e%/}-processed" done printf '\n]\n' } >"${tmp}.block"