From cfd2c717cbfc4bddce3df2678bcde80954d5560f Mon Sep 17 00:00:00 2001 From: geoffreyweal Date: Fri, 31 Jul 2026 08:19:47 +1200 Subject: [PATCH] Add OOD embedding, custom templates, a script browser and a Nodes page Recorded under [Unreleased] in CHANGELOG.md. No version number is assigned and VERSION stays at 2.0.2, since the release number is the maintainer's call. Adds Open OnDemand embedding, a custom template system, a script browser, a Nodes page, module-aware form widgets and a substantially extended history page. Open OnDemand integration - OOD-integrated (embedded) mode: Open Composer renders inside OOD's own chrome (live OOD navbar and footer) via a reverse-proxy dashboard initializer under ood_integration/, with no iframe. Home page, templates and the script browser - All Templates page (/all_templates): every template as one flat list, including applications hidden from the index page. - Custom templates: save the current form values as a reusable template, with inline rename/re-describe, delete, and drag-and-drop ordering. - Load or Create a Script from Disk: a full-page file browser that opens .sl/.sh/.bash/.sbatch/.batch/.slurm files in the generic scheduler form with script location and name pre-filled. - New Script and New Custom Template pickers. - home_format: small (compact letter-grouped listing), per-category badge colors, and automatic GPU badges via the tags manifest key. Application forms - module_load widget: a dropdown of every available version of an environment module, rewriting the existing module load line in place so surrounding hand-written script lines survive. - dependent_module_select widget: a module list that follows the value of another (driver) widget. - remember_last on select widgets. - Script-to-form parsing: editing the script pane updates the widgets, and opening a job from the history page reconstructs the whole form. - Empty-script submission warning. History page - sacct is now the source of truth, so jobs submitted outside Open Composer appear too, with their script fetched live. - Job Efficiency panel: wall time, CPU, memory, and GPU utilization and memory for GPU jobs. - Live job details modal, with disclosure of the exact command run. - Output/Error columns with an in-page file viewer. - Cancel All Jobs, Delete All History and Refresh, plus per-job cancellation with a progress bar. - Per-column sorting and Cancelled/Unknown status filters. Nodes page - A new page listing the cluster's compute nodes with state, CPU, memory and generic resources, built from sinfo, with one dynamically discovered column per GRES type. Job submission - Automatic SSH key provisioning on Submit (opt-in, off by default). - Configurable modules_list_url for the site module catalog. Configuration and theming - The accent palette, GPU memory capacities and navbar/footer branding are now configuration rather than hard-coded values, so a site can be restyled without editing a view. - conf.yml.erb.sample reorganized into ten numbered sections, matching the installation documentation. Also extends the form.yml smoke tests under misc/tests/ to cover the new widgets, falling back to macOS's bundled jsc when node is unavailable. Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 12 +- CHANGELOG.md | 38 + README.md | 123 +- conf.yml.erb.sample | 286 +++- docs/application.html | 431 +++++- docs/application_ja.html | 312 +++- docs/img/all_templates.png | Bin 0 -> 363671 bytes docs/img/application_orca.png | Bin 0 -> 554971 bytes docs/img/application_slurm.png | Bin 0 -> 444779 bytes docs/img/bundle.png | Bin 129797 -> 148213 bytes docs/img/clone.png | Bin 85856 -> 128317 bytes docs/img/dependent_module_select.png | Bin 0 -> 27692 bytes docs/img/hide-script-header.png | Bin 249931 -> 166736 bytes docs/img/history.png | Bin 0 -> 490769 bytes docs/img/history_page.png | Bin 179817 -> 558005 bytes docs/img/history_page2.png | Bin 152182 -> 371290 bytes docs/img/history_page3.png | Bin 169828 -> 455316 bytes docs/img/history_page5.png | Bin 108498 -> 104600 bytes docs/img/history_table3.png | Bin 59991 -> 92415 bytes docs/img/home.png | Bin 0 -> 357473 bytes docs/img/load_script.png | Bin 0 -> 169263 bytes docs/img/navbar.png | Bin 35485 -> 106125 bytes docs/img/new_repo.png | Bin 167165 -> 237569 bytes docs/img/newapp.png | Bin 21215 -> 28915 bytes docs/img/no_entry_warning_message.png | Bin 0 -> 168443 bytes docs/img/nodes.png | Bin 0 -> 132563 bytes docs/img/overwrite_warning.png | Bin 50949 -> 0 bytes docs/img/path2.png | Bin 74012 -> 79851 bytes docs/img/submit2.png | Bin 192620 -> 100693 bytes docs/install.html | 561 +++++-- docs/install_ja.html | 550 ++++++- docs/manual.html | 343 +++- docs/manual_ja.html | 335 +++- generic_apps/Fujitsu_TCS/form.yml | 12 + generic_apps/Fujitsu_TCS/logo.png | Bin 0 -> 3512 bytes generic_apps/Fujitsu_TCS/manifest.yml | 6 + generic_apps/Grid_Engine/form.yml | 12 + generic_apps/Grid_Engine/logo.png | Bin 0 -> 34931 bytes generic_apps/Grid_Engine/manifest.yml | 6 + generic_apps/PBS/form.yml | 12 + generic_apps/PBS/icon.png | Bin 0 -> 4131 bytes generic_apps/PBS/manifest.yml | 6 + generic_apps/Slurm/README_icon.md | 1 + generic_apps/Slurm/Slurm_logo.svg | 6 + generic_apps/Slurm/form.yml | 12 + generic_apps/Slurm/manifest.yml | 6 + lib/form.rb | 302 +++- lib/history.rb | 1374 ++++++++--------- lib/index.rb | 171 +- lib/scheduler.rb | 61 + lib/schedulers/fujitsu_tcs.rb | 85 + lib/schedulers/miyabi.rb | 54 +- lib/schedulers/pbspro.rb | 124 ++ lib/schedulers/sge.rb | 154 ++ lib/schedulers/slurm.rb | 601 ++++++- misc/check_erb.rb | 81 + misc/tests/README.md | 7 +- misc/tests/extract_samples.rb | 67 +- misc/tests/run_tests.rb | 28 +- ood_integration/ADMIN_CHECKLIST.md | 114 ++ ood_integration/dashboard.env.example | 45 + .../initializers/opencomposer_embed.rb | 370 +++++ .../views/apps/opencomposer_embed.html.erb | 8 + public/OpenOnDemand_powered_by_invert.svg | 37 + public/RIKEN_powered_by.svg | 59 + public/Riken_Logo.gif | Bin 0 -> 9636 bytes public/app_default.svg | 29 + public/favicon.ico | Bin 0 -> 32038 bytes public/form.js | 505 +++++- public/history.js | 613 +++++++- run.rb | 1310 ++++++++++++++-- views/all_templates.erb | 10 + views/form.erb | 360 +++-- views/history.erb | 185 ++- views/index.erb | 254 ++- views/layout.erb | 323 +++- views/load_script.erb | 367 +++++ views/new_script.erb | 24 + views/new_template.erb | 107 ++ views/nodes.erb | 454 ++++++ 80 files changed, 9732 insertions(+), 1621 deletions(-) create mode 100644 docs/img/all_templates.png create mode 100644 docs/img/application_orca.png create mode 100644 docs/img/application_slurm.png create mode 100644 docs/img/dependent_module_select.png create mode 100644 docs/img/history.png create mode 100644 docs/img/home.png create mode 100644 docs/img/load_script.png create mode 100644 docs/img/no_entry_warning_message.png create mode 100644 docs/img/nodes.png delete mode 100644 docs/img/overwrite_warning.png create mode 100644 generic_apps/Fujitsu_TCS/form.yml create mode 100644 generic_apps/Fujitsu_TCS/logo.png create mode 100644 generic_apps/Fujitsu_TCS/manifest.yml create mode 100644 generic_apps/Grid_Engine/form.yml create mode 100644 generic_apps/Grid_Engine/logo.png create mode 100644 generic_apps/Grid_Engine/manifest.yml create mode 100644 generic_apps/PBS/form.yml create mode 100644 generic_apps/PBS/icon.png create mode 100644 generic_apps/PBS/manifest.yml create mode 100644 generic_apps/Slurm/README_icon.md create mode 100644 generic_apps/Slurm/Slurm_logo.svg create mode 100644 generic_apps/Slurm/form.yml create mode 100644 generic_apps/Slurm/manifest.yml create mode 100644 misc/check_erb.rb create mode 100644 ood_integration/ADMIN_CHECKLIST.md create mode 100644 ood_integration/dashboard.env.example create mode 100644 ood_integration/initializers/opencomposer_embed.rb create mode 100644 ood_integration/views/apps/opencomposer_embed.html.erb create mode 100644 public/OpenOnDemand_powered_by_invert.svg create mode 100644 public/RIKEN_powered_by.svg create mode 100644 public/Riken_Logo.gif create mode 100644 public/app_default.svg create mode 100644 public/favicon.ico create mode 100644 views/all_templates.erb create mode 100644 views/load_script.erb create mode 100644 views/new_script.erb create mode 100644 views/new_template.erb create mode 100644 views/nodes.erb diff --git a/.gitignore b/.gitignore index cc53b3f6..55a81587 100644 --- a/.gitignore +++ b/.gitignore @@ -2,12 +2,18 @@ .DS_Store # bundle -Gemfile -Gemfile.lock +# Anchored to the repo root so the misc/Gemfile template stays tracked — the +# install docs tell administrators to copy it here. +/Gemfile +/Gemfile.lock vendor/ .bundle -TODO_ja.md + +# Local site configuration, copied from conf.yml.erb.sample conf.yml.erb +# Passenger restart trigger +tmp/ + # Regenerated from docs by misc/tests/run_tests.rb misc/tests/samples/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 94c0e197..2f6af533 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,43 @@ # Changelog +## [Unreleased] +### Add +- Add an OOD-integrated (embedded) mode that renders Open Composer inside Open OnDemand's own navbar and footer, via a reverse-proxy dashboard initializer under `ood_integration/` and without an iframe. +- Add an All Templates page (`/all_templates`) listing every template as one flat list, including applications hidden from the index page. +- Add custom templates: save the current form values as a reusable template, then rename, re-describe, delete or reorder them from the index page. +- Add a "Load or Create a Script from Disk" file browser that opens an existing batch script in the generic scheduler form with its location and name pre-filled. +- Add New Script and New Custom Template pickers for choosing which application to start from. +- Add a Nodes page listing the cluster's compute nodes with their state, CPU, memory and generic resources, with one dynamically discovered column per GRES type. +- Add a `module_load` widget: a dropdown of every available version of an environment module, which rewrites the existing `module load` line in place so surrounding script lines are preserved. +- Add a `dependent_module_select` widget whose module list follows the value of another widget. +- Add `remember_last` to `select` widgets, restoring the user's previous choice from browser storage. +- Add script-to-form parsing, so editing the script pane, or opening a job from the history page, reconstructs the form from the script's scheduler directives. +- Add an empty-script submission warning (`empty_script_warning`). +- Add a Job Efficiency panel to the history page (`history_efficiency`), reporting wall time, CPU, memory and, for GPU jobs, GPU utilization and GPU memory. +- Add output and error columns to the history page, with an in-page file viewer. +- Add "Cancel All Jobs", "Delete All History" and "Refresh" actions to the history page, plus per-job cancellation with progress reporting. +- Add optional SSH key provisioning on submit (`ensure_ssh_key`), disabled by default. +- Add a `modules_list_url` setting for the module catalog behind the module widgets. +- Add `home_format`, `category_badge_colors`, `accent_color`, `body_text_color`, `gpu_memory` and `favicon` settings. +- Add `hidden`, `documentation` and `tags` manifest keys, and allow `category` to be given as a list. + +### Changed +- Use `sacct` as the source of truth on the history page, so jobs submitted outside Open Composer also appear, with their script fetched live. +- Move the generic per-scheduler script templates to `generic_apps_dir`, reachable at `/_generic/` and used as the fallback when an application's `form.yml` is missing. +- Replace the hard-coded accent palette and GPU memory capacities with settings, so a site can be restyled without editing a view. +- Reorganize `conf.yml.erb.sample` into ten numbered sections, matching section 2 of `docs/install.html`. +- Pass `scheduler_env` to every scheduler command, not only to submission, cancellation and status queries. +- Extend the `form.yml` smoke tests to cover the new widgets, and fall back to macOS's bundled `jsc` when `node` is unavailable. + +### Fixed +- Anchor the `enable--\n" + html += "\n" + @table_index += 1 + html + output_help(key, value) + end + + # JavaScript to initialize a dependent_module_select: watches a driver widget and re-fetches + # the module version list whenever the driver's selected value changes prefix group. + def output_dependent_module_select_js(key, value) + driver = value['driver'].to_s + modules = value['options'] || [] + sn = @script_name.to_s + + mod_map_js = modules.map { |m| + parts = [] + parts << "prefix: #{m['prefix'].to_s.to_json}" if m['prefix'] + parts << "contains: #{m['contains'].to_s.to_json}" if m['contains'] + parts << "module: #{m['module'].to_s.to_json}" + "{#{parts.join(', ')}}" + }.join(", ") + + <<~JS + (function() { + var sel = document.getElementById(#{key.to_json}); + var driver = document.getElementById(#{driver.to_json}); + if (!sel) return; + var modMap = [#{mod_map_js}]; + var sn = #{sn.to_json}; + var lastModule = null; + + function moduleForValue(val) { + var s = String(val); + for (var i = 0; i < modMap.length; i++) { + if (modMap[i].prefix && s.startsWith(modMap[i].prefix)) return modMap[i].module; + if (modMap[i].contains && s.includes(modMap[i].contains)) return modMap[i].module; + } + return modMap.length > 0 ? modMap[modMap.length - 1].module : ''; + } + + function loadModules(moduleName) { + if (!moduleName || moduleName === lastModule) return; + lastModule = moduleName; + var urlParams = new URLSearchParams(window.location.search); + var cluster = urlParams.get('_cluster_name') || ''; + fetch(sn + '/_module_avail?module=' + encodeURIComponent(moduleName) + '&cluster=' + encodeURIComponent(cluster)) + .then(function(r) { return r.json(); }) + .then(function(mods) { + sel.innerHTML = ''; + if (!mods.length) { + var opt = document.createElement('option'); + opt.value = ''; opt.dataset.value = ''; opt.textContent = 'No modules found'; + sel.appendChild(opt); return; + } + mods.forEach(function(m) { + var opt = document.createElement('option'); + opt.value = m; opt.dataset.value = m; opt.textContent = m; + sel.appendChild(opt); + }); + if (sel.selectedIndex === -1) sel.selectedIndex = 0; + sel.dispatchEvent(new Event('change')); + }) + .catch(function() { + sel.innerHTML = ''; + }); + } + + function updateFromDriver() { + if (!driver) return; + var idx = driver.selectedIndex; + var driverVal = (idx >= 0 && driver.options[idx] && driver.options[idx].dataset.value) + ? driver.options[idx].dataset.value + : (driver.value || ''); + loadModules(moduleForValue(driverVal)); + } + + updateFromDriver(); + if (driver) driver.addEventListener('change', updateFromDriver); + })(); + JS + end + # Output a path widget. def output_path_html(key, value, script_content, submit_content, app_name, dir_name) favorites = value['favorites'] ? value['favorites'].select { |path| File.exist?(path) } : [] @@ -554,7 +775,7 @@ def output_path_html(key, value, script_content, submit_content, app_name, dir_n 'submit' end if type - html += "oninput=\"ocForm.confirmOverwrite('#{type}', '#{key}', function(){ocForm.updateArea('#{type}', '#{key}')})\" " + html += "oninput=\"ocForm.updateArea('#{type}', '#{key}')\" " html += "onfocus=\"ocForm.storePreviousValue('#{key}')\"" html += " style=\"background-color: #{@conf["submit_color"]};\"" if type == 'submit' else @@ -638,7 +859,7 @@ def output_path_html(key, value, script_content, submit_content, app_name, dir_n HTML html += "" \ + "\n " \ + : "" + <<~HTML