path-cc · CannonLock · Jul 31, 2026 · Jul 30, 2026 · Jul 30, 2026 · Jul 31, 2026
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -312,4 +312,4 @@ DEPENDENCIES
   jekyll-sitemap
 
 BUNDLED WITH
-  4.0.1
+  4.0.17
diff --git a/images/nairr/final-report/diagrams/dataflow_checkpoint_chain.mmd b/images/nairr/final-report/diagrams/dataflow_checkpoint_chain.mmd
@@ -0,0 +1,17 @@
+sequenceDiagram
+    participant AP
+    participant W1 as Worker (epoch 1)
+    participant W2 as Worker (epoch 2)
+    participant W3 as Worker (epoch 3)
+
+    AP->>W1: training data + script + container
+    W1->>W1: train epoch 1
+    W1->>AP: output/ (checkpoints)
+
+    AP->>W2: training data + script + container + checkpoint
+    W2->>W2: train epoch 2 (resume from last.ckpt)
+    W2->>AP: output/ (checkpoints)
+
+    AP->>W3: training data + script + container + checkpoint
+    W3->>W3: train epoch 3 (resume from last.ckpt)
+    W3->>AP: output/ (checkpoints)
diff --git a/images/nairr/final-report/diagrams/dataflow_checkpoint_chain.png b/images/nairr/final-report/diagrams/dataflow_checkpoint_chain.png
diff --git a/images/nairr/final-report/diagrams/dataflow_epoch.mmd b/images/nairr/final-report/diagrams/dataflow_epoch.mmd
@@ -0,0 +1,29 @@
+flowchart LR
+    subgraph OSDF["OSDF (cached from AP)"]
+        img["Container image\nmetl_global_fixed_time_2025-10-10.sif"]
+        data["Training data\nprocessed-global.tar.gz (~19 GB)"]
+    end
+
+    subgraph AP["Access Point (ap40)"]
+        script["pretrain.sh"]
+        ckpt_in["Checkpoint dir\noutput/training_logs/{run_uuid}/\n(epoch > 1 only)"]
+        ckpt_out["Checkpoint dir\noutput/training_logs/{run_uuid}/"]
+        logs["Job logs\n{run_uuid}/{epoch}_{CLUSTERID}.out/.err"]
+    end
+
+    subgraph Worker["Worker Node"]
+        sandbox["HTCondor sandbox"]
+        train["train_source_model.py"]
+    end
+
+    WandB["WandB\n(external)"]
+
+    img -->|"OSDF pull"| sandbox
+    data -->|"OSDF pull"| sandbox
+    script -->|"file transfer"| sandbox
+    ckpt_in -->|"file transfer"| sandbox
+
+    sandbox --> train
+    train -->|"metrics stream"| WandB
+    train -->|"ON_EXIT_OR_EVICT"| ckpt_out
+    train -->|"ON_EXIT_OR_EVICT"| logs
diff --git a/images/nairr/final-report/diagrams/dataflow_epoch.png b/images/nairr/final-report/diagrams/dataflow_epoch.png
diff --git a/images/nairr/final-report/images/DAG.png b/images/nairr/final-report/images/DAG.png
diff --git a/images/nairr/final-report/images/dagman_monitor.png b/images/nairr/final-report/images/dagman_monitor.png
diff --git a/images/nairr/final-report/images/data_transfer_by_resource.png b/images/nairr/final-report/images/data_transfer_by_resource.png
diff --git a/images/nairr/final-report/images/experiment_summary_report.txt b/images/nairr/final-report/images/experiment_summary_report.txt
@@ -0,0 +1,61 @@
+EXPERIMENT ANALYSIS SUMMARY REPORT
+==================================================
+
+Analysis Period: 2025-04-16 to 2025-08-16
+Total Job Attempts: 936
+Unique Jobs: 470
+
+RESOURCE UTILIZATION:
+--------------------
+OSPOOL:
+  • Successful Jobs: 558 (100.0%)
+  • Computation Time: 2,295.1 hours
+  • Time Efficiency: 20.6%
+
+EXPANSE:
+  • Successful Jobs: 18 (100.0%)
+  • Computation Time: 295.6 hours
+  • Time Efficiency: 15.2%
+
+DELTA:
+  • Successful Jobs: 17 (100.0%)
+  • Computation Time: 404.4 hours
+  • Time Efficiency: 13.6%
+
+BRIDGES2:
+  • Successful Jobs: 7 (100.0%)
+  • Computation Time: 31.3 hours
+  • Time Efficiency: 5.5%
+
+ANVIL:
+  • Successful Jobs: 8 (100.0%)
+  • Computation Time: 202.1 hours
+  • Time Efficiency: 41.7%
+
+OVERALL METRICS:
+---------------
+Total Success Rate: 100.0%
+Total Computation Time: 3,228.5 hours
+Overall Time Efficiency: 100.0%
+
+GPU UTILIZATION:
+----------------
+Total GPU Hours: 5,318.6
+GPU Types Used: 6
+
+NVIDIA A100-PCIE-40GB: 6.0 jobs, 100.0% success rate
+NVIDIA A100-SXM4-40GB: 38.0 jobs, 100.0% success rate
+NVIDIA A100-SXM4-80GB: 73.0 jobs, 100.0% success rate
+NVIDIA A40: 108.0 jobs, 100.0% success rate
+NVIDIA RTX A5000: 354.0 jobs, 100.0% success rate
+Tesla V100-SXM2-32GB: 41.0 jobs, 100.0% success rate
+
+DATA TRANSFER:
+-------------
+Total Data Transferred: 18,386.2 GB
+
+anvil: 15.0 GB (8.0 jobs)
+bridges2: 24.3 GB (7.0 jobs)
+delta: 43.4 GB (21.0 jobs)
+expanse: 34.1 GB (21.0 jobs)
+ospool: 18,269.3 GB (640.0 jobs)
diff --git a/images/nairr/final-report/images/gpu_efficiency_analysis.png b/images/nairr/final-report/images/gpu_efficiency_analysis.png
diff --git a/images/nairr/final-report/images/resource_epoch_breakdown.png b/images/nairr/final-report/images/resource_epoch_breakdown.png
diff --git a/...irr/final-report/mldag_reports/global_pretraining/data_transfer_by_resource.png b/...irr/final-report/mldag_reports/global_pretraining/data_transfer_by_resource.png
diff --git a/...rr/final-report/mldag_reports/global_pretraining/epochs_completed_over_time.png b/...rr/final-report/mldag_reports/global_pretraining/epochs_completed_over_time.png
diff --git a/.../nairr/final-report/mldag_reports/global_pretraining/epochs_trained_per_day.png b/.../nairr/final-report/mldag_reports/global_pretraining/epochs_trained_per_day.png
diff --git a/images/nairr/final-report/mldag_reports/global_pretraining/experiment_summary_report.txt b/images/nairr/final-report/mldag_reports/global_pretraining/experiment_summary_report.txt
@@ -0,0 +1,84 @@
+EXPERIMENT ANALYSIS SUMMARY REPORT
+==================================================
+
+Analysis Period: 2025-08-20 09:51 to 2026-06-20 05:54
+Total Job Attempts: 1,382
+Unique Jobs: 361
+Total Completed Epochs: 600
+
+RESOURCE UTILIZATION:
+--------------------
+EXPANSE:
+  • Successful Jobs: 78 (100.0%)
+  • Computation Time: 479.7 hours
+  • Time Efficiency: 10.6%
+
+OSPOOL:
+  • Successful Jobs: 367 (100.0%)
+  • Computation Time: 4,278.8 hours
+  • Time Efficiency: 10.1%
+
+BRIDGES2:
+  • Successful Jobs: 18 (100.0%)
+  • Computation Time: 116.4 hours
+  • Time Efficiency: 8.0%
+
+DELTA:
+  • Successful Jobs: 65 (100.0%)
+  • Computation Time: 680.8 hours
+  • Time Efficiency: 11.2%
+
+AWS:
+  • Successful Jobs: 59 (100.0%)
+  • Computation Time: 306.8 hours
+  • Time Efficiency: 22.3%
+
+ANVIL:
+  • Successful Jobs: 1 (100.0%)
+  • Computation Time: 21.6 hours
+  • Time Efficiency: 0.9%
+
+OVERALL METRICS:
+---------------
+Total Success Rate: 102.0%
+Total Computation Time: 5,884.1 hours
+Overall Time Efficiency: 100.0%
+
+GPU UTILIZATION:
+----------------
+Total GPU Hours: 20,045.1
+GPU Types Used: 8
+
+NVIDIA A100 80GB PCIe: 64.0 jobs, 100.0% success rate
+NVIDIA A100-SXM4-40GB: 61.0 jobs, 100.0% success rate
+NVIDIA A100-SXM4-80GB: 135.0 jobs, 100.0% success rate
+NVIDIA A40: 37.0 jobs, 100.0% success rate
+NVIDIA H100 80GB HBM3: 47.0 jobs, 100.0% success rate
+NVIDIA H200: 121.0 jobs, 100.0% success rate
+NVIDIA L40S: 55.0 jobs, 100.0% success rate
+Tesla V100-SXM2-32GB: 92.0 jobs, 100.0% success rate
+
+DATA TRANSFER:
+-------------
+Total Data Transferred: 25,375.2 GB
+
+anvil: 26.0 GB (1.0 jobs)
+aws: 1,957.7 GB (62.0 jobs)
+bridges2: 195.6 GB (27.0 jobs)
+delta: 1,891.3 GB (71.0 jobs)
+expanse: 743.3 GB (104.0 jobs)
+ospool: 20,561.3 GB (557.0 jobs)
+
+INPUT FILE TRANSFER TIMING:
+---------------------------
+Jobs with Transfer Timing: 1,382
+Mean Transfer Duration: 6.6 minutes
+Median Transfer Duration: 2.5 minutes
+Total Transfer Time: 152.4 hours
+
+anvil: median 4.0 min, mean 5.3 min (8 jobs)
+aws: median 4.5 min, mean 6.3 min (74 jobs)
+bridges2: median 7.8 min, mean 13.7 min (34 jobs)
+delta: median 2.2 min, mean 4.6 min (92 jobs)
+expanse: median 2.2 min, mean 4.5 min (138 jobs)
+ospool: median 2.4 min, mean 6.9 min (1036 jobs)
diff --git a/.../nairr/final-report/mldag_reports/global_pretraining/gpu_capability_heatmap.png b/.../nairr/final-report/mldag_reports/global_pretraining/gpu_capability_heatmap.png
diff --git a/...irr/final-report/mldag_reports/global_pretraining/gpu_distribution_analysis.png b/...irr/final-report/mldag_reports/global_pretraining/gpu_distribution_analysis.png
diff --git a/...nairr/final-report/mldag_reports/global_pretraining/gpu_efficiency_analysis.png b/...nairr/final-report/mldag_reports/global_pretraining/gpu_efficiency_analysis.png
diff --git a/images/nairr/final-report/mldag_reports/global_pretraining/gpu_hours_over_time.png b/images/nairr/final-report/mldag_reports/global_pretraining/gpu_hours_over_time.png
diff --git a/...nal-report/mldag_reports/global_pretraining/gpu_runtime_by_resource_boxplot.png b/...nal-report/mldag_reports/global_pretraining/gpu_runtime_by_resource_boxplot.png
diff --git a/...nal-report/mldag_reports/global_pretraining/resource_breakdown_bars_glidein.png b/...nal-report/mldag_reports/global_pretraining/resource_breakdown_bars_glidein.png
diff --git a/...report/mldag_reports/global_pretraining/resource_computation_time_breakdown.png b/...report/mldag_reports/global_pretraining/resource_computation_time_breakdown.png
diff --git a/...ldag_reports/global_pretraining/resource_computation_time_breakdown_glidein.png b/...ldag_reports/global_pretraining/resource_computation_time_breakdown_glidein.png
diff --git a/...airr/final-report/mldag_reports/global_pretraining/resource_epoch_breakdown.png b/...airr/final-report/mldag_reports/global_pretraining/resource_epoch_breakdown.png
diff --git a/...irr/final-report/mldag_reports/global_pretraining/resource_epochs_breakdown.png b/...irr/final-report/mldag_reports/global_pretraining/resource_epochs_breakdown.png
diff --git a/...l-report/mldag_reports/global_pretraining/resource_epochs_breakdown_glidein.png b/...l-report/mldag_reports/global_pretraining/resource_epochs_breakdown_glidein.png
diff --git a/...l-report/mldag_reports/global_pretraining/transfer_input_timing_by_resource.png b/...l-report/mldag_reports/global_pretraining/transfer_input_timing_by_resource.png
diff --git a/...nal-report/mldag_reports/global_pretraining/transfer_input_timing_over_time.png b/...nal-report/mldag_reports/global_pretraining/transfer_input_timing_over_time.png
diff --git a/...r/final-report/mldag_reports/ospool_misconfigured/data_transfer_by_resource.png b/...r/final-report/mldag_reports/ospool_misconfigured/data_transfer_by_resource.png
diff --git a/.../final-report/mldag_reports/ospool_misconfigured/epochs_completed_over_time.png b/.../final-report/mldag_reports/ospool_misconfigured/epochs_completed_over_time.png
diff --git a/...airr/final-report/mldag_reports/ospool_misconfigured/epochs_trained_per_day.png b/...airr/final-report/mldag_reports/ospool_misconfigured/epochs_trained_per_day.png
diff --git a/images/nairr/final-report/mldag_reports/ospool_misconfigured/experiment_summary_report.txt b/images/nairr/final-report/mldag_reports/ospool_misconfigured/experiment_summary_report.txt
@@ -0,0 +1,75 @@
+EXPERIMENT ANALYSIS SUMMARY REPORT
+==================================================
+
+Analysis Period: 2025-04-16 13:02 to 2025-08-16 16:59
+Total Job Attempts: 936
+Unique Jobs: 470
+Total Completed Epochs: 666
+
+RESOURCE UTILIZATION:
+--------------------
+OSPOOL:
+  • Successful Jobs: 557 (100.0%)
+  • Computation Time: 2,291.9 hours
+  • Time Efficiency: 33.3%
+
+EXPANSE:
+  • Successful Jobs: 18 (100.0%)
+  • Computation Time: 295.6 hours
+  • Time Efficiency: 15.2%
+
+DELTA:
+  • Successful Jobs: 17 (100.0%)
+  • Computation Time: 404.4 hours
+  • Time Efficiency: 14.6%
+
+BRIDGES2:
+  • Successful Jobs: 8 (100.0%)
+  • Computation Time: 34.5 hours
+  • Time Efficiency: 0.7%
+
+ANVIL:
+  • Successful Jobs: 8 (100.0%)
+  • Computation Time: 202.1 hours
+  • Time Efficiency: 37.9%
+
+OVERALL METRICS:
+---------------
+Total Success Rate: 109.5%
+Total Computation Time: 3,228.5 hours
+Overall Time Efficiency: 100.0%
+
+GPU UTILIZATION:
+----------------
+Total GPU Hours: 5,318.6
+GPU Types Used: 6
+
+NVIDIA A100-PCIE-40GB: 6.0 jobs, 100.0% success rate
+NVIDIA A100-SXM4-40GB: 38.0 jobs, 100.0% success rate
+NVIDIA A100-SXM4-80GB: 73.0 jobs, 100.0% success rate
+NVIDIA A40: 108.0 jobs, 100.0% success rate
+NVIDIA RTX A5000: 354.0 jobs, 100.0% success rate
+Tesla V100-SXM2-32GB: 41.0 jobs, 100.0% success rate
+
+DATA TRANSFER:
+-------------
+Total Data Transferred: 18,386.2 GB
+
+anvil: 15.0 GB (8.0 jobs)
+bridges2: 57.0 GB (10.0 jobs)
+delta: 43.4 GB (21.0 jobs)
+expanse: 34.1 GB (21.0 jobs)
+ospool: 18,236.6 GB (637.0 jobs)
+
+INPUT FILE TRANSFER TIMING:
+---------------------------
+Jobs with Transfer Timing: 936
+Mean Transfer Duration: 8.1 minutes
+Median Transfer Duration: 1.6 minutes
+Total Transfer Time: 125.8 hours
+
+anvil: median 2.1 min, mean 8.4 min (20 jobs)
+bridges2: median 6.6 min, mean 49.0 min (65 jobs)
+delta: median 3.9 min, mean 15.2 min (49 jobs)
+expanse: median 1.4 min, mean 15.0 min (47 jobs)
+ospool: median 1.6 min, mean 3.6 min (755 jobs)
diff --git a/...airr/final-report/mldag_reports/ospool_misconfigured/gpu_capability_heatmap.png b/...airr/final-report/mldag_reports/ospool_misconfigured/gpu_capability_heatmap.png
diff --git a/...r/final-report/mldag_reports/ospool_misconfigured/gpu_distribution_analysis.png b/...r/final-report/mldag_reports/ospool_misconfigured/gpu_distribution_analysis.png
diff --git a/...irr/final-report/mldag_reports/ospool_misconfigured/gpu_efficiency_analysis.png b/...irr/final-report/mldag_reports/ospool_misconfigured/gpu_efficiency_analysis.png
diff --git a/...s/nairr/final-report/mldag_reports/ospool_misconfigured/gpu_hours_over_time.png b/...s/nairr/final-report/mldag_reports/ospool_misconfigured/gpu_hours_over_time.png
diff --git a/...l-report/mldag_reports/ospool_misconfigured/gpu_runtime_by_resource_boxplot.png b/...l-report/mldag_reports/ospool_misconfigured/gpu_runtime_by_resource_boxplot.png
diff --git a/...l-report/mldag_reports/ospool_misconfigured/resource_breakdown_bars_glidein.png b/...l-report/mldag_reports/ospool_misconfigured/resource_breakdown_bars_glidein.png
diff --git a/...port/mldag_reports/ospool_misconfigured/resource_computation_time_breakdown.png b/...port/mldag_reports/ospool_misconfigured/resource_computation_time_breakdown.png
diff --git a/...ag_reports/ospool_misconfigured/resource_computation_time_breakdown_glidein.png b/...ag_reports/ospool_misconfigured/resource_computation_time_breakdown_glidein.png
diff --git a/...rr/final-report/mldag_reports/ospool_misconfigured/resource_epoch_breakdown.png b/...rr/final-report/mldag_reports/ospool_misconfigured/resource_epoch_breakdown.png
diff --git a/...r/final-report/mldag_reports/ospool_misconfigured/resource_epochs_breakdown.png b/...r/final-report/mldag_reports/ospool_misconfigured/resource_epochs_breakdown.png
diff --git a/...report/mldag_reports/ospool_misconfigured/resource_epochs_breakdown_glidein.png b/...report/mldag_reports/ospool_misconfigured/resource_epochs_breakdown_glidein.png
diff --git a/...report/mldag_reports/ospool_misconfigured/transfer_input_timing_by_resource.png b/...report/mldag_reports/ospool_misconfigured/transfer_input_timing_by_resource.png
diff --git a/...l-report/mldag_reports/ospool_misconfigured/transfer_input_timing_over_time.png b/...l-report/mldag_reports/ospool_misconfigured/transfer_input_timing_over_time.png
diff --git a/images/nairr/final-report/plots/correlation_heatmap.png b/images/nairr/final-report/plots/correlation_heatmap.png
diff --git a/images/nairr/final-report/plots/key_metrics_combined.png b/images/nairr/final-report/plots/key_metrics_combined.png
diff --git a/images/nairr/final-report/plots/scatterplot_test pearson_total_score.png b/images/nairr/final-report/plots/scatterplot_test pearson_total_score.png
diff --git a/images/nairr/final-report/plots/scatterplot_test_loss.png b/images/nairr/final-report/plots/scatterplot_test_loss.png
diff --git a/images/nairr/final-report/plots/test_loss_distribution.png b/images/nairr/final-report/plots/test_loss_distribution.png
diff --git a/images/nairr/final-report/plots/test_loss_site_comparison.png b/images/nairr/final-report/plots/test_loss_site_comparison.png
diff --git a/images/nairr/final-report/plots/test_pearson_ref_distribution.png b/images/nairr/final-report/plots/test_pearson_ref_distribution.png
diff --git a/images/nairr/final-report/plots/test_pearson_total_score_distribution.png b/images/nairr/final-report/plots/test_pearson_total_score_distribution.png
diff --git a/images/nairr/final-report/plots/test_pearson_total_score_site_comparison.png b/images/nairr/final-report/plots/test_pearson_total_score_site_comparison.png
diff --git a/images/nairr/final-report/plots/top_100_metrics_comparison.png b/images/nairr/final-report/plots/top_100_metrics_comparison.png
diff --git a/images/nairr/final-report/plots/top_25_metrics_comparison.png b/images/nairr/final-report/plots/top_25_metrics_comparison.png
diff --git a/images/nairr/final-report/plots/top_2_metrics_comparison.png b/images/nairr/final-report/plots/top_2_metrics_comparison.png
diff --git a/images/nairr/final-report/plots/top_56_metrics_comparison.png b/images/nairr/final-report/plots/top_56_metrics_comparison.png
-Original file line number
+Diff line change
@@ Expand Up / @@ -312,4 +312,4 @@ DEPENDENCIES @@
       jekyll-sitemap
     BUNDLED WITH
-.0.1
+.0.17