FIREFLY-2064: Failed to save large downloads

src/firefly/java/edu/caltech/ipac/firefly/server/RequestAgent.java

@@ -12,9 +12,12 @@
 import jakarta.servlet.http.HttpServletResponse;
 import java.io.File;
 import java.io.IOException;
+import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static edu.caltech.ipac.util.StringUtils.applyIfNotEmpty;
@@ -158,6 +161,8 @@ protected Map<String, Cookie> extractCookies() {
 
     public static final class HTTP extends RequestAgent {
         private static final String AUTH_KEY = "JOSSO_SESSIONID";
+        // headers that may be passed as query parameters instead, e.g. when downloading via form submit, which cannot set headers
+        private static final List<String> QUERY_PARAM_HEADERS = List.of("FF-connID", "FF-channel");
         private static final Logger.LoggerImpl LOG = Logger.getLogger();
         private final HashMap<String, String> headers = new HashMap<>();      // key stored as lowercase;
         private final HashMap<String, Cookie> cookies = new HashMap<>();
@@ -172,6 +177,11 @@ public HTTP(HttpServletRequest request, HttpServletResponse response) {
             Collections.list(request.getHeaderNames()).forEach(h -> {
                 headers.put(h.toLowerCase(), request.getHeader(h));
             });
+            QUERY_PARAM_HEADERS.forEach(h -> {
+                if (StringUtils.isEmpty(headers.get(h.toLowerCase()))) {
+                    applyIfNotEmpty(getQueryParam(request, h), v -> headers.put(h.toLowerCase(), v));
+                }
+            });
             applyIfNotEmpty(request.getCookies(), v -> {
                 Arrays.stream(v).forEach(c -> cookies.put(c.getName(), c));
             });
@@ -242,6 +252,21 @@ public void sendCookie(Cookie cookie) {
             }
         }
 
+        /**
+         * Read a parameter from the query string only.  Unlike request.getParameter, this does not consume the body of a POST request.
+         */
+        private static String getQueryParam(HttpServletRequest request, String name) {
+            String qs = request.getQueryString();
+            if (StringUtils.isEmpty(qs)) return null;
+            for (String kv : qs.split("&")) {
+                String[] parts = kv.split("=", 2);
+                if (URLDecoder.decode(parts[0], StandardCharsets.UTF_8).equals(name)) {
+                    return parts.length > 1 ? URLDecoder.decode(parts[1], StandardCharsets.UTF_8) : "";
+                }
+            }
+            return null;
+        }
+
         @Override
         public String getHeader(String name, String def) {
             String retval = name == null ? null : headers.get(name.toLowerCase());

src/firefly/java/edu/caltech/ipac/firefly/server/servlets/AnyFileDownload.java

@@ -13,9 +13,11 @@
 import edu.caltech.ipac.util.cache.Cache;
 import edu.caltech.ipac.util.cache.CacheManager;
 import edu.caltech.ipac.util.download.FailedRequestException;
+import edu.caltech.ipac.util.download.URLDownload;
 import edu.caltech.ipac.util.download.UriRef;
 import edu.caltech.ipac.util.download.UriRefParams;
 import edu.caltech.ipac.visualize.net.URLParms;
+import jakarta.servlet.http.Cookie;
 import jakarta.servlet.http.HttpServletRequest;
 import jakarta.servlet.http.HttpServletResponse;
 
@@ -43,6 +45,7 @@ public class AnyFileDownload extends BaseHttpServlet {
     public static final String FILE_PARAM  = "file"; // required for other request
     public static final String RETURN_PARAM= "return"; // a name for the file, if empty or USE_SERVER_NAME the use file name on server
     public static final String LOG_PARAM   = "log"; // if true, log status
+    public static final String DOWNLOAD_TOKEN = "ffDownloadToken"; // parameter sent by the client to be notified when the download starts
 
 
     public static final String USE_SERVER_NAME = "USE_SERVER_NAME";
@@ -52,6 +55,7 @@ public class AnyFileDownload extends BaseHttpServlet {
 
     private static final String BASE_URL = "servlet/Download?"+ LOG_PARAM +"=true&" + FILE_PARAM +"=";
     private static final String RET_FILE = "&"+RETURN_PARAM+"=";
+    private static final String DOWNLOAD_COOKIE_PREFIX = "ffdl_";
 
     public static String getDownloadURL(File file, String suggestedFileName) {
         return getDownloadURL(file, suggestedFileName, ServerContext.getRequestOwner().getBaseUrl());
@@ -206,6 +210,15 @@ private void handleExternalURLRequest(HttpServletRequest req, HttpServletRespons
     }
 
 
+    /** Let the client know the download has started by setting a short-lived cookie named after its download token. */
+    public static void sendDownloadStartedCookie(String token, HttpServletResponse res) {
+        if (isEmpty(token) || !token.matches("[A-Za-z0-9_-]{1,64}")) return;
+        Cookie cookie = new Cookie(DOWNLOAD_COOKIE_PREFIX + token, "1");
+        cookie.setPath("/");
+        cookie.setMaxAge(60);
+        res.addCookie(cookie);
+    }
+
     private void sendFileToClient(HttpServletRequest req, HttpServletResponse res, File f, String local) throws IOException {
         SrvParam sp= new SrvParam(req.getParameterMap());
         boolean log= sp.getOptionalBoolean(LOG_PARAM,false);
@@ -218,8 +231,9 @@ private void sendFileToClient(HttpServletRequest req, HttpServletResponse res, F
         String retFileStr= (local!=null) ? local : f.getName();
         if (retFileStr.equals(USE_SERVER_NAME)) retFileStr= f.getName();
         if (!isEmpty(retFileStr)) {
-            res.addHeader("Content-Disposition", "attachment; filename=" + retFileStr);
+            res.addHeader("Content-Disposition", URLDownload.makeContentDisposition(retFileStr));
         }
+        sendDownloadStartedCookie(sp.getOptional(DOWNLOAD_TOKEN), res);
 
 
 

src/firefly/java/edu/caltech/ipac/firefly/server/servlets/CommandService.java

@@ -8,6 +8,7 @@
 import static edu.caltech.ipac.util.FormatUtil.Format.*;
 import edu.caltech.ipac.firefly.server.ServerContext;
 import edu.caltech.ipac.firefly.server.SrvParam;
+import edu.caltech.ipac.firefly.server.filters.CorsFilter;
 import edu.caltech.ipac.firefly.server.util.Logger;
 import org.json.simple.JSONObject;
 
@@ -42,6 +43,7 @@ public static void sendError(HttpServletRequest req, HttpServletResponse res, Ma
         rval.put("success", false);
         rval.put("error", error);
         String msg = rval.toJSONString();
+        resetForError(req, res);
         res.setStatus(statusCode);
         res.setContentLength(msg.length());
         res.setContentType(JSON.mime());
@@ -54,6 +56,15 @@ public static void sendError(HttpServletRequest req, HttpServletResponse res, Ma
 
     }
 
+    /**
+     * If a download fails before sending data, reset the response to prevent downloading the error.
+     */
+    private static void resetForError(HttpServletRequest req, HttpServletResponse res) {
+        if (res.isCommitted() || !res.containsHeader("Content-Disposition")) return;
+        res.reset();
+        CorsFilter.enableCors(req, res);
+    }
+
     protected void processRequest(HttpServletRequest req, HttpServletResponse res) throws Exception {
         try {
             long start = System.currentTimeMillis();

src/firefly/java/edu/caltech/ipac/firefly/server/servlets/HttpServCommands.java

@@ -17,6 +17,7 @@
 import edu.caltech.ipac.util.FormatUtil;
 import edu.caltech.ipac.util.StringUtils;
 import edu.caltech.ipac.table.TableUtil;
+import edu.caltech.ipac.util.download.URLDownload;
 import org.json.simple.JSONObject;
 
 import jakarta.servlet.http.HttpServletRequest;
@@ -85,7 +86,8 @@ public void processRequest(HttpServletRequest req, HttpServletResponse res, SrvP
                 return;
             }
 
-            res.setHeader("Content-Disposition", "attachment; filename=" + fileName + fileNameExt);
+            res.setHeader("Content-Disposition", URLDownload.makeContentDisposition(fileName + fileNameExt));
+            AnyFileDownload.sendDownloadStartedCookie(sp.getOptional(AnyFileDownload.DOWNLOAD_TOKEN), res);
             FileInfo fi = am.save(res.getOutputStream(), request, tblFormat, mode);
             if (fi != null) {
                 long length = fi.getSizeInBytes(); // if written from the db, the length is 0

src/firefly/java/edu/caltech/ipac/util/download/URLDownload.java

@@ -36,8 +36,12 @@
 import java.net.URI;
 import java.net.URL;
 import java.net.URLConnection;
+import java.net.URLDecoder;
+import java.net.URLEncoder;
 import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.security.KeyManagementException;
 import java.security.NoSuchAlgorithmException;
 import java.security.cert.X509Certificate;
@@ -49,6 +53,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.InflaterInputStream;
 
@@ -174,17 +180,42 @@ private static FileInfo exceptionToFileInfo(Exception e) {
         return new FileInfo(codeFromException(e),ResponseMessage.getNetworkCallFailureMessage(e));
     }
 
+    private static final Pattern DISP_EXT_FILENAME = Pattern.compile("filename\\*\\s*=\\s*([^']*)'[^']*'([^;\\s]+)", Pattern.CASE_INSENSITIVE);
+    private static final Pattern DISP_FILENAME = Pattern.compile("filename\\s*=\\s*(?:\"([^\"]*)\"|([^;]+))", Pattern.CASE_INSENSITIVE);
+
+    /**
+     * Parse the file name from a Content-Disposition header value.
+     * Handles quoted and unquoted filename, and the RFC 6266 filename* extended form, which takes precedence.
+     * @param disposition the Content-Disposition header value
+     * @return a sanitized file name, or null if none is found
+     */
     public static String getSuggestedFileName(String disposition) {
         if (disposition == null) return null;
-        String[] strs = disposition.split(";");
-        if (strs.length != 2) return null;
-        String[] fname = strs[1].split("=");
-        if (fname[0].toLowerCase().contains("filename")) {
-            return sanitizeFilename(fname[1]);
+        Matcher m = DISP_EXT_FILENAME.matcher(disposition);
+        if (m.find()) {
+            try {
+                Charset cs = isEmpty(m.group(1)) ? StandardCharsets.UTF_8 : Charset.forName(m.group(1).trim());
+                return sanitizeFilename(URLDecoder.decode(m.group(2).replace("+", "%2B"), cs));
+            } catch (Exception ignored) {}   // bad encoding; fall back to filename
         }
+        m = DISP_FILENAME.matcher(disposition);
+        if (m.find()) return sanitizeFilename(m.group(1) != null ? m.group(1) : m.group(2));
         return null;
     }
 
+    /**
+     * Create a Content-Disposition header value for sending the given file name as an attachment.
+     * Includes a quoted ASCII-only filename for older clients, and filename* (RFC 6266) to preserve the full name.
+     * @param fileName the file name the client should save as
+     * @return the header value
+     */
+    public static String makeContentDisposition(String fileName) {
+        String name = fileName == null ? "" : fileName.replaceAll("[\\p{Cntrl}/\\\\]", "_");     // no control chars or path separators
+        String asciiName = name.replaceAll("[^\\x20-\\x7E]|\"", "_");
+        String encoded = URLEncoder.encode(name, StandardCharsets.UTF_8).replace("+", "%20").replace("*", "%2A");
+        return "attachment; filename=\"" + asciiName + "\"; filename*=UTF-8''" + encoded;
+    }
+
     public static String getFileNameFromUrl(URL url) {
         if (url == null) return null;
         String urlPath = url.getPath();

src/firefly/js/core/background/JobMonitor.jsx

@@ -523,10 +523,8 @@ function doDownload(job, index) {
         return;
     }
     dispatchBgJobInfo( updateSet(job, ['downloadState',index], 'WORKING') );
-    download(url).then( () => {
-        dispatchBgJobInfo( updateSet(job, ['downloadState',index], 'DONE') );
-    }).catch(() => {
-        dispatchBgJobInfo( updateSet(job, ['downloadState',index], 'FAIL') );
+    download(url).then( (started) => {
+        dispatchBgJobInfo( updateSet(job, ['downloadState',index], started ? 'DONE' : 'FAIL') );
     });
 }
 

src/firefly/js/tables/ui/TableSave.jsx

@@ -20,17 +20,19 @@ import {DownloadOptionsDialog, fileNameValidator, getTypeData,
 import {WS_SERVER_PARAM, isWsFolder, isValidWSFolder,
         getWorkspacePath, dispatchWorkspaceUpdate} from  '../../visualize/WorkspaceCntlr.js';
 import {ServerParams} from '../../data/ServerParams.js';
-import {INFO_POPUP} from '../../ui/PopupUtil.jsx';
+import {INFO_POPUP, showInfoPopup} from '../../ui/PopupUtil.jsx';
 import FieldGroupCntlr from '../../fieldGroup/FieldGroupCntlr.js';
 import {getFieldVal} from '../../fieldGroup/FieldGroupUtils.js';
 import {getWorkspaceConfig} from '../../visualize/WorkspaceCntlr.js';
 import {ListBoxInputField} from '../../ui/ListBoxInputField.jsx';
 import {download, makeDefaultDownloadFileName} from '../../util/fetch';
-import {getCmdSrvSyncURL} from '../../util/WebUtil';
+import {callWhileAwaiting, getCmdSrvSyncURL} from '../../util/WebUtil';
 import {RadioGroupInputField} from 'firefly/ui/RadioGroupInputField.jsx';
 import {useStoreConnector} from 'firefly/ui/SimpleComponent.jsx';
 import {findTableCenterColumns} from 'firefly/voAnalyzer/TableAnalysis';
 import {Stacker} from 'firefly/ui/Stacker.jsx';
+import {IfWorkingMaskById} from 'firefly/ui/panel/MaskPanel.jsx';
+import {dispatchAddWorkingTask} from 'firefly/core/AppDataCntlr.js';
 
 const fKeyDef = {
     fileName: {fKey: 'fileName', label: 'File name'},
@@ -79,6 +81,8 @@ const defValues = {
 };
 
 const tblDownloadGroupKey = 'TABLE_DOWNLOAD_FORM';
+const TBL_SAVE_MASK_ID = 'TableSavePanel';
+const WORKING_DELAY = 1000;         // ms before showing the working mask
 
 export function showTableDownloadDialog({tbl_id, tbl_ui_id}) {
     return () => {
@@ -136,7 +140,7 @@ function TableSavePanel({tbl_id, tbl_ui_id, onComplete}) {
     const sizing = wsSelected ? {height:'60vh', minHeight:'28em', resize:'both'} :
                    isWs ? {height:'20em'} : {height:'19em'};
     return (
-        <Stack spacing={1} p={1} overflow='hidden' minWidth='40em' sx={sizing}>
+        <Stack spacing={1} p={1} overflow='hidden' minWidth='40em' position='relative' sx={sizing}>
             <FieldGroup groupKey={tblDownloadGroupKey} reducerFunc={TableDLReducer(tbl_id)}
                         sx={{display:'flex', overflow:'hidden', flexGrow:1}}>
                 <Stack spacing={1} flexGrow={1}>
@@ -162,6 +166,7 @@ function TableSavePanel({tbl_id, tbl_ui_id, onComplete}) {
                     text={'Save'}/>
                 <Button onClick={() => onComplete?.()}>Cancel</Button>
             </Stacker>
+            <IfWorkingMaskById id={TBL_SAVE_MASK_ID} message='Preparing file for download...'/>
         </Stack>
     );
 }
@@ -257,32 +262,40 @@ function resultSuccess(tbl_id, tbl_ui_id, onComplete, cenCols) {
             return Object.assign(params, {file_format : fileFormat, mode});
         };
 
+        // resolves to true when the file is on its way; false if it failed and the error was shown
         const downloadFile = (urlOrOp) => {
             if (isWorkspace()) {
                 doDownloadWorkspace(getCmdSrvSyncURL(), {params: urlOrOp});
+                return true;
             } else {
-                download(urlOrOp);
+                return download(urlOrOp);
             }
-
-            onComplete?.();
         };
 
-        const {origTableModel} = getTblById(tbl_id) || {};
-        if (origTableModel) {
-            getAsyncTableSourceUrl(tbl_ui_id, getOtherParams(fileName)).then((urlOrOp) => {
-                downloadFile(urlOrOp);
-            });
-        } else {
-            let urlOrOp;
-            if (tbl_ui_id) {
-                urlOrOp= getTableSourceUrl(tbl_ui_id, getOtherParams(fileName));
-            }
-            else {
+        const getUrlOrOp = async () => {
+            const {origTableModel} = getTblById(tbl_id) || {};
+            if (origTableModel) {
+                return getAsyncTableSourceUrl(tbl_ui_id, getOtherParams(fileName));
+            } else if (tbl_ui_id) {
+                return getTableSourceUrl(tbl_ui_id, getOtherParams(fileName));
+            } else {
                 const table= getTblById(tbl_id);
-                urlOrOp = makeTableSourceUrl(table.tableData.columns, table.request, getOtherParams(fileName));
+                return makeTableSourceUrl(table.tableData.columns, table.request, getOtherParams(fileName));
             }
-            downloadFile(urlOrOp);
+        };
+
+        const saving = getUrlOrOp()
+            .then(downloadFile)
+            .catch((e) => {
+                showInfoPopup(e?.message || 'Unable to save the table', 'Save table');
+                return false;
+            });
+
+        // if the download has not started within WORKING_DELAY, mask the dialog until it does, then close it.  on error, keep it open.
+        if (!isWorkspace()) {
+            callWhileAwaiting(saving, (p) => dispatchAddWorkingTask(TBL_SAVE_MASK_ID, p, 'Preparing file for download...'), WORKING_DELAY);
         }
+        saving.then((ok) => ok && onComplete?.());
     };
 }