vfs/shell: Add more send/append methods

configure.ac

@@ -735,6 +735,8 @@ tests/src/vfs/extfs/helpers-list/Makefile
 tests/src/vfs/extfs/helpers-list/data/config.sh
 tests/src/vfs/extfs/helpers-list/misc/Makefile
 tests/src/vfs/ftpfs/Makefile
+tests/src/vfs/shell/Makefile
+tests/src/vfs/shell/helpers/Makefile
 ])
 
 AC_OUTPUT

src/filemanager/file.c

@@ -2918,6 +2918,12 @@ copy_file_file (file_op_context_t *ctx, const char *src_path, const char *dst_pa
         if (temp_status == FILE_IGNORE_ALL)
             ctx->ignore_all = TRUE;
         return_status = temp_status;
+
+        // On destination close error, DEST_FULL is not a good dst_status anymore.
+        // A remote write error in vfs/shell manifests itself as a destination close error here.
+        if (dst_status == DEST_FULL)
+            dst_status = DEST_SHORT_QUERY;
+
         break;
     }
 

src/vfs/shell/helpers/append

@@ -1,15 +1,61 @@
+case "${SHELL_HAVE_POSIX_DD-0}${SHELL_HAVE_HEAD-0}${SHELL_HAVE_PERL-0}${SHELL_HAVE_DD-0}" in
+1???|0001)
+	send_read_data() {
+		while [ "$bs" -gt "$SHELL_FILESIZE" ]; do
+			bs=`expr "$bs" / 2`
+		done
+		[ "$bs" -gt 511 ] || bs=1
+		count=`expr "$SHELL_FILESIZE" / "$bs"`
+		dd bs="$bs" count="$count" ${SHELL_HAVE_POSIX_DD+iflag=fullblock} 2>/dev/null
+	}
+	;;
+01??)	send_read_data() { head -c "$SHELL_FILESIZE"; }
+	;;
+001?)	send_read_data() {
+		perl -e "\$n = $SHELL_FILESIZE; \$bs = $bs;"'
+			while ($n) {
+				 $m = sysread(STDIN, $b, ($n < $bs) ? $n : $bs);
+				 syswrite(STDOUT, $b, $m) if ($m);
+				 $n -= $m;
+			}'
+	}
+	;;
+esac
+
+send_piece() {
+	{ send_read_data | tee -a "$FILENAME" | wc -c; } 2>&1 | while read -r _output; do
+		if [ -z "$_output" ] || [ "${_output#+}" != "${_output}" ]; then
+			:
+		elif expr "$_output" : '[[:space:]]*[0-9][0-9]*$' >/dev/null; then
+			echo "SHELL_FILESIZE=`expr $SHELL_FILESIZE - $_output`"
+		else
+			echo 'FILENAME=/dev/null'
+			[ -n "$_err" ] || _err="$_output"
+		fi
+	done
+}
+
+append_for_real() {
+	>> "$FILENAME" || { echo "### 500 Could not reopen $FILENAME for appending"; return 1; }
+	echo "### 001"
+	bs=1048576
+	_err=""
+	while [ "$SHELL_FILESIZE" -gt 0 ]; do
+		eval "`send_piece`"
+	done
+	if [ -n "$_err" ]; then
+		echo "### 500 $_err"
+	else
+		echo "### 200"
+	fi
+}
+
 FILENAME="/${SHELL_FILENAME}"
-echo "### 001"
-{
-    bss=4096
-    bsl=4095
-    if [ $SHELL_FILESIZE -lt $bss ]; then
-        bss=1;
-        bsl=0;
-    fi
-    while [ $SHELL_FILESIZE -gt 0 ]; do
-        cnt=`expr \\( $SHELL_FILESIZE + $bsl \\) / $bss`
-        n=`dd bs=$bss count=$cnt | tee -a "${FILENAME}" | wc -c`
-        SHELL_FILESIZE=`expr $SHELL_FILESIZE - $n`
-    done
-}; echo "### 200"
+
+if [ -d "$FILENAME" ]; then
+	echo "### 500 $FILENAME is a directory"
+elif [ "${SHELL_HAVE_POSIX_DD-0}${SHELL_HAVE_HEAD-0}${SHELL_HAVE_PERL-0}${SHELL_HAVE_DD-0}" = "0000" ]; then
+	echo "### 500 No supported send method"
+else
+	append_for_real
+fi

src/vfs/shell/helpers/info

@@ -7,8 +7,10 @@ export LC_TIME
 #SHELL_HAVE_LSQ         16
 #SHELL_HAVE_DATE_MDYT   32
 #SHELL_HAVE_TAIL        64
+#SHELL_HAVE_DD         128
+#SHELL_HAVE_POSIX_DD   256
 res=0
-if `echo yes| head -c 1 > /dev/null 2>&1` ; then
+if [ "`echo yes | { head -c 1 > /dev/null; cat; } 2>&1`" = es ]; then
     res=`expr $res + 1`
 fi
 if `echo 1 | sed 's/1/2/' >/dev/null 2>&1` ; then
@@ -40,5 +42,11 @@ fi
 if `echo yes| tail -c +1 - > /dev/null 2>&1` ; then
     res=`expr $res + 64`
 fi
+if dd if=/dev/null >/dev/null 2>&1; then
+    res=`expr $res + 128`
+fi
+if dd if=/dev/null iflag=fullblock >/dev/null 2>&1; then
+    res=`expr $res + 256`
+fi
 echo $res
 echo "### 200"

src/vfs/shell/helpers/send

@@ -1,16 +1,61 @@
+case "${SHELL_HAVE_POSIX_DD-0}${SHELL_HAVE_HEAD-0}${SHELL_HAVE_PERL-0}${SHELL_HAVE_DD-0}" in
+1???|0001)
+	send_read_data() {
+		while [ "$bs" -gt "$SHELL_FILESIZE" ]; do
+			bs=`expr "$bs" / 2`
+		done
+		[ "$bs" -gt 511 ] || bs=1
+		count=`expr "$SHELL_FILESIZE" / "$bs"`
+		dd bs="$bs" count="$count" ${SHELL_HAVE_POSIX_DD+iflag=fullblock} 2>/dev/null
+	}
+	;;
+01??)	send_read_data() { head -c "$SHELL_FILESIZE"; }
+	;;
+001?)	send_read_data() {
+		perl -e "\$n = $SHELL_FILESIZE; \$bs = $bs;"'
+			while ($n) {
+				 $m = sysread(STDIN, $b, ($n < $bs) ? $n : $bs);
+				 syswrite(STDOUT, $b, $m) if ($m);
+				 $n -= $m;
+			}'
+	}
+	;;
+esac
+
+send_piece() {
+	{ send_read_data | tee -a "$FILENAME" | wc -c; } 2>&1 | while read -r _output; do
+		if [ -z "$_output" ] || [ "${_output#+}" != "${_output}" ]; then
+			:
+		elif expr "$_output" : '[[:space:]]*[0-9][0-9]*$' >/dev/null; then
+			echo "SHELL_FILESIZE=`expr $SHELL_FILESIZE - $_output`"
+		else
+			echo 'FILENAME=/dev/null'
+			[ -n "$_err" ] || _err="$_output"
+		fi
+	done
+}
+
+send_for_real() {
+	> "$FILENAME" || { echo "### 500 Could not truncate $FILENAME"; return 1; }
+	echo "### 001"
+	bs=1048576
+	_err=""
+	while [ "$SHELL_FILESIZE" -gt 0 ]; do
+		eval "`send_piece`"
+	done
+	if [ -n "$_err" ]; then
+		echo "### 500 $_err"
+	else
+		echo "### 200"
+	fi
+}
+
 FILENAME="/${SHELL_FILENAME}"
-echo "### 001"
-{
-    > "${FILENAME}"
-    bss=4096
-    bsl=4095
-    if [ $SHELL_FILESIZE -lt $bss ]; then
-        bss=1;
-        bsl=0;
-    fi
-    while [ $SHELL_FILESIZE -gt 0 ]; do
-        cnt=`expr \\( $SHELL_FILESIZE + $bsl \\) / $bss`
-        n=`dd bs=$bss count=$cnt | tee -a "${FILENAME}" | wc -c`
-        SHELL_FILESIZE=`expr $SHELL_FILESIZE - $n`
-    done
-}; echo "### 200"
+
+if [ -d "$FILENAME" ]; then
+	echo "### 500 $FILENAME is a directory"
+elif [ "${SHELL_HAVE_POSIX_DD-0}${SHELL_HAVE_HEAD-0}${SHELL_HAVE_PERL-0}${SHELL_HAVE_DD-0}" = "0000" ]; then
+	echo "### 500 No supported send method"
+else
+	send_for_real
+fi

src/vfs/shell/shell.c

@@ -115,6 +115,8 @@ int shell_directory_timeout = 900;
 #define SHELL_HAVE_LSQ         16
 #define SHELL_HAVE_DATE_MDYT   32
 #define SHELL_HAVE_TAIL        64
+#define SHELL_HAVE_DD          128
+#define SHELL_HAVE_POSIX_DD    256
 
 #define SHELL_SUPER(super)     ((shell_super_t *) (super))
 #define SHELL_FILE_HANDLER(fh) ((shell_file_handler_t *) fh)
@@ -480,6 +482,12 @@ shell_set_env (int flags)
     if ((flags & SHELL_HAVE_TAIL) != 0)
         g_string_append (ret, "SHELL_HAVE_TAIL=1 export SHELL_HAVE_TAIL; ");
 
+    if ((flags & SHELL_HAVE_DD) != 0)
+        g_string_append (ret, "SHELL_HAVE_DD=1 export SHELL_HAVE_DD; ");
+
+    if ((flags & SHELL_HAVE_POSIX_DD) != 0)
+        g_string_append (ret, "SHELL_HAVE_POSIX_DD=1 export SHELL_HAVE_POSIX_DD; ");
+
     return ret;
 }
 
@@ -1686,6 +1694,28 @@ shell_fill_names (struct vfs_class *me, fill_names_f func)
 static void *
 shell_open (const vfs_path_t *vpath, int flags, mode_t mode)
 {
+    if ((flags & O_WRONLY) != 0)
+    {
+        // We need to check if the path is writable to return the error ASAP
+        const char *name;
+        char *quoted_name;
+        char *command;
+        struct vfs_s_super *super;
+        struct vfs_class *me;
+        int r;
+
+        me = VFS_CLASS (vfs_path_get_last_path_vfs (vpath));
+        name = vfs_s_get_path (vpath, &super, 0);
+        quoted_name = str_shell_escape (name);
+        command = g_strdup_printf ("if : %s /%s; then echo '### 200'; else echo '### 500'; fi\n",
+                                   ((flags & O_APPEND) != 0) ? ">>" : ">", quoted_name);
+        g_free (quoted_name);
+        r = shell_command (me, super, WAIT_REPLY, command, -1);
+        g_free (command);
+        if (r != COMPLETE)
+            ERRNOR (E_REMOTE, NULL);
+    }
+
     /*
        sorry, i've places hack here
        cause shell don't able to open files with O_EXCL flag

tests/src/vfs/Makefile.am

@@ -8,3 +8,7 @@ endif
 if ENABLE_VFS_FTP
 SUBDIRS += ftpfs
 endif
+
+if ENABLE_VFS_SHELL
+SUBDIRS += shell
+endif

tests/src/vfs/shell/Makefile.am

@@ -0,0 +1,2 @@
+
+SUBDIRS = helpers

tests/src/vfs/shell/helpers/Makefile.am

@@ -0,0 +1,6 @@
+PACKAGE_STRING = "/src/vfs/shell/helpers"
+
+# Tests to run on 'make check'
+TESTS = \
+	test_append.sh	\
+	test_send.sh

tests/src/vfs/shell/helpers/test_append.sh

@@ -0,0 +1,55 @@
+#!/bin/sh
+# shellcheck disable=SC2034,SC1091
+SELF="$0"
+# shellcheck disable=SC2164
+SELF_DIR="$(cd "${0%/*}" 2>/dev/null; pwd)"
+. "$SELF_DIR/testutil.sh"
+. "$SELF_DIR/test_send_common.sh"
+
+setup() {
+	setup_common
+	HELPER_NAME="append"
+}
+
+test_append_reports_errors() {
+	# Test appending to a write-protected file
+
+	setup_send_method "$1" || return 0
+	dir=; get_temp dir
+	echo "a" >"$dir/test.txt"
+	chmod 0555 "$dir/test.txt" || abort "chmod failed"
+
+	RUN try_send "$dir/test.txt" 5 <<EOF
+test
+EOF
+	assert_output_match '^### 500'
+}
+
+test_append_short_reads() {
+	# Test if short reads resulting from network congestion don't affect us
+
+	setup_send_method "$1" || return 0
+	dir=; get_temp dir
+	dd if=/dev/random bs=1000 count=70 of="$dir/source.rnd" 2>/dev/null
+	echo "a" >"$dir/test.rnd"
+
+	RUN try_send_with_jitter "$dir/source.rnd" "$dir/test.rnd" 70000
+	assert_output_match '^### 200'
+	dd_runs="$(if [ -f "$state_dir/dd_runs" ]; then wc -l <"$state_dir/dd_runs"; else echo 0; fi)"
+	if [ "$dd_runs" -gt 3 ]; then
+		log "NOTE: $run_name: dd had to run $dd_runs times, probably due to partial reads"
+	fi
+
+	RUN wc -c < "$dir/test.rnd"
+	assert_output_match '^[[:space:]]*70002$'
+}
+
+header "$@"
+
+# test every method that our script may choose
+for method in posix_dd head_c perl nonposix_dd; do
+	run_test test_append_reports_errors $method
+	run_test test_append_short_reads $method
+done
+
+summary

tests/src/vfs/shell/helpers/test_send.sh

@@ -0,0 +1,57 @@
+#!/bin/sh
+# shellcheck disable=SC2034,SC1091
+SELF="$0"
+# shellcheck disable=SC2164
+SELF_DIR="$(cd "${0%/*}" 2>/dev/null; pwd)"
+. "$SELF_DIR/testutil.sh"
+. "$SELF_DIR/test_send_common.sh"
+
+setup() {
+	setup_common
+	HELPER_NAME="send"
+}
+
+test_send_reports_errors() {
+	# Test writing to a write-protected directory
+
+	setup_send_method "$1" || return 0
+	dir=; get_temp dir
+	chmod 0555 "$dir" || abort "chmod failed"
+
+	RUN try_send "$dir/test.txt" 5 <<EOF
+test
+EOF
+	assert_output_match '^### 500'
+}
+
+test_send_short_reads() {
+	# Test if short reads resulting from network congestion don't affect us
+
+	setup_send_method "$1" || return 0
+	dir=; get_temp dir
+	dd if=/dev/random bs=1000 count=70 of="$dir/source.rnd" 2>/dev/null
+
+	RUN try_send_with_jitter "$dir/source.rnd" "$dir/test.rnd" 70000
+	assert_output_match '^### 200'
+	dd_runs="$(if [ -f "$state_dir/dd_runs" ]; then wc -l <"$state_dir/dd_runs"; else echo 0; fi)"
+	if [ "$dd_runs" -gt 3 ]; then
+		log "NOTE: $run_name: dd had to run $dd_runs times, probably due to partial reads"
+	fi
+
+	RUN wc -c < "$dir/test.rnd"
+	assert_output_match '^[[:space:]]*70000$'
+
+	RUN cmp "$dir/source.rnd" "$dir/test.rnd"
+	assert_success
+	assert_output ''
+}
+
+header "$@"
+
+# test every method that our script may choose
+for method in posix_dd head_c perl nonposix_dd; do
+	run_test test_send_reports_errors $method
+	run_test test_send_short_reads $method
+done
+
+summary