diff --git a/configure.ac b/configure.ac index 755e4a75e3..6f096aebd2 100644 --- a/configure.ac +++ b/configure.ac @@ -735,6 +735,8 @@ tests/src/vfs/extfs/helpers-list/Makefile tests/src/vfs/extfs/helpers-list/data/config.sh tests/src/vfs/extfs/helpers-list/misc/Makefile tests/src/vfs/ftpfs/Makefile +tests/src/vfs/shell/Makefile +tests/src/vfs/shell/helpers/Makefile ]) AC_OUTPUT diff --git a/src/filemanager/file.c b/src/filemanager/file.c index d4b08ab39c..4b4ad5fe4b 100644 --- a/src/filemanager/file.c +++ b/src/filemanager/file.c @@ -2918,6 +2918,12 @@ copy_file_file (file_op_context_t *ctx, const char *src_path, const char *dst_pa if (temp_status == FILE_IGNORE_ALL) ctx->ignore_all = TRUE; return_status = temp_status; + + // On destination close error, DEST_FULL is not a good dst_status anymore. + // A remote write error in vfs/shell manifests itself as a destination close error here. + if (dst_status == DEST_FULL) + dst_status = DEST_SHORT_QUERY; + break; } diff --git a/src/vfs/shell/helpers/append b/src/vfs/shell/helpers/append index a73d2cd3f4..1c80c91a58 100644 --- a/src/vfs/shell/helpers/append +++ b/src/vfs/shell/helpers/append @@ -1,15 +1,61 @@ +case "${SHELL_HAVE_POSIX_DD-0}${SHELL_HAVE_HEAD-0}${SHELL_HAVE_PERL-0}${SHELL_HAVE_DD-0}" in +1???|0001) + send_read_data() { + while [ "$bs" -gt "$SHELL_FILESIZE" ]; do + bs=`expr "$bs" / 2` + done + [ "$bs" -gt 511 ] || bs=1 + count=`expr "$SHELL_FILESIZE" / "$bs"` + dd bs="$bs" count="$count" ${SHELL_HAVE_POSIX_DD+iflag=fullblock} 2>/dev/null + } + ;; +01??) send_read_data() { head -c "$SHELL_FILESIZE"; } + ;; +001?) send_read_data() { + perl -e "\$n = $SHELL_FILESIZE; \$bs = $bs;"' + while ($n) { + $m = sysread(STDIN, $b, ($n < $bs) ? $n : $bs); + syswrite(STDOUT, $b, $m) if ($m); + $n -= $m; + }' + } + ;; +esac + +send_piece() { + { send_read_data | tee -a "$FILENAME" | wc -c; } 2>&1 | while read -r _output; do + if [ -z "$_output" ] || [ "${_output#+}" != "${_output}" ]; then + : + elif expr "$_output" : '[[:space:]]*[0-9][0-9]*$' >/dev/null; then + echo "SHELL_FILESIZE=`expr $SHELL_FILESIZE - $_output`" + else + echo 'FILENAME=/dev/null' + [ -n "$_err" ] || _err="$_output" + fi + done +} + +append_for_real() { + >> "$FILENAME" || { echo "### 500 Could not reopen $FILENAME for appending"; return 1; } + echo "### 001" + bs=1048576 + _err="" + while [ "$SHELL_FILESIZE" -gt 0 ]; do + eval "`send_piece`" + done + if [ -n "$_err" ]; then + echo "### 500 $_err" + else + echo "### 200" + fi +} + FILENAME="/${SHELL_FILENAME}" -echo "### 001" -{ - bss=4096 - bsl=4095 - if [ $SHELL_FILESIZE -lt $bss ]; then - bss=1; - bsl=0; - fi - while [ $SHELL_FILESIZE -gt 0 ]; do - cnt=`expr \\( $SHELL_FILESIZE + $bsl \\) / $bss` - n=`dd bs=$bss count=$cnt | tee -a "${FILENAME}" | wc -c` - SHELL_FILESIZE=`expr $SHELL_FILESIZE - $n` - done -}; echo "### 200" + +if [ -d "$FILENAME" ]; then + echo "### 500 $FILENAME is a directory" +elif [ "${SHELL_HAVE_POSIX_DD-0}${SHELL_HAVE_HEAD-0}${SHELL_HAVE_PERL-0}${SHELL_HAVE_DD-0}" = "0000" ]; then + echo "### 500 No supported send method" +else + append_for_real +fi diff --git a/src/vfs/shell/helpers/info b/src/vfs/shell/helpers/info index 053423af41..5ba7ab0670 100644 --- a/src/vfs/shell/helpers/info +++ b/src/vfs/shell/helpers/info @@ -7,8 +7,10 @@ export LC_TIME #SHELL_HAVE_LSQ 16 #SHELL_HAVE_DATE_MDYT 32 #SHELL_HAVE_TAIL 64 +#SHELL_HAVE_DD 128 +#SHELL_HAVE_POSIX_DD 256 res=0 -if `echo yes| head -c 1 > /dev/null 2>&1` ; then +if [ "`echo yes | { head -c 1 > /dev/null; cat; } 2>&1`" = es ]; then res=`expr $res + 1` fi if `echo 1 | sed 's/1/2/' >/dev/null 2>&1` ; then @@ -40,5 +42,11 @@ fi if `echo yes| tail -c +1 - > /dev/null 2>&1` ; then res=`expr $res + 64` fi +if dd if=/dev/null >/dev/null 2>&1; then + res=`expr $res + 128` +fi +if dd if=/dev/null iflag=fullblock >/dev/null 2>&1; then + res=`expr $res + 256` +fi echo $res echo "### 200" diff --git a/src/vfs/shell/helpers/send b/src/vfs/shell/helpers/send index 66c2a1c7b5..760d9413e4 100644 --- a/src/vfs/shell/helpers/send +++ b/src/vfs/shell/helpers/send @@ -1,16 +1,61 @@ +case "${SHELL_HAVE_POSIX_DD-0}${SHELL_HAVE_HEAD-0}${SHELL_HAVE_PERL-0}${SHELL_HAVE_DD-0}" in +1???|0001) + send_read_data() { + while [ "$bs" -gt "$SHELL_FILESIZE" ]; do + bs=`expr "$bs" / 2` + done + [ "$bs" -gt 511 ] || bs=1 + count=`expr "$SHELL_FILESIZE" / "$bs"` + dd bs="$bs" count="$count" ${SHELL_HAVE_POSIX_DD+iflag=fullblock} 2>/dev/null + } + ;; +01??) send_read_data() { head -c "$SHELL_FILESIZE"; } + ;; +001?) send_read_data() { + perl -e "\$n = $SHELL_FILESIZE; \$bs = $bs;"' + while ($n) { + $m = sysread(STDIN, $b, ($n < $bs) ? $n : $bs); + syswrite(STDOUT, $b, $m) if ($m); + $n -= $m; + }' + } + ;; +esac + +send_piece() { + { send_read_data | tee -a "$FILENAME" | wc -c; } 2>&1 | while read -r _output; do + if [ -z "$_output" ] || [ "${_output#+}" != "${_output}" ]; then + : + elif expr "$_output" : '[[:space:]]*[0-9][0-9]*$' >/dev/null; then + echo "SHELL_FILESIZE=`expr $SHELL_FILESIZE - $_output`" + else + echo 'FILENAME=/dev/null' + [ -n "$_err" ] || _err="$_output" + fi + done +} + +send_for_real() { + > "$FILENAME" || { echo "### 500 Could not truncate $FILENAME"; return 1; } + echo "### 001" + bs=1048576 + _err="" + while [ "$SHELL_FILESIZE" -gt 0 ]; do + eval "`send_piece`" + done + if [ -n "$_err" ]; then + echo "### 500 $_err" + else + echo "### 200" + fi +} + FILENAME="/${SHELL_FILENAME}" -echo "### 001" -{ - > "${FILENAME}" - bss=4096 - bsl=4095 - if [ $SHELL_FILESIZE -lt $bss ]; then - bss=1; - bsl=0; - fi - while [ $SHELL_FILESIZE -gt 0 ]; do - cnt=`expr \\( $SHELL_FILESIZE + $bsl \\) / $bss` - n=`dd bs=$bss count=$cnt | tee -a "${FILENAME}" | wc -c` - SHELL_FILESIZE=`expr $SHELL_FILESIZE - $n` - done -}; echo "### 200" + +if [ -d "$FILENAME" ]; then + echo "### 500 $FILENAME is a directory" +elif [ "${SHELL_HAVE_POSIX_DD-0}${SHELL_HAVE_HEAD-0}${SHELL_HAVE_PERL-0}${SHELL_HAVE_DD-0}" = "0000" ]; then + echo "### 500 No supported send method" +else + send_for_real +fi diff --git a/src/vfs/shell/shell.c b/src/vfs/shell/shell.c index 7fbcd4d565..e39351336b 100644 --- a/src/vfs/shell/shell.c +++ b/src/vfs/shell/shell.c @@ -115,6 +115,8 @@ int shell_directory_timeout = 900; #define SHELL_HAVE_LSQ 16 #define SHELL_HAVE_DATE_MDYT 32 #define SHELL_HAVE_TAIL 64 +#define SHELL_HAVE_DD 128 +#define SHELL_HAVE_POSIX_DD 256 #define SHELL_SUPER(super) ((shell_super_t *) (super)) #define SHELL_FILE_HANDLER(fh) ((shell_file_handler_t *) fh) @@ -480,6 +482,12 @@ shell_set_env (int flags) if ((flags & SHELL_HAVE_TAIL) != 0) g_string_append (ret, "SHELL_HAVE_TAIL=1 export SHELL_HAVE_TAIL; "); + if ((flags & SHELL_HAVE_DD) != 0) + g_string_append (ret, "SHELL_HAVE_DD=1 export SHELL_HAVE_DD; "); + + if ((flags & SHELL_HAVE_POSIX_DD) != 0) + g_string_append (ret, "SHELL_HAVE_POSIX_DD=1 export SHELL_HAVE_POSIX_DD; "); + return ret; } @@ -1686,6 +1694,28 @@ shell_fill_names (struct vfs_class *me, fill_names_f func) static void * shell_open (const vfs_path_t *vpath, int flags, mode_t mode) { + if ((flags & O_WRONLY) != 0) + { + // We need to check if the path is writable to return the error ASAP + const char *name; + char *quoted_name; + char *command; + struct vfs_s_super *super; + struct vfs_class *me; + int r; + + me = VFS_CLASS (vfs_path_get_last_path_vfs (vpath)); + name = vfs_s_get_path (vpath, &super, 0); + quoted_name = str_shell_escape (name); + command = g_strdup_printf ("if : %s /%s; then echo '### 200'; else echo '### 500'; fi\n", + ((flags & O_APPEND) != 0) ? ">>" : ">", quoted_name); + g_free (quoted_name); + r = shell_command (me, super, WAIT_REPLY, command, -1); + g_free (command); + if (r != COMPLETE) + ERRNOR (E_REMOTE, NULL); + } + /* sorry, i've places hack here cause shell don't able to open files with O_EXCL flag diff --git a/tests/src/vfs/Makefile.am b/tests/src/vfs/Makefile.am index 8530f99bf8..4840cda49b 100644 --- a/tests/src/vfs/Makefile.am +++ b/tests/src/vfs/Makefile.am @@ -8,3 +8,7 @@ endif if ENABLE_VFS_FTP SUBDIRS += ftpfs endif + +if ENABLE_VFS_SHELL +SUBDIRS += shell +endif diff --git a/tests/src/vfs/shell/Makefile.am b/tests/src/vfs/shell/Makefile.am new file mode 100644 index 0000000000..0d4ff78017 --- /dev/null +++ b/tests/src/vfs/shell/Makefile.am @@ -0,0 +1,2 @@ + +SUBDIRS = helpers diff --git a/tests/src/vfs/shell/helpers/Makefile.am b/tests/src/vfs/shell/helpers/Makefile.am new file mode 100644 index 0000000000..880f04ca53 --- /dev/null +++ b/tests/src/vfs/shell/helpers/Makefile.am @@ -0,0 +1,6 @@ +PACKAGE_STRING = "/src/vfs/shell/helpers" + +# Tests to run on 'make check' +TESTS = \ + test_append.sh \ + test_send.sh diff --git a/tests/src/vfs/shell/helpers/test_append.sh b/tests/src/vfs/shell/helpers/test_append.sh new file mode 100755 index 0000000000..443b79477d --- /dev/null +++ b/tests/src/vfs/shell/helpers/test_append.sh @@ -0,0 +1,55 @@ +#!/bin/sh +# shellcheck disable=SC2034,SC1091 +SELF="$0" +# shellcheck disable=SC2164 +SELF_DIR="$(cd "${0%/*}" 2>/dev/null; pwd)" +. "$SELF_DIR/testutil.sh" +. "$SELF_DIR/test_send_common.sh" + +setup() { + setup_common + HELPER_NAME="append" +} + +test_append_reports_errors() { + # Test appending to a write-protected file + + setup_send_method "$1" || return 0 + dir=; get_temp dir + echo "a" >"$dir/test.txt" + chmod 0555 "$dir/test.txt" || abort "chmod failed" + + RUN try_send "$dir/test.txt" 5 </dev/null + echo "a" >"$dir/test.rnd" + + RUN try_send_with_jitter "$dir/source.rnd" "$dir/test.rnd" 70000 + assert_output_match '^### 200' + dd_runs="$(if [ -f "$state_dir/dd_runs" ]; then wc -l <"$state_dir/dd_runs"; else echo 0; fi)" + if [ "$dd_runs" -gt 3 ]; then + log "NOTE: $run_name: dd had to run $dd_runs times, probably due to partial reads" + fi + + RUN wc -c < "$dir/test.rnd" + assert_output_match '^[[:space:]]*70002$' +} + +header "$@" + +# test every method that our script may choose +for method in posix_dd head_c perl nonposix_dd; do + run_test test_append_reports_errors $method + run_test test_append_short_reads $method +done + +summary diff --git a/tests/src/vfs/shell/helpers/test_send.sh b/tests/src/vfs/shell/helpers/test_send.sh new file mode 100755 index 0000000000..1ed5eea73d --- /dev/null +++ b/tests/src/vfs/shell/helpers/test_send.sh @@ -0,0 +1,57 @@ +#!/bin/sh +# shellcheck disable=SC2034,SC1091 +SELF="$0" +# shellcheck disable=SC2164 +SELF_DIR="$(cd "${0%/*}" 2>/dev/null; pwd)" +. "$SELF_DIR/testutil.sh" +. "$SELF_DIR/test_send_common.sh" + +setup() { + setup_common + HELPER_NAME="send" +} + +test_send_reports_errors() { + # Test writing to a write-protected directory + + setup_send_method "$1" || return 0 + dir=; get_temp dir + chmod 0555 "$dir" || abort "chmod failed" + + RUN try_send "$dir/test.txt" 5 </dev/null + + RUN try_send_with_jitter "$dir/source.rnd" "$dir/test.rnd" 70000 + assert_output_match '^### 200' + dd_runs="$(if [ -f "$state_dir/dd_runs" ]; then wc -l <"$state_dir/dd_runs"; else echo 0; fi)" + if [ "$dd_runs" -gt 3 ]; then + log "NOTE: $run_name: dd had to run $dd_runs times, probably due to partial reads" + fi + + RUN wc -c < "$dir/test.rnd" + assert_output_match '^[[:space:]]*70000$' + + RUN cmp "$dir/source.rnd" "$dir/test.rnd" + assert_success + assert_output '' +} + +header "$@" + +# test every method that our script may choose +for method in posix_dd head_c perl nonposix_dd; do + run_test test_send_reports_errors $method + run_test test_send_short_reads $method +done + +summary diff --git a/tests/src/vfs/shell/helpers/test_send_common.sh b/tests/src/vfs/shell/helpers/test_send_common.sh new file mode 100644 index 0000000000..611f06343f --- /dev/null +++ b/tests/src/vfs/shell/helpers/test_send_common.sh @@ -0,0 +1,95 @@ +# shellcheck shell=sh +export SHELL_FOR_TESTING="${SHELL_FOR_TESTING:-/bin/sh}" + +setup_common() { + HELPERS="$SELF_DIR/../../../../../src/vfs/shell/helpers" + bin_dir=; get_temp bin_dir + mock_bin echo expr ls tee wc +} + +which() { _bin="$(type "$1")" && { _bin="${_bin##* }"; _bin="${_bin#\(}"; _bin="${_bin%\)}"; printf '%s' "$_bin"; } } + +mock_bin() { + for _arg; do + ln -s "$(which "$_arg")" "$bin_dir/$_arg" + done +} + +setup_send_method() { + case "$1" in + posix_dd) + if dd if=/dev/zero bs=1 count=1 iflag=fullblock >/dev/null 2>&1; then + cat >"$bin_dir/dd" <<-EOF_POSIX_DD + #!/bin/sh + echo "dd $*" >>"$state_dir/dd_runs" + exec $(which dd) "\$@" + EOF_POSIX_DD + chmod 0755 "$bin_dir/dd" + HELPER_ENV="SHELL_HAVE_POSIX_DD=1" + else + SKIP "POSIX dd not available" + return 1 + fi + ;; + head_c) if [ "$(echo abcd | { head -c 2 >/dev/null 2>&1; cat; })" = cd ]; then + mock_bin head + HELPER_ENV="SHELL_HAVE_HEAD=1" + else + SKIP "head -c not available" + return 1 + fi + ;; + perl) if type perl >/dev/null 2>&1; then + mock_bin perl + HELPER_ENV="SHELL_HAVE_PERL=1" + else + SKIP "Perl not available" + return 1 + fi + ;; + nonposix_dd) + if type dd >/dev/null 2>&1; then + cat >"$bin_dir/dd" <<-EOF_NONPOSIX_DD + #!/bin/sh + case "\$*" in *fullblock*) exit 1 ;; esac + echo "dd $*" >>"$state_dir/dd_runs" + exec $(which dd) "\$@" + EOF_NONPOSIX_DD + chmod 0755 "$bin_dir/dd" + HELPER_ENV="SHELL_HAVE_DD=1" + else + SKIP "dd not available" + return 1 + fi + ;; + esac +} + +rand() { + read -r _ _t _ </dev/null + _rest=$((_rest - _size)) + done < "$1" | try_send "$2" "$3" +} diff --git a/tests/src/vfs/shell/helpers/testutil.sh b/tests/src/vfs/shell/helpers/testutil.sh new file mode 100644 index 0000000000..9e58ab27dd --- /dev/null +++ b/tests/src/vfs/shell/helpers/testutil.sh @@ -0,0 +1,61 @@ +# shellcheck shell=sh disable=SC2006,SC1091 +set -eu + +# shellcheck disable=SC2034 +SELF_DIR=`cd "${SELF%/*}" && pwd` +temp_paths= +trap '${temp_paths:+rm -fr $temp_paths}' INT TERM EXIT +trap 'exit 124' ALRM + +log() { printf '%s\n' "$*" 1>&2; } +abort() { log "$@"; exit 1; } +quote() { printf %s\\n "$1" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; } +set_var() { eval "$1=\"\$2\""; } + +type timeout >/dev/null 2>&1 || timeout() { perl -e 'alarm shift; exec @ARGV' "$@"; } +[ ! -x /usr/xpg4/bin/grep ] || grep() { /usr/xpg4/bin/grep "$@"; } + +run_test() { + get_temp state_dir + # shellcheck disable=SC2086 + timeout 4 "$SELF" $set_x run_inner "$state_dir" "$@" || { + [ $? != 124 ] || log "FAIL: $*: the test has timed out, output: $(tail -1000 "$state_dir/output")" + n_failed=$(( n_failed + 1 )) + } +} + +run_inner() { + state_dir="$1"; shift + run_name="$*" + run_result= + setup + "$@" + [ -z "${failed:-}" ] +} + +get_temp() { + _temp_path="$(mktemp -d)" || abort "mktemp failed" + temp_paths="${temp_paths:+"$temp_paths "}$_temp_path" + set_var "$1" "$_temp_path" +} + +RUN() { set +e; "$@" 2>&1 | tee "$state_dir/output" >/dev/null; run_result=$?; set -e; run_cmd="$(quote "$*")"; } +FAIL() { log "FAIL: $run_name Command: ${run_cmd:-}; $1"; failed=true; } +SKIP() { log "SKIP: $run_name: $1"; } +assert_failure() { [ "$run_result" != 0 ] || FAIL "Expected: failure, actual: success"; } +assert_success() { [ "$run_result" = 0 ] || FAIL "Expected: success, actual: exit code $run_result"; } +assert_output() { diff_output=`printf '%s' "$1" | diff -u - "$state_dir/output" | sed -n '3,$p'` || FAIL "Expected output mismatch: +$diff_output"; } +assert_output_match() { grep -q "$@" "$state_dir/output" || FAIL "Output does not match '$*', actual: $(tail -50 "$state_dir/output")"; } +assert_output_nomatch() { grep -q "$@" "$state_dir/output" || return 0; FAIL "Output matches '$*': $(grep -m1 "$@" "$state_dir/output")"; } + +header() { + set_x=; [ "${1:-}" != "-x" ] || { set_x="$1"; shift; } + n_failed=0 + [ $# = 0 ] || { "$@"; exit $?; } +} + +summary() { + printf '%d failed tests\n' "$n_failed" + [ "$n_failed" = 0 ] +}