de linuxmanr4, 3 años han pasado desde su publicación, escrito en Bash.
Enlace corto
http://gw.gd/dXVpg
Insertar vínculo
  1. #!/bin/bash
  2. set -e
  3.  
  4. # Usage:
  5. #   rsync_parallel.sh [--parallel=N] [rsync args...]
  6. #
  7. # Options:
  8. #   --parallel=N        Use N parallel processes for transfer. Defaults to 10.
  9. #
  10. # Notes:
  11. #   * Requires GNU Parallel
  12. #   * Use with ssh-keys. Lots of password prompts will get very annoying.
  13. #   * Does an itemize-changes first, then chunks the resulting file list and launches N parallel
  14. #     rsyncs to transfer a chunk each.
  15. #   * be a little careful with the options you pass through to rsync. Normal ones will work, you
  16. #     might want to test weird options upfront.
  17. #
  18.  
  19. if [[ "$1" == --parallel=* ]]; then
  20.         PARALLEL="${1##*=}"
  21.         shift
  22. else
  23.         PARALLEL=10
  24. fi
  25. echo "Using up to $PARALLEL processes for transfer..."
  26.  
  27. TMPDIR=$(mktemp -d)
  28. trap "rm -rf $TMPDIR" EXIT
  29.  
  30. echo "Figuring out file list..."
  31. # sorted by size (descending)
  32. rsync $@ --out-format="%l %n" --no-v --dry-run | sort -n -r > $TMPDIR/files.all
  33.  
  34. # check for nothing-to-do
  35. TOTAL_FILES=$(cat $TMPDIR/files.all | wc -l)
  36. if [ "$TOTAL_FILES" -eq "0" ]; then
  37.         echo "Nothing to transfer :)"
  38.         exit 0
  39. fi
  40.  
  41. function array_min {
  42.         # return the (index, value) of the minimum element in the array
  43.         IC=($(tr ' ' '\n' <<<$@ | cat -n | sort -k2,2nr | tail -n1))
  44.         echo $((${IC[0]} - 1)) ${IC[1]}
  45. }
  46.  
  47. echo "Calculating chunks..."
  48. # declare chunk-size array
  49. for ((I = 0 ; I < PARALLEL ; I++ )); do
  50.         CHUNKS["$I"]=0
  51. done
  52.  
  53. # add each file to the emptiest chunk, so they're as balanced by size as possible
  54. while read FSIZE FPATH; do
  55.         MIN=($(array_min ${CHUNKS[@]}))
  56.         CHUNKS["${MIN[0]}"]=$((${CHUNKS["${MIN[0]}"]} + $FSIZE))
  57.         echo $FPATH >> $TMPDIR/chunk.${MIN[0]}
  58. done < $TMPDIR/files.all
  59.  
  60. find "$TMPDIR" -type f -name "chunk.*" -printf "\n*** %p ***\n" -exec cat {} \;
  61.  
  62. echo "Starting transfers..."
  63. find "$TMPDIR" -type f -name "chunk.*" | parallel -j $PARALLEL -t --verbose --progress rsync --files-from={} $@