rclone upload/download split/restore large file
mouse 344 · person cloud · link
Last update
2019-08-02
2019
08-02
«upload/backup a big file to your online drive with auto split/concatenation»
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
#!/usr/bin/env ruby

require 'shellwords'

RCLONE_OPTS = ENV['RCLONE_OPTS'].to_s

def print_usage_and_exit
  progr = File.basename __FILE__
  puts "USO: #{progr} <service_name> <ul|dl|rm|ls> <src_file|-> [<dst_file|->] [chunk_size_MB]"
  puts "  splits/concatenate and upload/download a file via rclone"
  puts "  eg: tar -cf - / | pigz | #{progr} gdrive ul - /backup/vps/root.tgz"
  puts "  eg: #{progr} gdrive dl /backup/vps/root.tgz - | tar -xzf - -C /"
  puts "  eg: #{progr} gdrive ul homes.tgz /backup/vps/homes.tgz"
  puts "  eg: #{progr} gdrive dl /backup/vps/homes.tgz homes.tgz"
  puts "  eg: #{progr} gdrive rm /backup/vps/homes.tgz"
  puts "  eg: #{progr} gdrive ls /backup/vps"
  exit 1
end

# temp files cleanup on exit
$temp_files = []
def remove_temp_files; $temp_files.each{|f| File.unlink(f) rescue nil }; end
Signal.trap('INT' ){ remove_temp_files; exit }
at_exit{ remove_temp_files }

# sanitize parameters
srv_name, action, src_file, dst_file, chunk_size = ARGV
srv_name   = srv_name.to_s.strip
src_file   = src_file.to_s.strip
dst_file   = dst_file.to_s.strip
chunk_size = 1024 * 1024 * (chunk_size.to_i > 0 ? chunk_size.to_i : 100) # default 100MB

case action
when 'rm'
  print_usage_and_exit if ARGV.size < 3

  puts "deleting files..."
  system %Q|rclone #{RCLONE_OPTS} --include #{src_file.shellescape}.rc\\* delete #{srv_name}:|
when 'ls'
  print_usage_and_exit if ARGV.size < 3

  puts "listing files..."
  files = `rclone #{RCLONE_OPTS} lsf #{srv_name}:#{src_file.shellescape}`.split("\n")
  puts files.map{|i| i =~ /\.rc[0-9]{4}/ ? "*#{i[0..-8]}" : " #{i}"}.sort.uniq
when 'ul'
  print_usage_and_exit if ARGV.size < 4

  puts "deleting any eventual old files..."
  system %Q|rclone #{RCLONE_OPTS} --include #{dst_file.shellescape}.rc\\* delete #{srv_name}:|

  puts "split and upload of [#{src_file}]:"
  fd_src = src_file == '-' ? STDIN : File.open(src_file)
  chunk_idx = -1
  until fd_src.eof?
    remote_name = "#{dst_file}.rc#{"%04d" % (chunk_idx+=1)}"
    tmp_name    = `mktemp -u /tmp/rclone-bkup.XXXXXXXXXX`.strip
    $temp_files << tmp_name
    File.open(tmp_name, 'wb'){|f| f << fd_src.read(chunk_size) }
    ObjectSpace.garbage_collect
    puts "  #{tmp_name} => #{remote_name}"
    system %Q|rclone #{RCLONE_OPTS} copyto  #{tmp_name.shellescape}  #{srv_name}:#{remote_name.shellescape}|
    File.unlink tmp_name
  end
  fd_src.close
when 'dl'
  print_usage_and_exit if ARGV.size < 4

  remote_names = `rclone #{RCLONE_OPTS} --include #{src_file.shellescape}.rc\\* lsf #{srv_name}: -R --files-only`.split("\n").sort
  if remote_names.size == 0
    puts "no file found"
    exit 1
  end

  STDERR.puts "download and restore to [#{dst_file}]:"
  fd_dst = dst_file == '-' ? STDOUT : File.open(dst_file, 'wb')
  remote_names.each do |remote_name|
    tmp_name = `mktemp -u /tmp/rclone-bkup.XXXXXXXXXX`.strip
    $temp_files << tmp_name
    STDERR.puts "  #{remote_name} => #{tmp_name}"
    system %Q|rclone #{RCLONE_OPTS} copyto  #{srv_name}:#{remote_name.shellescape}  #{tmp_name.shellescape}|
    File.open(tmp_name) do |fd_chunk| # apppendi il pezzo 10MB al colpo
      fd_dst << fd_chunk.read(1024 * 1024 * 10) until fd_chunk.eof?
    end
    File.unlink tmp_name
  end
  fd_dst.close
else
  puts "action [#{action}] unknown"

  print_usage_and_exit
end