S
Simon Strandgaard
------=_Part_23615_1648111.1138193506103
Content-Type: text/plain; charset=ISO-8859-1
Content-Transfer-Encoding: quoted-printable
Content-Disposition: inline
Purpose:
create a repository backup, compress, encrypt and split
it into small chunks that can be mailed to a gmail account.
Actually its a Rake file with the following rules:
rake backup # creates a hotcopy backup of the repository.
rake checkout # checkout's a working copy.
rake clean # wipe all temporary files.
rake create # creates an empty repository that is FSFS based.
rake recover # recover a repository from chunks.
Similar the "recover" rule concats the chunks, decrypt, and
decompress it.
prerequisits:
* ruby-1.8.x
* rake
* tmail
* GPG is used for doing an symmetric encryption.
* subversion
If you find it useful then please consider contributing.
--
Simon Strandgaard
------=_Part_23615_1648111.1138193506103
Content-Type: application/octet-stream; name=rakefile
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment; filename="rakefile"
#!/usr/bin/local/rake
=begin
Subversion Backup Tool
Copyright 2006 by Simon Strandgaard
prerequisits:
* ruby-1.8.x
* rake
* tmail
* gnupg
* subversion
=end
task :default => :backup
# settings is prefixed with the taskname, that uses that setting.
$settings = {
'logfile' => 'rake.log',
'create_repository_name' => 'repo',
'checkout_url' => 'file:///home/username',
'backup_tmpdir_name' => 'tmp_backup',
'backup_repository_path' => 'repo',
'backup_zipfile' => 'backup.tgz',
'backup_cryptfile' => 'crypt',
'backup_chunk_size' => 1024*1024,
'backup_chunk_prefix' => 'chunk_',
'recover_concat_file' => 'concat',
'recover_chunk_prefix' => 'chunk_',
'recover_file' => 'recover.tgz',
'passphrase' => 'relativesecretcode',
'passphrase_file' => 'passphrase',
'smtp_server' => 'mail.provider.com',
'mail_sender' => 'Server <[email protected]>',
'backup_recievers' => 'Admin1 <[email protected]>, Admin2 <[email protected]>'
}
def log(msg)
str = Time.now.to_s + ": " + msg + "\n"
file = $settings['logfile']
File.open(file, 'a+') {|f| f.write str }
end
require 'fileutils'
require 'net/smtp'
require 'tmail'
##########################################################################
## Mail stuff ##
## ##
##########################################################################
class TMail::Mail
def add_attachment(this_filename,this_content_type,send_name = nil)
justfile = File.basename(this_filename)
send_name ||= justfile
l_attach = TMail::Mail.new
l_attach.content_disposition = 'attachment; filename="'+send_name+'"'
types = this_content_type.split('/')
l_attach.set_content_type types[0], types[1],{"filename" => send_name}
l_attach.transfer_encoding = "base64"
l_attach.body = [IO.read(this_filename)].pack('m')
if self.parts.empty?
l_bodypart = TMail::Mail.new
l_bodypart.set_content_type "text","plain"
l_bodypart.content_disposition = "inline"
l_bodypart.body = self.body
self.set_content_type("multipart","mixed")
self.parts[0] = l_bodypart
end
self.parts.push l_attach
end
end
def send_mail(reciever, subject, bodytext, attachments)
mail = TMail::Mail.new
mail.to = reciever
mail_sender = $settings['mail_sender']
smtp_server = $settings['smtp_server']
mail.from = mail_sender
mail.subject = subject
mail.date = Time.now
mail.mime_version = '1.0'
mail.set_content_type 'text', 'plain', {'charset'=>'ISO-8859-1'}
mail.body = bodytext
attachments.each do |filename, mime|
mail.add_attachment(filename,mime)
end
mail.write_back
Net::SMTP.start(smtp_server) do |smtp|
smtp.sendmail(mail.encoded, mail.from, mail.to)
end
end
##########################################################################
## Subversion stuff ##
## ##
##########################################################################
def create_empty_repository(dirname)
FileUtils.mkdir(dirname)
sh "svnadmin create --fs-type fsfs #{dirname}"
end
def checkout_working_copy(wc_dirname, checkout_url)
FileUtils.mkdir(wc_dirname)
Dir.chdir(wc_dirname) do
sh "svn co #{checkout_url}"
end
end
def create_hotcopy(repositorypath, backupdir)
if File.directory?(backupdir)
raise "ERROR: #{backupdir.inspect} already exists, you must remove it!"
end
sh "svnadmin hotcopy #{repositorypath} #{backupdir}"
end
def youngest_revision(repositorypath)
revstr = `svnlook youngest #{repositorypath}`
revstr.match(/\d+/).to_s
end
# check that integrity of the repository is ok
def verify(repositorypath)
sh "svnadmin verify #{repositorypath}"
end
##########################################################################
## Compression stuff ##
## ##
##########################################################################
def compress_dir(dirname, tgzname)
if File.file?(tgzname)
raise "ERROR: #{tgzname.inspect} already exists, you must remove it!"
end
sh "tar cvzf #{tgzname} #{dirname}"
end
def decompress(zipfile)
sh "tar xzvf #{zipfile}"
end
##########################################################################
## Encryption stuff ##
## ##
##########################################################################
def encrypt_file(infile, outfile, password)
raise "ERROR: #{outfile.inspect} already exists" if File.file?(outfile)
passphrase_file = $settings['passphrase_file']
File.open(passphrase_file, "w+") {|f| f.write(password) }
sh "gpg --symmetric --output #{outfile} --passphrase-fd 0 #{infile} < #{passphrase_file}"
end
def decrypt_file(infile, outfile, password)
passphrase_file = $settings['passphrase_file']
File.open(passphrase_file, "w+") {|f| f.write(password) }
sh "gpg --decrypt --output #{outfile} --passphrase-fd 0 #{infile} < #{passphrase_file}"
end
##########################################################################
## Chunk stuff ##
## ##
##########################################################################
def split_file(filename, chunksize, prefix)
sh "split -b #{chunksize} #{filename} #{prefix}"
end
def concat_chunks(inchunks, outfile)
raise 'ERROR: expected at least 1 chunk.' if inchunks.empty?
sh "cat #{inchunks.join(' ')} >> #{outfile}"
end
##########################################################################
## creates an empty SVN repository that is FSFS based. ##
## NOTE: FSFS is more robust compared to BDB based repositories. ##
##########################################################################
desc "creates an empty repository that is FSFS based."
task
create) do
name = $settings['create_repository_name']
create_empty_repository(name)
puts <<-MSG
NOTE: be sure to edit the following config files.
prompt> sudo vi inetd.conf
# add this line
svn stream tcp nowait USER /usr/bin/svnserve svnserve -i -r REPO
# USER = neoneye
# REPO = /home/svn/repo
prompt> sudo vi services
# add these 2 lines
svn 3690/tcp # Subversion
svn 3690/udp # Subversion
NOTE: be sure to edit these config files.
MSG
end
##########################################################################
##########################################################################
desc "checkout's a working copy."
task :checkout do
checkout_url = $settings['checkout_url']
checkout_working_copy('wc', checkout_url)
end
##########################################################################
##########################################################################
desc "creates a hotcopy backup of the repository."
task :backup => [:clean] do
ok = true
log('create hotcopy')
tmpdir = $settings['backup_tmpdir_name']
repo_path = $settings['backup_repository_path']
create_hotcopy(repo_path, tmpdir)
log('compressing')
zipfile = $settings['backup_zipfile']
compress_dir(tmpdir, zipfile)
log('encrypting')
passphrase = $settings['passphrase']
cryptfile = $settings['backup_cryptfile']
encrypt_file(zipfile, cryptfile, passphrase)
log('splitting into chunks')
# split file into small chunks (that can go with the mail)
prefix = $settings['backup_chunk_prefix']
size = $settings['backup_chunk_size']
split_file(cryptfile, size, prefix)
# send a mail with each chunk attached
rev = youngest_revision(tmpdir)
time = Time.now.strftime('%Y%m%d')
subject = "hotcopy#{time}_rev#{rev}"
reciever = $settings['backup_recievers']
mime = 'application/octet-stream'
chunks = Dir.glob(prefix + '*').sort
log("revision #{rev}, consists of #{chunks.size} chunks.")
chunks.each_with_index do |filename, index|
log("sending chunk##{index+1}.")
bodytext = "this is chunk##{index+1} out of #{chunks.size} in total."
attachments = [[filename, mime]]
begin
send_mail(reciever, subject, bodytext, attachments)
rescue => e
log("ERROR: failed sending, #{e.inspect}")
ok = false
end
end
msg = ok ? "OK" : "with error!"
log("backup completed #{msg}\n\n")
end
##########################################################################
##########################################################################
desc "recover a repository from chunks."
task :recover do
concatfile = $settings['recover_concat_file']
chunk_pattern = $settings['recover_chunk_prefix']
chunks = Dir.glob(chunk_pattern + '*').sort
concat_chunks(chunks, concatfile)
passphrase = $settings['passphrase']
recoverfile = $settings['recover_file']
decrypt_file(concatfile, recoverfile, passphrase)
decompress(recoverfile)
verify('tmp_backup') # TODO: identify what got decompressed
end
##########################################################################
##########################################################################
desc "wipe all temporary files."
task :clean do
zipfile = $settings['backup_zipfile']
FileUtils.rm zipfile, {:force=>true}
tmpdir = $settings['backup_tmpdir_name']
FileUtils.rm_r tmpdir, {:force=>true}
cryptfile = $settings['backup_cryptfile']
FileUtils.rm_r cryptfile, {:force=>true}
chunk_pattern = $settings['backup_chunk_prefix']
chunks = Dir.glob(chunk_pattern + '*')
FileUtils.rm chunks, {:force=>true}
recoverfile = $settings['recover_file']
FileUtils.rm recoverfile, {:force=>true}
concatfile = $settings['recover_concat_file']
FileUtils.rm concatfile, {:force=>true}
passphrase_file = $settings['passphrase_file']
FileUtils.rm passphrase_file, {:force=>true}
end
------=_Part_23615_1648111.1138193506103--
Content-Type: text/plain; charset=ISO-8859-1
Content-Transfer-Encoding: quoted-printable
Content-Disposition: inline
Purpose:
create a repository backup, compress, encrypt and split
it into small chunks that can be mailed to a gmail account.
Actually its a Rake file with the following rules:
rake backup # creates a hotcopy backup of the repository.
rake checkout # checkout's a working copy.
rake clean # wipe all temporary files.
rake create # creates an empty repository that is FSFS based.
rake recover # recover a repository from chunks.
Similar the "recover" rule concats the chunks, decrypt, and
decompress it.
prerequisits:
* ruby-1.8.x
* rake
* tmail
* GPG is used for doing an symmetric encryption.
* subversion
If you find it useful then please consider contributing.
--
Simon Strandgaard
------=_Part_23615_1648111.1138193506103
Content-Type: application/octet-stream; name=rakefile
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment; filename="rakefile"
#!/usr/bin/local/rake
=begin
Subversion Backup Tool
Copyright 2006 by Simon Strandgaard
prerequisits:
* ruby-1.8.x
* rake
* tmail
* gnupg
* subversion
=end
task :default => :backup
# settings is prefixed with the taskname, that uses that setting.
$settings = {
'logfile' => 'rake.log',
'create_repository_name' => 'repo',
'checkout_url' => 'file:///home/username',
'backup_tmpdir_name' => 'tmp_backup',
'backup_repository_path' => 'repo',
'backup_zipfile' => 'backup.tgz',
'backup_cryptfile' => 'crypt',
'backup_chunk_size' => 1024*1024,
'backup_chunk_prefix' => 'chunk_',
'recover_concat_file' => 'concat',
'recover_chunk_prefix' => 'chunk_',
'recover_file' => 'recover.tgz',
'passphrase' => 'relativesecretcode',
'passphrase_file' => 'passphrase',
'smtp_server' => 'mail.provider.com',
'mail_sender' => 'Server <[email protected]>',
'backup_recievers' => 'Admin1 <[email protected]>, Admin2 <[email protected]>'
}
def log(msg)
str = Time.now.to_s + ": " + msg + "\n"
file = $settings['logfile']
File.open(file, 'a+') {|f| f.write str }
end
require 'fileutils'
require 'net/smtp'
require 'tmail'
##########################################################################
## Mail stuff ##
## ##
##########################################################################
class TMail::Mail
def add_attachment(this_filename,this_content_type,send_name = nil)
justfile = File.basename(this_filename)
send_name ||= justfile
l_attach = TMail::Mail.new
l_attach.content_disposition = 'attachment; filename="'+send_name+'"'
types = this_content_type.split('/')
l_attach.set_content_type types[0], types[1],{"filename" => send_name}
l_attach.transfer_encoding = "base64"
l_attach.body = [IO.read(this_filename)].pack('m')
if self.parts.empty?
l_bodypart = TMail::Mail.new
l_bodypart.set_content_type "text","plain"
l_bodypart.content_disposition = "inline"
l_bodypart.body = self.body
self.set_content_type("multipart","mixed")
self.parts[0] = l_bodypart
end
self.parts.push l_attach
end
end
def send_mail(reciever, subject, bodytext, attachments)
mail = TMail::Mail.new
mail.to = reciever
mail_sender = $settings['mail_sender']
smtp_server = $settings['smtp_server']
mail.from = mail_sender
mail.subject = subject
mail.date = Time.now
mail.mime_version = '1.0'
mail.set_content_type 'text', 'plain', {'charset'=>'ISO-8859-1'}
mail.body = bodytext
attachments.each do |filename, mime|
mail.add_attachment(filename,mime)
end
mail.write_back
Net::SMTP.start(smtp_server) do |smtp|
smtp.sendmail(mail.encoded, mail.from, mail.to)
end
end
##########################################################################
## Subversion stuff ##
## ##
##########################################################################
def create_empty_repository(dirname)
FileUtils.mkdir(dirname)
sh "svnadmin create --fs-type fsfs #{dirname}"
end
def checkout_working_copy(wc_dirname, checkout_url)
FileUtils.mkdir(wc_dirname)
Dir.chdir(wc_dirname) do
sh "svn co #{checkout_url}"
end
end
def create_hotcopy(repositorypath, backupdir)
if File.directory?(backupdir)
raise "ERROR: #{backupdir.inspect} already exists, you must remove it!"
end
sh "svnadmin hotcopy #{repositorypath} #{backupdir}"
end
def youngest_revision(repositorypath)
revstr = `svnlook youngest #{repositorypath}`
revstr.match(/\d+/).to_s
end
# check that integrity of the repository is ok
def verify(repositorypath)
sh "svnadmin verify #{repositorypath}"
end
##########################################################################
## Compression stuff ##
## ##
##########################################################################
def compress_dir(dirname, tgzname)
if File.file?(tgzname)
raise "ERROR: #{tgzname.inspect} already exists, you must remove it!"
end
sh "tar cvzf #{tgzname} #{dirname}"
end
def decompress(zipfile)
sh "tar xzvf #{zipfile}"
end
##########################################################################
## Encryption stuff ##
## ##
##########################################################################
def encrypt_file(infile, outfile, password)
raise "ERROR: #{outfile.inspect} already exists" if File.file?(outfile)
passphrase_file = $settings['passphrase_file']
File.open(passphrase_file, "w+") {|f| f.write(password) }
sh "gpg --symmetric --output #{outfile} --passphrase-fd 0 #{infile} < #{passphrase_file}"
end
def decrypt_file(infile, outfile, password)
passphrase_file = $settings['passphrase_file']
File.open(passphrase_file, "w+") {|f| f.write(password) }
sh "gpg --decrypt --output #{outfile} --passphrase-fd 0 #{infile} < #{passphrase_file}"
end
##########################################################################
## Chunk stuff ##
## ##
##########################################################################
def split_file(filename, chunksize, prefix)
sh "split -b #{chunksize} #{filename} #{prefix}"
end
def concat_chunks(inchunks, outfile)
raise 'ERROR: expected at least 1 chunk.' if inchunks.empty?
sh "cat #{inchunks.join(' ')} >> #{outfile}"
end
##########################################################################
## creates an empty SVN repository that is FSFS based. ##
## NOTE: FSFS is more robust compared to BDB based repositories. ##
##########################################################################
desc "creates an empty repository that is FSFS based."
task
create) doname = $settings['create_repository_name']
create_empty_repository(name)
puts <<-MSG
NOTE: be sure to edit the following config files.
prompt> sudo vi inetd.conf
# add this line
svn stream tcp nowait USER /usr/bin/svnserve svnserve -i -r REPO
# USER = neoneye
# REPO = /home/svn/repo
prompt> sudo vi services
# add these 2 lines
svn 3690/tcp # Subversion
svn 3690/udp # Subversion
NOTE: be sure to edit these config files.
MSG
end
##########################################################################
##########################################################################
desc "checkout's a working copy."
task :checkout do
checkout_url = $settings['checkout_url']
checkout_working_copy('wc', checkout_url)
end
##########################################################################
##########################################################################
desc "creates a hotcopy backup of the repository."
task :backup => [:clean] do
ok = true
log('create hotcopy')
tmpdir = $settings['backup_tmpdir_name']
repo_path = $settings['backup_repository_path']
create_hotcopy(repo_path, tmpdir)
log('compressing')
zipfile = $settings['backup_zipfile']
compress_dir(tmpdir, zipfile)
log('encrypting')
passphrase = $settings['passphrase']
cryptfile = $settings['backup_cryptfile']
encrypt_file(zipfile, cryptfile, passphrase)
log('splitting into chunks')
# split file into small chunks (that can go with the mail)
prefix = $settings['backup_chunk_prefix']
size = $settings['backup_chunk_size']
split_file(cryptfile, size, prefix)
# send a mail with each chunk attached
rev = youngest_revision(tmpdir)
time = Time.now.strftime('%Y%m%d')
subject = "hotcopy#{time}_rev#{rev}"
reciever = $settings['backup_recievers']
mime = 'application/octet-stream'
chunks = Dir.glob(prefix + '*').sort
log("revision #{rev}, consists of #{chunks.size} chunks.")
chunks.each_with_index do |filename, index|
log("sending chunk##{index+1}.")
bodytext = "this is chunk##{index+1} out of #{chunks.size} in total."
attachments = [[filename, mime]]
begin
send_mail(reciever, subject, bodytext, attachments)
rescue => e
log("ERROR: failed sending, #{e.inspect}")
ok = false
end
end
msg = ok ? "OK" : "with error!"
log("backup completed #{msg}\n\n")
end
##########################################################################
##########################################################################
desc "recover a repository from chunks."
task :recover do
concatfile = $settings['recover_concat_file']
chunk_pattern = $settings['recover_chunk_prefix']
chunks = Dir.glob(chunk_pattern + '*').sort
concat_chunks(chunks, concatfile)
passphrase = $settings['passphrase']
recoverfile = $settings['recover_file']
decrypt_file(concatfile, recoverfile, passphrase)
decompress(recoverfile)
verify('tmp_backup') # TODO: identify what got decompressed
end
##########################################################################
##########################################################################
desc "wipe all temporary files."
task :clean do
zipfile = $settings['backup_zipfile']
FileUtils.rm zipfile, {:force=>true}
tmpdir = $settings['backup_tmpdir_name']
FileUtils.rm_r tmpdir, {:force=>true}
cryptfile = $settings['backup_cryptfile']
FileUtils.rm_r cryptfile, {:force=>true}
chunk_pattern = $settings['backup_chunk_prefix']
chunks = Dir.glob(chunk_pattern + '*')
FileUtils.rm chunks, {:force=>true}
recoverfile = $settings['recover_file']
FileUtils.rm recoverfile, {:force=>true}
concatfile = $settings['recover_concat_file']
FileUtils.rm concatfile, {:force=>true}
passphrase_file = $settings['passphrase_file']
FileUtils.rm passphrase_file, {:force=>true}
end
------=_Part_23615_1648111.1138193506103--