#!/usr/bin/perl
#
# Copyright (c) 2006 Zmanda Inc.  All Rights Reserved.
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License version 2 as published
# by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
#
# Contact information: Zmanda Inc, 505 N Mathlida Ave, Suite 120
# Sunnyvale, CA 94085, USA, or: http://www.zmanda.com
#
#

#
# The objective of this module is to provide some common functionality
# across ZRM.
#

package ZRM::Common;

use strict;
use warnings;

use Getopt::Long;
use File::Copy;
use File::Find;
use File::Path;
use File::Basename;
use File::Spec::Functions;
use POSIX qw(strftime);
use Fcntl ':flock';
use File::Temp qw/ :POSIX /;

our $ZRM_HOME = "";
my $notifyRoutine;
eval "use ZmcSocketInterface";
if( ! $@ ){
	$notifyRoutine = \&notifyZMC;
	$ZRM_HOME = &getInstallDir();
	$ENV{'ZRM_HOME'} = $ZRM_HOME;
}

require Exporter;

use vars qw(@ISA @EXPORT_OK);
our @ISA = qw(Exporter);
our @EXPORT = qw($start_time $onWindows $date $ZRM_HOME $PERL_BINARY $MYSQL_ZRM_BASEDIR $TMPDIR $VARLIB $LOGNAME_LOCATION $LOGNAME $LOGGER $INDEX_FILENAME $MYSQL_ZRM_CONFIG_FILE $TAR $TAR_COMPRESS_OPTIONS $TAR_UNCOMPRESS_OPTIONS $TAR_EXCLUDE_OPTION $COMPRESS_FILENAME $COMPRESS_LOGICAL_FILENAME $EXTRACTED_FILENAME $MOUNT_POINT $LINK_POINT $ZRM_MOUNT_DETAILS $GZIP $CAT $CP $BACKUPSQLFILE $abort_flag $def_perm $default_backupset %inputs %indexdata $confFileHandle $zrm_version $backupset_dir $backupset $action $lastBackupDir $log_open $exitRoutine $indexRoutine %opt $zrm_error $verbose $USAGE_COMMON_OPTIONS_STRING $USAGE_STRING &usage &my_exit &abortAndDie &printAndDie &printWarning &printError &lockLog &unlockLog &printLog &printCommandOutputToLog &execCmdAndGetOutput &validateParams &checkForBlank &backSlashesToForwardSlashes &singleQuoteWithDoubleQuote &readOptionsFile &isEmpty &getZrmVersion &getLastBackupDir &removeExtraSlashesFromPath &doCompressAndEncrypt &removeUncompressedBackup &copyOneFileRemote &copyOneFileLocal &copyTo &copyFiles &copyUsingTar &copyUsingTarFromRemote &getUniqueFileName &getRetentionTimeInSecs &parseGeneralOptionFile &parseIndexFile &isNumeric &validateData &removeBlanks &execPlugin &uncompressBackup &processOptionFiles &processLastBackupDir &processCommandLineParams &createConfigFile &openLogFile &processTmpDir &processCopyPlugin &initCommon &convertGlobToPattern &filterPattern &verifySnapshotParams);

our $oldPATH = $ENV{'PATH'};
our $PERL_BINARY = "";
our $onWindows = 0;
if( $^O =~ /^MSWin/ ) {
	$onWindows = 1;
	$PERL_BINARY = "\"$^X \"";
}else{
	$ENV{'PATH'} = "/usr/local/bin:/opt/csw/bin:/usr/bin:/usr/sbin:/sbin:/bin:/usr/ucb";
}

$ENV{'ZRM_CONF'} = "";
our $start_time=time();
our $date=strftime("%Y%m%d%H%M%S", localtime );

#All data specifc to the utility is stored inside this directory
#e.g. mysql-zrm.conf, the backup-set information etc.
our $MYSQL_ZRM_BASEDIR="/etc/mysql-zrm";
if( $onWindows ){
	$MYSQL_ZRM_BASEDIR=$ZRM_HOME."\\etc\\mysql-zrm";
}

our $TMPDIR;
our $VARLIB="/var/lib";
our $LOGNAME_LOCATION="/var/log/mysql-zrm/";
if( $onWindows ) {
        $LOGNAME_LOCATION="$ZRM_HOME"."\\var\\log\\mysql-zrm\\";
        $VARLIB="$ZRM_HOME\\var\\lib";
}
our $LOGNAME=catfile($LOGNAME_LOCATION, "mysql-zrm.log");

our $LOGGER;

our $INDEX_FILENAME="index";
our $MYSQL_ZRM_CONFIG_FILE=catfile( $MYSQL_ZRM_BASEDIR, "mysql-zrm.conf");
#Compress & Encrypt progs
our $TAR="gtar";
our $TAR_COMPRESS_OPTIONS=" --same-owner -f- -cpsC ";
our $TAR_UNCOMPRESS_OPTIONS=" --same-owner -f- -xpsC ";
our $TAR_EXCLUDE_OPTION=" --exclude";
our $COMPRESS_FILENAME="backup-data";
our $COMPRESS_LOGICAL_FILENAME="backup-sql";
our $GZIP="gzip";
our $EXTRACTED_FILENAME=".extracted";
our $MOUNT_POINT="ZRM_MOUNTS";
our $LINK_POINT="ZRM_LINKS";
our $ZRM_MOUNT_DETAILS=".zrm_mount_details";
#End of compress & encrypt progs

our $CAT="cat";
our $CP="cp";

our $BACKUPSQLFILE="backup.sql";

#List of params that are allowed in the mysql-zrm.conf file. This is a union of all parameters acroos all utilities of mysql-zrm
our @validParams=qw/
		user password host port socket 
		mysql-binpath mysql-binlog-path
		comment all-databases database databases tables
		exclude-pattern
		backup-level backup-mode replication lvm-snapshot snapshot-size destination backup-type
		backup-name default-character-set routines mailto retention-policy
		source-directory bin-logs start-position stop-position offset
		start-datetime stop-datetime
		ssl-options all-backups index till-lastfull
		parse-binlogs-plugin parse-binlogs-plugin-options
		pre-backup-plugin pre-backup-plugin-options snapshot-plugin
		post-backup-plugin post-backup-plugin-options pre-scheduler
		pre-restore-plugin post-restore-plugin
		compress compress-plugin encrypt encrypt-plugin decrypt-option compress-mysqldump-onthefly
		html-report-directory html-reports webserver-url rss-header-location
		copy-plugin passfile ssh-user remote-mysql-binpath socket-remote-port
		windows-backup-port windows-restore-port
		ihb-plugin ihb-sleep ihb-use-memory ihb-binpath
		emc-host navicli-binpath storage-group
		snapshot-device vxfs-binpath
		mysql-shutdown retry-count retry-delay tmpdir quiet verbose help
		global remove-options synchronous-checksum check-innodb config-file-list
		single-transaction extra-mysqldump-options
		compress-data-during-transport mail-policy row-event-details
		bluearc-host bluearc-user bluearc-password bluearc-ssc-bin-path
		output-format
		ssh-identity-file
                device-fs-map dismount-existing
		/;

our $abort_flag = 0;
our $def_perm=0700;
our $default_backupset="BackupSet1";
our %inputs;
our %indexdata;
our $confFileHandle;
our $zrm_version;
our $backupset_dir;
our $backupset;
our $action="none";
our $lastBackupDir;
our $log_open=0;
our $exitRoutine;
our $indexRoutine;
our %opt;
# $zrm_error = 0 means there was no error
# $zrm_error = 1 means some error happened
# $zrm_error = 2 means fatal error
our $zrm_error=0;

our $verbose = 0;

our $USAGE_COMMON_OPTIONS_STRING =
                "\t\t[--backup-set\ <backup-set\ name>]\n".
                "\t\t[--options-file\ <filename>]\n".
		"\t\t[--tmpdir\ <temporary\ directory\ to\ be\ used>]\n".
                "\t\t[--quiet|--noquiet]\n".
                "\t\t[--verbose]\n".
                "\t\t[--help]\n";

my @COMMON_OPTIONS = qw/
	      backup-set=s
	      options-file=s
	      tmpdir=s
              quiet!
              verbose
              help/;

our $USAGE_STRING="Usage string is not initialized. Please use setUsageString() to initialize the usage string";

#Denotes usage/help
sub usage()
{
        print STDERR "$_[0]\n";
        print STDERR "USAGE:\n";
	if( defined $ENV{"ZRM_CMD_NAME"} && $ENV{"ZRM_CMD_NAME"} ne "" ){
		print STDERR $ENV{"ZRM_CMD_NAME"}."\n";
	}else{
		print STDERR "$0\n";
	}
        print STDERR "$USAGE_STRING\n";
        $zrm_error = 2;
        &my_exit();
}

# This will call exit. Do any cleanup required
sub my_exit()
{
	if( defined $exitRoutine ) {
		&$exitRoutine();
	}
        if( $LOGGER ){
                unlink( $LOGGER );
        }

        if( $ENV{'ZRM_CONF'} ){
                unlink( $ENV{'ZRM_CONF'} );
        }
        $ENV{'PATH'} = $oldPATH;
	if( $action eq "backup" ){
		&printLog( "END OF BACKUP\n" );
	}
	if( $log_open ){
		close LOG;
	}
	&notifyMonitor($action."_ZRM_STOPPED");
        exit( $zrm_error );
}

sub abortAndDie()
{
        &printAndDie( "Process Aborted\n" );
}


#This subroutine prints an error message and exits
sub printAndDie()
{
	&printError( $_[0] );
	$zrm_error = 2;
	if( $action ne "check" ){
        	&my_exit();
	}
}

#print error messages
sub printWarning()
{
	&printLog( $_[0], "WARNING: " );
}

#print error messages
sub printError()
{
	$zrm_error = 1;
	&printLog( $_[0], "ERROR: " );
}

sub lockLog()
{
        flock(LOG, LOCK_EX);
        seek(LOG, 0, 2);
}

sub unlockLog()
{
        flock(LOG, LOCK_UN);
}

#print messages to Log
#$_[0] the message
#$_[1] the type (if not present type is assumed as INFO)
sub printLog()
{
	my @tcomp = localtime();
	my $fdate=strftime("%Y%m%d%H%M%S", @tcomp );
	my $d = strftime("%a %b %d %H:%M:%S %Y", @tcomp );
	my $type = "INFO: ";
	if( $_[1] ){
		$type = $_[1];
	}
	my $x = $_[0];
	#blank out the password if present
	$x=~s/-password=("[^"]*")/-password="*****"/;
	$x=~s/-secret-key=("[^"]*")/-secret-key="*****"/;
	my $orgMsg = $x;
	my $msg = "";
	my $bset = "";
	if( $backupset && $action ne "purge" ){
		$msg .= "$backupset:";
		$bset = $backupset;
	}
	$msg .= "$action:";
	$msg .= $type.$x;
	if( $log_open ){
		&lockLog();
		print LOG $d.": ".$msg;
		&unlockLog();
		chop($type);
		chop($type);
		if( $type eq "ERROR" ){
			$type = "failure";
		}elsif( $type eq "WARNING" ){
			$type = "warning";
		}elsif( $type eq "INFO" ){
			$type = "info";
		}

		chomp( $orgMsg );
		&notifyMonitor($orgMsg, 2, $action, $bset, $type, $fdate);
	}
	if( !defined $inputs{"quiet"} || $inputs{"quiet"} == 0 ){
		print STDERR $msg;
	}
}

#Read the stuff from given file and put into log
#$_[0] indicates if this is an error or a warning or info
#$_[1] message to preface the output with
#$_[2] is the name of file to read from
sub printCommandOutputToLog( )
{
	unless (open(TMPCONF, $_[2])) {
		&printWarning( "Could not read file ".$_[2]."\n" );
		return;
        }
        my @tmparr = <TMPCONF>;
        close(TMPCONF);
	if( ! @tmparr ) {
		return;
	}
	my $x = "Output of command: '".$_[1]."' is {\n@tmparr}\n";
	if( $_[0] eq "WARNING" ){
		&printWarning( $x );
	} elsif ( $_[0] eq "ERROR" ){
		&printError( $x );
	}else{
		&printLog( $x );
	}
}

# Execute an external program and get its output
# $_[0] command to execute
# if $_[1] is present, then stderr is not captured
# return the output of the command on success else returns nothing
sub execCmdAndGetOutput()
{
	my $n = tmpnam();
	my $cmd = " > \"$n\"";
	if( !$_[1] ){
		$cmd = $cmd." 2>\"$LOGGER\"";
	}
        if( $abort_flag ){
                &abortAndDie( );
        }

	my $r = system( "$_[0] $cmd" );
        if( $abort_flag ){
                &abortAndDie( );
        }
	if( $r != 0 ){
		if( !$_[1] ){
			&printCommandOutputToLog( "ERROR", "$_[0]", $LOGGER );
			unlink( $LOGGER );
		}else{
			&printError( "Command $_[0] returned error\n" );
		}
		unlink $n;
		return;
	}
	if( !$_[1] ){
		unlink( $LOGGER );
	}
        unless (open(TMPCONF, $n)) {
                &printError( "Could not read file ".$n."\n" );
		unlink $n;
                return;
        }
        my @tmparr = <TMPCONF>;
        close(TMPCONF);
	unlink $n;
	my $x = "";
        if( ! @tmparr ) {
                return $x;
        }
	foreach( @tmparr ){
		$x = $x.$_;
	}
	return $x;
}

# $_[0] pattern to convert
sub convertGlobToPattern()
{
	my $s = $_[0];
	my $orig = $s;
	my %pattern = (
			'*' => '.*',
			'?' => '.',
			'|' => '|',
			'[' => '[',
			']' => ']',
		     );
	$s =~ s{(.)} { $pattern{$1} || "\Q$1" }ge;
	$s = '^'.$s.'$';
	my $regex = eval { qr/$s/ };
	if( $@ ){
		&printAndDie( "Exclude regex $orig ($s) doesn't seem valid : $@" );	 
	}
	return $s;
}

# @_ list of strings to filter
# returns list of filtered strings
sub filterPattern()
{
	my @a = @_;
	my @f;
	if( ! defined $inputs{"exclude-pattern"} ){
		return @a;
	}
	if( $verbose ){
		&printLog( "Using following exclude pattern ".$inputs{"exclude-pattern"}."\n" );
	}
	foreach( @a ){
		if( $_ !~ /$inputs{"exclude-pattern"}/ ){
			push @f, $_;	
		}else{
			if( $verbose ){
				&printLog( "excluding $_ from backup\n" );
			}
		}
	}
	return @f;
}

#Used for validating parameters in supplied options file
sub validateParams()
{
        my $param;
        $param = $_[0];

        foreach (@validParams) {
                if ($param  eq $_) {
                        return 1;
                }
        }
        return 0;
}

sub checkForBlank()
{
	if( $_[0] eq "" || $_[0] eq "\n" || $_[0]=~/^\s+$/){
		return;
	}elsif( $_[0]=~/^\s*#/ ){
		return;
	}elsif($_[0]=~/#/){
		return $`;
	}else{
		return $_[0];
	}
}

#$_[0] path string
sub backSlashesToForwardSlashes()
{
        my $r = $_[0];
        $r =~ s/\\/\//g;
        return $r;
}

# this is to handle some path names with spaces that just would otherwise not work in windows
sub singleQuoteWithDoubleQuote()
{
        my $r = $_[0];
        $r =~ s/'/"/g;
        return $r;

}

#Reads a given option file
#$_[0] specifies file to be read (full path needs to be provided)
#$_[1] specifies the hash to store the options
sub readOptionsFile()
{
	if( !defined $_[0] ){
		&printAndDie( "Please supply name of option file to parse\n" );
	}
	if( !defined $_[1] ){
		&printAndDie( "hash to store options not found\n" );
	}
	my $oo = $_[1];
	unless (open(TMPCONF, $_[0])) {
		&printAndDie("cannot open input options file $_[0]. $!\n");
		return;
        }
	if( $verbose ){
		&printLog( "Reading options from file ".$_[0]."\n" );
	}
        my @tmparr = <TMPCONF>;
        close(TMPCONF);
        chomp (@tmparr);
	my $var1;
        foreach $var1 (@tmparr) {
		$var1 = &checkForBlank($var1);
		if($var1){
			#Removing leading white spaces
                	$var1=~ s/^\s+//;
			#Removing trailing white spaces
                	$var1=~ s/\s+$//;
			my @v=split("=",$var1);
			my $v1 = shift @v;
			#Removing trailing white spaces
                	$v1=~ s/\s+$//;
			my $v2 = join( "=", @v );
			#Removing leading white spaces
                	$v2=~ s/^\s+//;
			#Removing leading quotes
			$v2=~ s/^\"//;
			#Removing trailing quotes
			$v2=~ s/\"$//;
			my $ret_val = &validateParams($v1);
			if(!$ret_val) {
				&printWarning("Unknown parameter $var1 in $_[0]. Assuming this is a custom parameter meant for one of the plugins\n");
			}
               		#$inputs{$v1}=$v2;
               		$oo->{$v1}=$v2;
		}
	}
}


# Checks if given directory is empty or not.
# $_[0] directory to check
sub isEmpty()
{
        opendir DIRH, $_[0] or return -1;
        my @a = grep !/^\.\.?$/, readdir DIRH;
        my $x = @a;
	closedir DIRH;
        return $x;
}

#Returns the info of which directory contains the last backup run data
sub getZrmVersion()
{
	my $file = catfile( $MYSQL_ZRM_BASEDIR, "mysql-zrm-release" );
	if(open(TMPCONF, $file)) {
        	my @tmparr = <TMPCONF>;
        	close(TMPCONF);
        	chomp (@tmparr);
		$zrm_version = $tmparr[0];
	}
	if( !$zrm_version ){
		$zrm_version = "ZRM for MySQL - version built from source";
	}
	&printLog( "$zrm_version\n" );
}

#Returns the info of which directory contains the last backup run data
sub getLastBackupDir()
{
        my $file = catfile( $backupset_dir, "last_backup" );
        unless (open(TMPCONF, $file)) {
                if( $verbose ){
                        &printWarning( "Could not open file $file. $!\n" );
                }
                return;
        }
        my @tmparr = <TMPCONF>;
        close(TMPCONF);
        chomp (@tmparr);
        return $tmparr[0];
}

#$_[0] path from which to remove extra slashes
sub removeExtraSlashesFromPath()
{
	my $dir = $_[0];
        $dir =~ s'//+'/'g;
        return $dir;
}

#Does the compression and encryption
#Returns 1 on success
#Return 0 on failure
sub doCompressAndEncrypt()
{
	if ( $onWindows ) {
		&printAndDie( "Compress and Encrypt routine should not be called for Windows\n" );
        }
	if( ! defined $indexRoutine ){
		&printAndDie( "Index Routine not defined. There is some strange error that should never happen\n" );
	}
	my $msg = "";
	if( $verbose ){
		if( defined $inputs{compress} ){
			$msg .= "Compressing ";
		}
		if( defined $inputs{encrypt} ){
			$msg .= "Encrypting ";
		}
		&printLog( "$msg backup\n" );
	}
	my $d = $inputs{"destination"};
	my $compfile = catfile( $d, $COMPRESS_FILENAME );
	my $logicalCompFile = catfile( $d, $COMPRESS_LOGICAL_FILENAME );
	my $cmd = "$TAR $TAR_COMPRESS_OPTIONS \"$d\" $TAR_EXCLUDE_OPTION=$COMPRESS_FILENAME $TAR_EXCLUDE_OPTION=$INDEX_FILENAME $TAR_EXCLUDE_OPTION=zrm_checksum";
	if( ! $inputs{"encrypt"} ){
		$cmd .= " $TAR_EXCLUDE_OPTION=$COMPRESS_LOGICAL_FILENAME";
	}
	my $k = tmpnam();
	$cmd .= " . 2>$k ";
	my $l = "";
	if( defined $inputs{"compress"} ){
		$cmd .= " | ";
		if( $inputs{"compress"} ne "" ){
			$cmd .= "\"$inputs{compress}\"";
		}else{
			$cmd .= $GZIP;
		}
		$l = tmpnam();
		$cmd .= " 2>$l";
	}
	if( $inputs{"encrypt"} ){
		$cmd .= " | \"$inputs{encrypt}\""
	}
	$cmd .= " > \"$compfile\"";
	$cmd = $cmd." 2>$LOGGER";
	if( $verbose ){
		&printLog( "Command used is '$cmd'\n" );
	}
        if( $abort_flag ){
                &abortAndDie( );
        }

	my $r = system( $cmd );
        if( $abort_flag ){
                &abortAndDie( );
        }

	if( $r > 0 ){
		&printError( "$msg backup failed\n" );
		if( $k ){
			&printCommandOutputToLog( "ERROR", "tar", $k );
			unlink $k;
		}
		my $msg = "compress/encrypt";
		if( $l ){
			&printCommandOutputToLog( "ERROR", $msg, $l );
			unlink $l;
		}
		&printCommandOutputToLog( "ERROR", $msg, $LOGGER );
		return 0;
	}else{
		if( $verbose ){
			if( $k ){
				&printCommandOutputToLog( "INFO", "tar", $k );
			}
			my $msg = "compress/encrypt";
			if( $l ){
				&printCommandOutputToLog( "INFO", $msg, $l );
			}
			&printCommandOutputToLog( "INFO", $msg, $LOGGER );
		}
		if( $l ){
			unlink $l;
		}
		if( $k ){
			unlink $k;
		}
	}
	if( defined $inputs{"compress"} ){
		if( $inputs{"compress"} ){
			&$indexRoutine( "compress=$inputs{compress}\n" );
		}else{
			&$indexRoutine( "compress=\n" );
		}
	}
	if( $inputs{"encrypt"} ){
		&$indexRoutine( "encrypt=$inputs{encrypt}\n" );
		&$indexRoutine( "decrypt-option=".$inputs{"decrypt-option"}."\n" );
	}
	my $t += -s $compfile;
	if( -f $logicalCompFile ){
		$t += -s $logicalCompFile;
	}
	my $SUF="MB";
        $t = $t/1048576;
        if( $t > 1000 ){
                $t = $t/1024;
                $SUF="GB";
        }
        $t = sprintf( "%0.2f", $t );
	&$indexRoutine( "backup-size-compressed=$t $SUF\n" );
	return 1;
}


#Will remove all backup data except for the index file and backup-data file
#$_[0] directory
#$_[1] reference to either inputs or indexdata hashtable
sub removeUncompressedBackup()
{
	my $topdir = &removeExtraSlashesFromPath( $_[0] );

	$topdir =~ s/\/$//g;

	my $ext = catfile( $topdir, $EXTRACTED_FILENAME );
	if( -f $ext ){
		if( $verbose ){
			&printLog( "Not removing uncompressed data\n" );
		}
		return;
	}

	if( $verbose ){
		&printLog( "Removing all of the uncompressed/unencrypted data\n" );
	}
	if( defined $_[1] ){
		my $oo = $_[1];
		if( defined $oo->{"encrypt"} ){
			my $ff = catfile( $topdir, $COMPRESS_LOGICAL_FILENAME );
			if( -f $ff ){
				unlink( $ff );
			}
		}
	}
        find( {wanted => sub{
			if( -f $_ ){
				my $n = basename($_);
				if( $n ne $INDEX_FILENAME &&
				    $n ne "zrm_checksum" &&
				    $n ne ".nochecksum" &&
				    $n ne ".checksum_pending" &&
				    $n ne $COMPRESS_FILENAME &&
				    $n ne $COMPRESS_LOGICAL_FILENAME ){
					my $r = unlink $_;
					if( $r eq 0 ){
						&printError( "could not delete $_ in directory $File::Find::dir\n" );
					}
				}
			}
		}, postprocess => sub
                {
			if( $File::Find::dir ne $topdir ){
				my $r = rmdir( $File::Find::dir );
				if( $r eq 0 ){
					&printError( "could not delete $File::Find::dir\n" );
				}
			}
                }, no_chdir=>1}, $topdir);
}


#Copies one file from source to destination preserving permissions
#If --copy-plugin is specified, that will be used
# Else copyOneFileLocal will be invoked
#$_[0] is source
#$_[1] is destination
#return 1 on success and on error return 0;
sub copyOneFileRemote()
{
	my $src = $_[0];
	my $dest = $_[1];

	if( $inputs{"copy-plugin"} ){
		my $srcHost;
		my $destHost = "localhost";
		if( $inputs{"host"} ){
			$srcHost = $inputs{"host"};
		}else{
			$srcHost = "localhost";
		}
		if( ! $onWindows && defined $action && $action eq "restore" ){
			my $tmp = $srcHost;
			$srcHost = $destHost;
			$destHost = $tmp;
		}
            	my $cmd="";
		my $plugin = $inputs{"copy-plugin"};
		$plugin = "\"$plugin\"";
                $cmd = "$cmd "."$plugin";
		my @params;
		push @params, "--source-host";
		push @params, $srcHost;
		push @params, "--source-file";
                $src = &backSlashesToForwardSlashes($src);
                $src = "\"$src\"";
		push @params, $src;
		push @params, "--destination-host";
		push @params, $destHost;
		push @params, "--destination-directory";
                $dest = &backSlashesToForwardSlashes($dest);
                $dest = "\"$dest\"";
		push @params, $dest;
		if ( $onWindows && defined $action ) {
			push @params, "--switch";
			if( $action eq "restore" ){
				push @params, "to";
			}else{
				push @params, "from";
			}	
		}

		my $r = system( "$PERL_BINARY $cmd @params" );
		if( $r ne 0 ){
			&printError( "Could not copy file ".$src."\n" );
			&printError( "copy-plugin exited with error $r\n" );
			return 0;
		}
		return 1;
	}else{
		return &copyOneFileLocal( $src, $dest );
	}
}

#Copies one file from source to destination preserving permissions
#$_[0] is source
#$_[1] is destination
#return 1 on success and on error return 0;
sub copyOneFileLocal()
{
	if( $onWindows ) {
        	my $src = "$_[0]";
            	my ($name,$path,$suffix) = fileparse("$src", qr /\.[^.]*/);
                my $dest = "$_[1]"."\\".$name.$suffix;
		return File::Copy::syscopy( "$src", "$dest" );
	} else {
		my $command = " -p \"$_[0]\" \"$_[1]\"";
		if( $verbose ) {
			&printLog( $CP.$command."\n" );
		}
		my $r = system( $CP.$command );
		if( $r > 0 ){
			&printError( "Copy from $_[0] to $_[1] failed\n" );
			return 0;
		}
	}
	return 1;
}

#Copies one file from source to destination preserving permissions
#If --copy-plugin is specified, that will be used
# Else copyOneFileLocal will be invoked
#$_[0] is source
#$_[1] is destination
#$_[2] is source host
#$_[3] if exists will not print an error even if there is an error
#return 1 on success and on error return 0;
sub copyTo()
{
	my $src = $_[0];
	my $dest = $_[1];
        $dest = &backSlashesToForwardSlashes($dest);
	if( $inputs{"copy-plugin"} ){
		my $cmd = $inputs{"copy-plugin"};
                $cmd = "\"$cmd\"";
		my @params;
		push @params, "--destination-host";
		if( $inputs{"host"} ){
			push @params, "$inputs{host}";
		}else{
			push @params, "localhost";
		}
		push @params, "--source-file";
                $src = "\"$src\"";
		push @params, $src;
		push @params, "--source-host";
		if( $_[2] ne "" ){
			push @params, $_[2];
		}else{
			push @params, "localhost";
		}
		push @params, "--destination-directory";
                $dest = "\"$dest\"";
		push @params, $dest;

		if ( $onWindows && defined $action ) {
			push @params, "--switch";
			if( $action eq "restore" ){
				push @params, "to";
			}else{
				push @params, "from";
			}	
		}

		my $r = system( "$PERL_BINARY $cmd @params" );
		if( $r > 0 ){
			if( ! defined $_[3] ){
				&printError( "Could not copy file ".$src."\n" );
				&printError( "copy-plugin exited with error $r\n" );
			}
			return 0;
		}
		return 1;
	}else{
		return &copyOneFileLocal( $src, $dest );
	}

}

#Copies a given list of files from sourcedir to destination
#$_[0] destination
#$_[1] sourcedir
#$_[2] filelist
sub copyFiles()
{
	my @files = split( " ", $_[2] );
	my $r = 1;
	foreach( @files ) {
		my $f = catfile( $_[1], $_ );
		my $x = &copyTo( $f, $_[0], "localhost" );
		if( $x == 0 ){
			$r = 0;
		}
	}
	return $r;
}

#$_[0] srcDir
#$_[1] srcFile
#$_[2] trgtDir
sub copyUsingTarFromRemote()
{
        my $srcFile = "$_[0]/$_[1]";
        my $destinationDirectory = $_[2];
        if( $verbose ){
                &printLog( "Start copying data from remote.\n" );
        }
        my $r = &copyOneFileRemote( $srcFile, $destinationDirectory );
        return ! $r;
}

#$_[0] srcDir
#$_[1] srcFile
#$_[2] trgtDir
sub copyUsingTar()
{
        if( defined $inputs{"host"} && $inputs{"host"} ne "localhost" ){
                return &copyUsingTarFromRemote( $_[0], $_[1], $_[2] );
        }
        my $srcDir = $_[0];
        my $srcFile = $_[1];
        my $trgtDir = $_[2];
        my $r;
        my $lsCmd = "";
        my $tmpf = tmpnam();
        my $fileList = $srcFile;
        if( $srcFile =~ /\*/){
                $lsCmd = "cd $srcDir; ls -1 $srcFile > $tmpf 2>/dev/null;";
                $fileList = " -T $tmpf";
        }

        my $cfile = tmpnam();
        my $xfile = tmpnam();
        my $tarCmd = "$TAR --same-owner -pshC ";
        my $srcCmd = "$lsCmd $tarCmd $srcDir -c $fileList 2>$cfile";
        my $destCmd = "$tarCmd $trgtDir -x 2>$xfile";
        my $command = "$srcCmd|$destCmd";
        if( $verbose ) {
                &printLog( "Copying all tables of database to target\n" );
                &printLog( $command."\n" );
        }
        $r = system( $command );
        if( $verbose ){
                &printCommandOutputToLog( "LOG", "$srcCmd", $cfile );
                &printCommandOutputToLog( "LOG", "$destCmd", $xfile );
        }
        unlink $cfile;
        unlink $xfile;
        if( $lsCmd ne "" ){
                unlink( $tmpf );
        }
        return $r;
}

#Tries to ensure we get a unique file name
#$_[0] name of file
#$_[1] a index number (use 0 when u initally call)
sub getUniqueFileName()
{
	my $x = $_[0];
	my $y = $_[1];
	if( -e "$x-$y" ){
		$y++;
		return &getUniqueFileName( $x, $y );
	}
	return "$x-$y";
}

#$_[0] time string
sub getRetentionTimeInSecs()
{
        my $x = $_[0];
        my $y = 0;
        if( !&isNumeric( $_[0] ) ){
                $y = chop( $x );
                if( !&isNumeric( $x ) ){
			&printError( "Bad retention policy $_[0]\n" );
                        return 0;
                }
        }
        my $secs = 0;
        if( $y eq 0 ){
                #24*60*60 (no of days)
                $secs = 86400;
        }elsif( $y eq "D" or $y eq "d" ){
                #24*60*60 (no of days)
                $secs = 86400;
        }elsif( $y eq "W" or $y eq "w" ){
                #24*7*60*60 (no of weeks)
                $secs = 604800;
        }elsif( $y eq "M" or $y eq "m" ){
                #24*30*60*60 (no of months)
                $secs = 2592000;
        }elsif( $y eq "Y" or $y eq "y" ){
                #24*365*60*60 (no of years)
                $secs = 31536000;
        }else{
                &printAndDie( "Bad retention policy $_[0]\n" );
                $secs = 0;
        }
        $x *= $secs;
        return $x;
}

sub parseGeneralOptionFile()
{
	if( !defined $_[0] ){
		&printAndDie( "Please supply name of option file to parse\n" );
	}
	if( !defined $_[1] ){
		&printAndDie( "hash to store options not found\n" );
	}
	my $oo = $_[1];
	unless (open(TMPCONF, $_[0])) {
		&printAndDie("cannot open input options file $_[0]. $!\n");
		return;
        }
	if( $verbose ){
		&printLog( "Reading options from file ".$_[0]."\n" );
	}
        my @tmparr = <TMPCONF>;
        close(TMPCONF);
        chomp (@tmparr);
	my $var1;
        foreach $var1 (@tmparr) {
		chomp($var1);
		$var1 = &checkForBlank($var1);
		if( $var1 ){
			my $v1;
			my $v2;
			($v1, $v2)=split("=",$var1);
			$oo->{$v1}=$v2;
		}
	}
	return 1;

}

#Parses the index file
#$_[0] specifies the directory containing the index file
#returns 0 if there is an error
#returns 1 on success
sub parseIndexFile()
{
	if( ! defined $_[0] ){
		return 0;
	}
	%indexdata = ();
	if( ! -e $_[0] ){
		return 0;
	}
	unless (open(TMPCONF, $_[0])) {
		return 0;
        }

        my @tmparr = <TMPCONF>;
        close(TMPCONF);
        chomp (@tmparr);
	my $var1;
        foreach $var1 (@tmparr) {
		chomp($var1);
		$var1 = &checkForBlank($var1);
		if( $var1 ){
			my $v1;
			my $v2;
			($v1, $v2)=split("=",$var1);
               		$indexdata{$v1}=$v2;
		}
	}
	return 1;
}

#checks if $_[0] is numeric nor not
#$_[0] string to check
sub isNumeric()
{
	if( $_[0]=~/^\d+$/ ){
		return 1;
	}
	return 0;
}

# Will check if the given data has only  '-' and ' ' and '.' and
# alphanumeric characters
# $_[0] data to check
# returns 1 if string is ok else returns 0
sub validateData()
{
        my $x = $_[0];
        if( !($x =~ /^([-\s\w.]+)$/) ) {
                return 0;
        }
        return 1;
}

#removes leading and trailing blanks
sub removeBlanks()
{
	my $x = $_[0];
	$x=~ s/^\s+//;
        $x=~ s/\s+$//;
	return $x;
}

#$_[0] is the name of plugin
#$_[1], $_[1]... are the compiled option(s) to be passed
sub execPlugin()
{
	if( $inputs{$_[0]} ){
		my $command = $inputs{$_[0]};
		if( ! -e $command ){
			&printAndDie( "Unable to find $_[0] $command\n" );
		}
		my @x = @_;
		shift @x;

		my $optStr = $_[0]."-options";
		if( $inputs{$optStr} ){
			unshift @x, "$inputs{$optStr}";
		}
		if( $verbose ){
			&printLog( "Command being executed is $command @x\n" );
		}
		return system( $command, @x );
	}
	return 0;
}

# $_[0] directory containing compressed data
# $_[1] == 1 uncompress the mysqldump file
# $_[1] == 0 dont uncompress the mysqldump file
# If $_[1] not specified uncompress the mysqldump file
#returns 0 on failure
#returns 1 on success
sub uncompressBackup()
{
	my $sdir = $_[0];
	my $pend = catfile( $sdir, ".checksum_pending" );
	if( -f $pend ){
		&printError( "Checksum creation in progress. In case you need to access the backup data immediately, please follow the instructions below.\n\na) Find the pid of the process doing the checksum creation by running the command: ps -ef|grep $sdir|grep create-checksum\nb) If there is a process, kill the process by running the command: kill -9 <pid>\nc) Run the command: touch $sdir/.nochecksum\nd) Run the command: rm $sdir/.checksum_pending\n\nYou can now access the data immediately." );
		return 0;
	}
	my $ext = catfile( $sdir, $EXTRACTED_FILENAME );
	if( -f $ext ){
		if( $verbose ){
			&printLog( "Data already uncompressed\n" );
		}
		return 1;
	}
	my $msg = "";
	my $filename = catfile( $sdir, $COMPRESS_FILENAME );
	my $cmd = "$CAT \"$filename\" ";
	if( $indexdata{"encrypt"} ){
		$cmd .= " | ";
		if( !$indexdata{"decrypt-option"} ){
			$indexdata{"decrypt-option"} = "-d";
		}
		$cmd .= "\"".$indexdata{"encrypt"}."\" ".$indexdata{"decrypt-option"};
	}
	if( defined $indexdata{"compress"} ){
		$cmd .= " | ";
		if( $indexdata{"compress"} ne "" ){
			$cmd .= " \"$indexdata{compress}\" -d ";
		}else{
			$cmd .= " $GZIP -d "
		}
	}
	$cmd .= " | $TAR $TAR_UNCOMPRESS_OPTIONS \"$sdir\" ";
	$cmd .= " 2>$LOGGER";
	if( $verbose ){
		&printLog( "Command used is '$cmd'\n" );
	}

	my $r = system( $cmd );
	if( $r > 0 ){
		&printError( "$msg backup failed \n" );
		&printCommandOutputToLog( "ERROR", "tar", $LOGGER );
		return 0;
	}else{
		if( $verbose ){
			&printCommandOutputToLog( "INFO", "tar", $LOGGER );
		}
	}
	if( ! defined $_[1] || $_[1] == 0 ){
		return &uncompressLogicalBackup( $sdir );
	}else{
		return 1;
	}
}

#$_[0] directory containing compressed data
#returns 0 on failure
#returns 1 on success
sub uncompressLogicalBackup()
{
	my $sdir = $_[0];
	my $msg = "";
	my $filename = catfile( $sdir, $COMPRESS_LOGICAL_FILENAME );
	my $destFilename = catfile( $sdir, $BACKUPSQLFILE );
	if( ! -f $filename ){
		return 1;
	}
	my $cmd = "$CAT \"$filename\" ";
	if( defined $indexdata{"compress"} ){
		$cmd .= " | ";
		if( $indexdata{"compress"} ne "" ){
			$cmd .= " \"$indexdata{compress}\" -d ";
		}else{
			$cmd .= " $GZIP -d "
		}
	}
	$cmd .= " >$destFilename 2>$LOGGER";
	if( $verbose ){
		&printLog( "Command used is '$cmd'\n" );
	}

	my $r = system( $cmd );
	if( $r > 0 ){
		&printError( "$msg backup failed \n" );
		&printCommandOutputToLog( "ERROR", "uncompress logical", $LOGGER );
		return 0;
	}else{
		if( $verbose ){
			&printCommandOutputToLog( "INFO", "uncompress logical", $LOGGER );
		}
	}
	return 1;
}


#process all of the options files and also takes care of setting up backupset
sub processOptionFiles()
{
	if( -e $MYSQL_ZRM_CONFIG_FILE ) {
		&readOptionsFile( $MYSQL_ZRM_CONFIG_FILE, \%inputs );
	}

	if( $opt{"backup-set"} ) {
		$backupset = $opt{"backup-set"};
		delete $opt{"backup-set"};
	}else{
		$backupset = $default_backupset;
	}
	if( !&validateData($backupset) ){
		&usage( "backup-set $backupset not valid\n" );
	}

	if( $action eq "backup" ){
		&printLog( "START OF BACKUP\n" );
		&printLog( "PHASE START: Initialization\n" );
	}
	$backupset_dir=catfile( $MYSQL_ZRM_BASEDIR, $backupset );
	if( !-d $backupset_dir ){
		if( $action ne "check" ){
			mkpath( $backupset_dir );
		}else{
			$backupset_dir = $MYSQL_ZRM_BASEDIR;
		}
	}else{
		my $backupset_conf=catfile($backupset_dir, "mysql-zrm.conf");
		if( -e $backupset_conf ){
			&readOptionsFile( $backupset_conf, \%inputs );
		}
	}

	if( $opt{"options-file"} ) {
		&readOptionsFile( $opt{"options-file"}, \%inputs );
	}

	#Override parameters in the options files
	#by the parameters supplied by the command line

	for (keys%opt) {
		if( $_ ne "options-file" ){
			$inputs{$_}=$opt{$_};
		}
	}
}

#read where the last backup is stored
sub processLastBackupDir()
{
	$lastBackupDir=&getLastBackupDir();
	if( $lastBackupDir && ! -d $lastBackupDir ){
		if( $action ne "check" ){
			&printWarning( "last backup directory is not valid\n" );
		}
		$lastBackupDir = "";
	}
}

#Do command line parameter processing
#@_ list of options specific to application calling this routine
sub processCommandLineParams()
{
	my $ret=0;
	$ret = GetOptions( \%opt,
		@COMMON_OPTIONS,
		@_
             );
	if( $opt{"verbose"} ){
		$verbose = 1;
		delete $opt{"verbose"};
	}
	if( $opt{"quiet"} ){
		$inputs{"quiet"} = $opt{"quiet"};
		delete $opt{"quiet"};
	}
	if( $opt{"schedule-file"} ){
		$inputs{"schedule-file"} = $opt{"schedule-file"};
		delete $opt{"schedule-file"};
	}
	if( $opt{"schedule-string"} ){
		$inputs{"schedule-string"} = $opt{"schedule-string"};
		delete $opt{"schedule-string"};
	}
	unless ( $ret ) {
	    	&usage("Invalid Parameters");
	}

	my $rem=@ARGV;
	if( $rem > 0 ){
    		&usage("Unknown Parameters @ARGV\n");
	}

	if($opt{"help"}) {
		&usage("");
	}

}

sub createConfigFile()
{
#	if( $action eq "check" ){
		#return;
	#}
	$confFileHandle = new File::Temp(TEMPLATE=>"tmpXXXXX",DIR=>$backupset_dir,SUFFIX=>".conf", UNLINK=>1);
	$ENV{'ZRM_CONF'} = "$confFileHandle";
	if( $verbose ){
		&printLog( "ZRM Temporary configuration file = $confFileHandle\n" );
	}
	if( $verbose ){
		&printLog( "{\n" );
	}
	for(keys%inputs) {
		print $confFileHandle "$_=$inputs{$_}\n";
		if( $verbose ){
                        if( $_ ne "password" && !($_ =~ /-password/)  && !($_ =~ /-secret-key/) ){
                                &printLog( "\t".$_."=".$inputs{$_}."\n" );
                        }else{
                                &printLog( "\t".$_."="."******"."\n" );
                        }
		}
	}
	if( $verbose ){
		print $confFileHandle "verbose=1\n";
		&printLog( "}\n" );
	}
	close( $confFileHandle );
}

# Opens the required log file
sub openLogFile()
{
	if( !-d $LOGNAME_LOCATION ) {
         	mkpath ($LOGNAME_LOCATION);
        }
	if( open (LOG, ">>$LOGNAME") ){
		my $h = select LOG;
		$| = 1;
		select STDOUT;
		$| = 1;
		select STDERR;
		$| = 1;
		$log_open=1;
		select $h;
	}else{
		if( $action ne "check" ){
			die( "ERROR: Unable to open logfile $LOGNAME.\n" );
		}else{
			print STDERR "ERROR: Unable to open logfile $LOGNAME.\n";
		}
	}
}

sub processTmpDir()
{
        if( $inputs{"tmpdir"} ){
                $TMPDIR = $inputs{"tmpdir"};
                if( ! -d $TMPDIR ){
                        &printAndDie( "Temporary directory $TMPDIR does not exist\n" );
                }
                if( ! -w $TMPDIR || ! -r $TMPDIR || ! -x $TMPDIR ){
                        &printAndDie( "Temporary directory $TMPDIR does not have sufficient permissions\n" );
                }
                $ENV{'TMPDIR'}=$TMPDIR;
        }else{
                $TMPDIR=File::Spec->tmpdir();
        }
        $LOGGER = tmpnam();
}

sub processDirPaths()
{
        if( $inputs{"destination"} ){
                $inputs{"destination"} = &removeExtraSlashesFromPath( $inputs{"destination"} );
        }
        if( $inputs{"source-directory"} ){
                $inputs{"source-directory"} = &removeExtraSlashesFromPath( $inputs{"source-directory"} );
        }
}

sub notifyMonitor()
{
	if( defined $notifyRoutine ){
		my $r = &$notifyRoutine( @_ );
		if( defined $r && $_[0] =~ /ZRM_STOPPED/ ){
			close $r;
		}
	}
	return 1;
}

sub processCopyPlugin()
{
	if( ! $onWindows ){
		return;
	}

	if( !defined $inputs{"copy-plugin"} ){
		$inputs{"copy-plugin"} = catfile( $ZRM_HOME, "usr\\share\\mysql-zrm\\plugins\\windows-copy.pl" );
		if( $verbose ){
			&printLog( "Using ".$inputs{"copy-plugin"}." as the default plugin since copy-plugin has not been specified\n" );
		}
	}
}


sub verifySnapshotParams()
{
	if( ! defined $inputs{"snapshot-plugin"} || $inputs{"backup-level"} == 1 ){
		return;
	}
	my $cmd = $inputs{"snapshot-plugin"};
        $cmd .= " --action verify-config";
	$cmd .= " 2>$LOGGER";
        if( $verbose ){
                &printLog( "Running snapshot verification using command $cmd\n" );

        }
	my $out = &execCmdAndGetOutput( $cmd, "NO ERROR" );
	if( ! defined $out ){
		&printCommandOutputToLog( "ERROR", "Snapshot Parameter Verification", $LOGGER );
		&printAndDie( "Verification of snapshot parameters failed\n" );	
	}
}

# gets zrm version, open logs files, process command line params and
# processes last backup dir
# @_ list of acceptable command line paramneters
sub initCommon()
{
        &getZrmVersion();
        &openLogFile();
	&notifyMonitor($action."_ZRM_STARTED");
        if( $verbose ){
                &printLog( "action being performed is ".$action."\n" );
        }
        &processCommandLineParams(@_);
        &processOptionFiles();
	&processTmpDir();
	&processDirPaths();
        &processLastBackupDir();
	&processCopyPlugin();
}

1;
__END__
