#!/bin/sh
# $1 [$2 $3 $4] = Schedule0...ScheduleN [ user_name ip_addr host_name ]
# $1 $2 = Schedule0 [job uuid] (HBS3)
#
QPKG_NAME="HybridBackup"
QTS_QPKG_CONF="/etc/config/qpkg.conf"
QPKG_INSTALL_PATH=$(getcfg ${QPKG_NAME} Install_Path -f ${QTS_QPKG_CONF})
SMB_CONF="/etc/smb.conf"
RSYNC_CMD="/mnt/ext/opt/HybridBackup/bin/rsync"
RSYNC_DEF_CONF="${QPKG_INSTALL_PATH}/rsync_default.conf"

STDOUT_LOG=yes
# debug flag for enable the rsync verbose log
# it is in the /mnt/ext/opt/HybridBackup/data/system/nas/[job uuid]/rsync-xxx-xxx
DEBUG=no

MSGID_JOB_STARTED="8"
MSGID_JOB_STOPPED_BY_USER="10"
MSGID_JOB_STOPPED_BY_SYSTEM="11"
MSGID_JOB_CANCELLED="13"
MSGID_JOB_FINISHED="15"
MSGID_JOB_FINISHED_WARNING="20"
MSGID_JOB_FAILED="40"
MSGID_JOB_FAILED_RETRY="84"
MSGID_JOB_CUSTOM_FINISHED="120"
MSGID_JOB_CUSTOM_FAILED="99"
MSGID_JOB_CUSTOM_FINISHED_WARNING="126"
MSGID_JOB_NET_ADAPTER="131"
MSGID_JOB_DEFAULT_ROUTE="135"

NCLOG_MSGS[${MSGID_JOB_STARTED}]="[{0}] Started {1} job: \"{2}\"."
NCLOG_MSGS[${MSGID_JOB_STOPPED_BY_USER}]="[{0}] User stopped {1} job: \"{2}\"."
NCLOG_MSGS[${MSGID_JOB_STOPPED_BY_SYSTEM}]="[{0}] System stopped {1} job: \"{2}\"."
NCLOG_MSGS[${MSGID_JOB_CANCELLED}]="[{0}] Canceled {1} job: \"{2}\". A previous job hasn't finished yet."
NCLOG_MSGS[${MSGID_JOB_FINISHED}]="[{0}] Finished {1} job: \"{2}\"."
NCLOG_MSGS[${MSGID_JOB_FINISHED_WARNING}]="[{0}] Finished {1} job \"{2}\" with errors. {3}"
NCLOG_MSGS[${MSGID_JOB_FAILED}]="[{0}] Failed to complete {1} job: \"{2}\". {3}"
NCLOG_MSGS[${MSGID_JOB_FAILED_RETRY}]="[{0}] Failed to complete {1} job: \"{2}\". {3} Attempting to retry. Error code: {4}."

NCLOG_MSGS[${MSGID_JOB_CUSTOM_FINISHED}]="[{0}] Finished {1} job: \"{2}\". Folder Pairs: {3}, Total Files: {4}, Unchanged Files: {5}, Skipped Files: {6}, {7}: {8}, Total File Size: {9}, Average Transmission Speed: {10}."
NCLOG_MSGS[${MSGID_JOB_CUSTOM_FINISHED_WARNING}]="[{0}] Finished {1} job \"{2}\" with errors. {3}"
NCLOG_MSGS[${MSGID_JOB_CUSTOM_FAILED}]="[{0}] Failed to complete {1} job: \"{2}\". {3}. Check logs for more information."
NCLOG_MSGS[${MSGID_JOB_NET_ADAPTER}]="[{0}] {1} job \"{2}\": Using network interface \"{3}\". Interface selection: {4}."
NCLOG_MSGS[${MSGID_JOB_DEFAULT_ROUTE}]="[{0}] {1} job \"{2}\": Interface selection: Default route."

NC_LEVEL_INFO="info"
NC_LEVEL_WARN="warn"
NC_LEVEL_ERROR="error"

ERROR_DESC[0]="Success."
ERROR_DESC[1]="Unexpected error."
ERROR_DESC[2]="Success with warnings" # bug 132297
ERROR_DESC[41]="Remote host does not exist." #retry
ERROR_DESC[42]="Connection refused. Check status and connection settings for remote service." #retry
ERROR_DESC[43]="Failed to authenticate credentials. Check username and password."
ERROR_DESC[44]="Destination folder path does not exist."
ERROR_DESC[45]="Connection reset by peer. Check status of remote backup service." #retry
ERROR_DESC[46]="Connection to remote host timed out. Check network connection." #retry
ERROR_DESC[47]="Failed to locate all source files. Check if source files were moved during backup." #retry

ERROR_DESC[48]="Replication job has been terminated by user."
ERROR_DESC[49]="The source folder path does not exist." #Failed to locate source folder. Check folder path.
ERROR_DESC[50]="The previous scheduled job has not been finished yet."

ERROR_DESC[51]="Insufficient storage space. Increase storage capacity or remove files." # bug 112665
ERROR_DESC[52]="Encountered I/O error for source data." # bug 107947
ERROR_DESC[53]="Encountered error from rsync protocol data stream. Check storage space at destination." # bug 112665 , rsyncd restart
ERROR_DESC[54]="File system is set to \"read-only\" or \"read/delete\"." # bug 115736
ERROR_DESC[55]="ACLs are not supported on this server" # retry, bug 103237 116220
ERROR_DESC[56]="Replication job has been terminated by system." # bug 132266
ERROR_DESC[57]="The selected source folders are locked. Unlock the storage space from File Station to access it" #spec20996
ERROR_DESC[58]="The destination folder path does not exist."
ERROR_DESC[59]="The selected destination folders are locked. Unlock the storage space from File Station to access it" #spec20996


NCLOG_MSGID_VARS[1]=${ERROR_DESC[1]}
NCLOG_MSGID_VARS[41]="%V012%"
NCLOG_MSGID_VARS[42]="%V013%"
NCLOG_MSGID_VARS[43]="%V014%"
NCLOG_MSGID_VARS[44]="%V194%"
NCLOG_MSGID_VARS[45]="%V016%"
NCLOG_MSGID_VARS[46]="%V017%"
NCLOG_MSGID_VARS[47]="%V018%"
# NCLOG_MSGID_VARS[48] is MSGID_JOB_STOPPED_BY_USER
NCLOG_MSGID_VARS[49]="%V019%"
# NCLOG_MSGID_VARS[50] is MSGID_JOB_CANCELLED
NCLOG_MSGID_VARS[51]="%V020%"
NCLOG_MSGID_VARS[52]="%V021%"
NCLOG_MSGID_VARS[53]="%V022%"
NCLOG_MSGID_VARS[54]="%V023%"
NCLOG_MSGID_VARS[55]="%V001%"
NCLOG_MSGID_VARS[57]="%V178%"
NCLOG_MSGID_VARS[205]="%V205%"
NCLOG_MSGID_VARS[206]="%V206%"
NCLOG_MSGID_VARS[58]="%V015%"
NCLOG_MSGID_VARS[59]="%V179%"

Echo_Time()
{
	echo "[$(date '+%Y/%m/%d %H:%M:%S')] $@"
}

Debug_Info()
{
	[ "$STDOUT_LOG" == "no" ] && return 0
	[ "$SHELL_OUTPUT" == "/dev/null" ] && return 0
	if [ -f "$SHELL_OUTPUT" ]; then
		Echo_Time "[I] L${BASH_LINENO[0]} $@" >> $SHELL_OUTPUT
	else
		Echo_Time "[I] L${BASH_LINENO[0]} $@"
	fi
}

bytes_to_human() {
	# converts a byte count to a human readable format in IEC binary notation.
	local _bytes="${1}"
	local _speed="${2:-size}"
	local _result=$(awk -v bytes="${_bytes}" -v speed="${_speed}" 'function human(x, speed) {
			base=1024
			s="BKMGTEPYZ"
			while (x>=base && length(s)>1)
				{x/=base; s=substr(s,2)}
			s=substr(s,1,1)
			if (speed=="size") {
				xf=((s=="B") ? "%d" : "%.2f")
			} else {
				xf=((s=="B") ? "%d" : "%.1f")
			}
			s=((s=="B") ? s : (s "B"))
			s=((speed=="size") ? s : (s "/s"))
			return sprintf((xf " %s"), x, s)
		}
		BEGIN{print human(bytes, speed)}')
	eval "$3='$_result'"
}

nclog()
{
	# $1 - message level
	# $2 - message id
	# $3 - job name
	# $4 - extra error desc id, only when message id is 40
	Debug_Info "$FUNCNAME( $@ ) ..."
	local _level=$1
	local _msgid=$2
	local _jobname=$3
	local _varsno=$4
	local _ext_words=$5
	local _defmsg=""

	local SH_RR_NCSEND="/mnt/ext/opt/HybridBackup/scripts/nc_adapter/rr_ncsend_hbs3.sh"
	if [ ! -f "${SH_RR_NCSEND}" ]; then # when qpkg stopping
		SH_RR_NCSEND="${QPKG_INSTALL_PATH}/scripts/nc_adapter/rr_ncsend_hbs3.sh"
	fi

	local _arg_uuid=""
	if [ ! -z "${RR_UUID}" ]; then
		if [ "$_msgid" == "${MSGID_JOB_CUSTOM_FINISHED_WARNING}" ] || \
			[ "$_msgid" == "${MSGID_JOB_CUSTOM_FAILED}" ] || \
			[ "$_msgid" == "${MSGID_JOB_CUSTOM_FINISHED}" ]; then
			_arg_uuid="-U none"
		else
			_arg_uuid="-U ${RR_UUID}"
		fi
	fi

	_defmsg="${NCLOG_MSGS[${_msgid}]}"
	if [ "$_msgid" == "${MSGID_JOB_FAILED}" ] || [ "$_msgid" == "${MSGID_JOB_CUSTOM_FAILED}" ]; then
		${SH_RR_NCSEND} ${_arg_uuid} -u "System" -l $_level -m $_msgid \
				"$_defmsg" "$app_name" "$service_type" "$_jobname" "${NCLOG_MSGID_VARS[${_varsno}]}"
	elif [ "$_msgid" == "${MSGID_JOB_FAILED_RETRY}" ]; then
		${SH_RR_NCSEND} ${_arg_uuid} -u "System" -l $_level -m $_msgid \
				"$_defmsg" "$app_name" "$service_type" "$_jobname" "${NCLOG_MSGID_VARS[${_varsno}]}" "${_varsno}"
	elif [ "$_msgid" == "${MSGID_JOB_FINISHED_WARNING}" ] || [ "$_msgid" == "${MSGID_JOB_CUSTOM_FINISHED_WARNING}" ]; then
		${SH_RR_NCSEND} ${_arg_uuid} -u "System" -l $_level -m $_msgid \
				"$_defmsg" "$app_name" "$service_type" "$_jobname" "${NCLOG_MSGID_VARS[0]}"
	elif [ "$_msgid" == "${MSGID_JOB_NET_ADAPTER}" ]; then
		${SH_RR_NCSEND} ${_arg_uuid} -u "System" -l $_level -m $_msgid \
				"$_defmsg" "$app_name" "$service_type" "$_jobname" "${_ext_words}" "${NCLOG_MSGID_VARS[${_varsno}]}"
	elif [ "$_msgid" == "${MSGID_JOB_DEFAULT_ROUTE}" ]; then
		${SH_RR_NCSEND} ${_arg_uuid} -u "System" -l $_level -m $_msgid \
				"$_defmsg" "$app_name" "$service_type" "$_jobname"
	elif [ "$_msgid" == "${MSGID_JOB_CUSTOM_FINISHED}" ]; then
		_total_files=$(/sbin/getcfg "${RR_sche}" "TotalFiles" -f "${RR_CONF}" -d "0")
		_unchanged_files=$(/sbin/getcfg "${RR_sche}" "SkippedFiles" -f "${RR_CONF}" -d "0")
		_skipped_files=0
		_synced_files=$(/sbin/getcfg "${RR_sche}" "SyncedFiles" -f "${RR_CONF}" -d "0")
		_total_file_size=$(/sbin/getcfg "${RR_sche}" "TotalFileSize" -f "${RR_CONF}" -d "0")
		_speed=$(/sbin/getcfg "${RR_sche}" "TransmitSpeed" -f "${RR_CONF}" -d "0")
		bytes_to_human ${_total_file_size} "size" _hsize
		bytes_to_human ${_speed} "speed" _hspeed
		${SH_RR_NCSEND} ${_arg_uuid} -u "System" -l $_level -m $_msgid \
				"$_defmsg" "$app_name" "$service_type" "$_jobname" \
				"1" "$_total_files" "$_unchanged_files" "$_skipped_files" \
				"Synced Files" "$_synced_files" "$_hsize" "$_hspeed" 
	else
		${SH_RR_NCSEND} ${_arg_uuid} -u "System" -l $_level -m $_msgid \
				"$_defmsg" "$app_name" "$service_type" "$_jobname"
	fi
}

function is_not_retry_errno() {
	Debug_Info "$FUNCNAME( $@ ) ..."
	local _errno=$1

	array=("0" "2" "43" "44" "48" "49" "50" "51" "52" "53" "54" "56" "57" "58" "59" "144")
	for i in "${array[@]}"
	do
		if [ "$i" -eq "$_errno" ] ; then
			return 0  # true
		fi
	done

	return 1  # false
}

function is_check_locked_errno() {
	Debug_Info "$FUNCNAME( $@ ) ..."
	local _errno=$1

	array=("0" "48" "56")
	for i in "${array[@]}"
	do
		if [ "$i" -eq "$_errno" ] ; then
			return 1  # false
		fi
	done

	return 0  # true
}

# Check if a string contains the specified prefix
# Is_Prefix_Matched string prefix
Is_Prefix_Matched()
{
	cbPrefix=`/usr/bin/expr length "$2"`
	SubStr=`/usr/bin/expr substr "$1" 1 $cbPrefix`
	if [ "$SubStr" == "$2" ]; then
		echo "1"
	else
		echo "0"
	fi
}

# Check if a string contains the specified postfix
# Is_Prefix_Matched string postfix
Is_Postfix_Matched()
{
	cbString=`/usr/bin/expr length "$1"`
	cbPostfix=`/usr/bin/expr length "$2"`
	nbPos=`/usr/bin/expr $cbString - $cbPostfix + 1`
	SubStr=`/usr/bin/expr substr "$1" $nbPos $cbPostfix`
	if [ "$SubStr" == "$2" ]; then
		echo "1"
	else
		nbPos=`/usr/bin/expr $cbString - $cbPostfix`
		SubStr=`/usr/bin/expr substr "$1" $nbPos $cbPostfix`
		if [ "$SubStr" == "$2" ]; then
			echo "1"
		else
			echo "0"
		fi
	fi
}

# Check the rsync error by comparing the key words
# Is_Error_Pattern_Matched string prefix postfix
Is_Error_Pattern_Matched()
{
	if [ "1" == $(Is_Prefix_Matched "$1" "$2") ] && [ "1" == $(Is_Postfix_Matched "$1" "$3") ]; then
		echo "1"
	else
		echo "0"
	fi
}

# Is_Contain_Substring() string substr
Is_Contain_Substring()
{
	[ ${#2} -eq 0 ] && { echo "0"; return; }
	case "$1" in
	*$2*) echo "1";;
	*) echo "0";;
	esac
}

# Filter_Log_Msg() msg_from msg_to
Filter_Log_Msg()
{
	local bRevised=0
	/bin/rm -f "$2" 1>/dev/null 2>/dev/null
	/bin/touch "$2" 1>/dev/null 2>/dev/null
	if [ -s "$1" ]; then
		while read szLine
		do
			if [ "1" == $( Is_Contain_Substring "$szLine" "Warning: Permanently added" ) ]; then
				continue
			elif [ "1" == $( Is_Error_Pattern_Matched "$szLine" "rsync: mkstemp " " Invalid argument (22)" ) ]; then
				continue
			elif [ "1" == $( Is_Error_Pattern_Matched "$szLine" "rsync: recv_generator: mkdir " " Invalid argument (22)" ) ]; then
				continue
			elif [ "1" == $( Is_Contain_Substring "$szLine" "*** Skipping any contents from this failed directory ***" ) ]; then
				continue
			elif [ "1" == $( Is_Error_Pattern_Matched "$szLine" "Could not create directory" "admin/.ssh'." ) ]; then
				continue
			elif [ "1" == $( Is_Error_Pattern_Matched "$szLine" "Failed to add the host to the list of known hosts" ".ssh/known_hosts)." ) ]; then
				continue
			elif [ "1" == $( Is_Error_Pattern_Matched "$szLine" "Could not chdir to home directory" "No such file or directory" ) ]; then
				continue #bug102995
			elif [ "1" == $( Is_Error_Pattern_Matched "$szLine" "This System is for the use of authorized users only" ) ]; then
				continue #spec12735 ES rsync over ssh send this message
			elif [ "1" == $( Is_Contain_Substring "$szLine" "(Server) Protocol versions:" ) ]; then
				continue #20171108 skip this that TS-228A to ES
			elif [ "1" == $( Is_Contain_Substring "$szLine" "ACLs are not supported" ) ]; then
				#bug 103237, Revise the warning message readable
				szLine="Replicate ACLs are not supported by the rsync server. Job will be performed without ACLs."
			elif [[ ( $bRevised -eq 0 && "1" == $( Is_Contain_Substring "$szLine" "ACL_TYPE_ACCESS" ) ) && \
				    ( "1" == $( Is_Contain_Substring "$szLine" "Operation not supported" )  || \
				      "1" == $( Is_Contain_Substring "$szLine" "Invalid argument" ) ) ]]; then
				#bug 116220, append once the readable message in the err file "/etc/logs/rsync/rsync-err-Schedule[?]"
				#for handle by Get_First_Error_Msg()
				echo "Replicate ACLs are not supported by the rsync server. Job will be performed without ACLs." >> "$2"
				bRevised=1
			fi
			Debug_Info "Filter_Log_Msg keep line: $szLine"
			/bin/echo "$szLine" >> "$2"
		done < "$1"
	fi
}

# SetStatus() "Schedule${i}" 2 
SetStatus()
{
	local iCnt=0
	while [ $iCnt -le 5 ]
	do
		iCnt=$(($iCnt+1))
 		/sbin/setcfg "${1}" "Status" "${2}" -f "${RR_CONF}"
		if [ $? = 0 ]; then
			break
		else
			/bin/sleep 1
		fi
	done
}

# Get a message line from the error log file.
Get_First_Error_Msg()
{
	SZLINE=""
	
	#//Tom_change, for #bug62855, 2015/06/12
	#
	#line0^$  <---empty line
	#line1:rsync: writefd_unbuffered failed to write 4 bytes to socket [sender]: Connection reset by peer (104)  
	#line2:rsync: connection unexpectedly closed (204 bytes received so far) [sender]
	#line3:rsync error: error in rsync protocol data stream (code 12) at io.c(601) [sender=3.0.7]
	#we want to parse line0 as error log

	# QTSHBS00-312
	grep -v -e "Operation not permitted (1)" -e '^$' "$1" > "${1}.strip"
	while read szLine
	do
		if [ "1" == $( Is_Error_Pattern_Matched "$szLine" "rsync: mkstemp " " Invalid argument (22)" ) ]; then
			continue
		fi

		if [ "1" == $( Is_Contain_Substring "$szLine" "Warning: " ) ]; then
			continue
		fi

		if [ "1" == $( Is_Contain_Substring "$szLine" "some files/attrs were not transferred" ) ]; then
			continue
		fi

		if [ "1" == $( Is_Error_Pattern_Matched "$szLine" "rsync: recv_generator: mkdir " " Invalid argument (22)" ) ]; then
			continue
		fi

		if [ "1" == $( Is_Contain_Substring "$szLine" "*** Skipping any contents from this failed directory ***" ) ]; then
			continue
		fi

		if [ "1" == $( Is_Contain_Substring "$szLine" "could not make way" ) ]; then
			# verify bug 112665 got "could not make way for new symlink: xxxx"
			# the error reason is not this line, skip this
			continue
		fi

		if [ "1" == $( Is_Contain_Substring "$szLine" "get 1024 bytes share memory" ) ]; then
			if is_remote2local; then
				continue # QTSHBS00-3682 active sync to skip this error
			fi
		fi

		if [ "1" == $( Is_Contain_Substring "$szLine" "ACLs are not supported" ) ]; then
			skipLineNum=0
			RR_EXTAAR="-X"
			RR_EXATTR_SERVER_SUPPORT="0"
			# bug 103237: not skip this message for handle by Filter_Log_Msg()
			# bug 116220 ACL to TS and ES to do fail retry without ACL
		fi

		#//Tom_change, for #bug62855, 2015/06/12
		if [ "$szLine" == "" ]; then
			continue
		fi			
		#//-/	

		SZLINE="$szLine"
		break
	done < "${1}.strip"
}

# Get reason of a message line from the error log file when has "Connection reset by peer"
Get_Reason_Connection_Reset_By_Peer()
{
	iRet=0
	while read _szLine
	do
		if [ "1" == $( Is_Contain_Substring "$_szLine" "No space left on device" ) ]; then
			iRet=51
		fi
	done < "$1"
	return $iRet
}

Revise_Error_Log_In_First()
{
	case "$iError" in
		44)
			echo "changdir \"${RR_remote_path}\" failed: No such file or directory" > $SZF_ERROR
			;;
	esac
}

# Check the rsync error code by parsing the rsync error log file. Will set error code to $iError.
# Parse_Error_Log_File rsync_error_log_file
Parse_Error_Log_File()
{
	SZLINE=""
	Get_First_Error_Msg "$1"

	#//Tom_change, for debug
	#echo "In Parse_Error_Log_File $SZLINE"  >> /share/Public/rsyncRR.log 2>&1
	#//-/			

	if [ "x" == "x$SZLINE" ]; then
		iError=0
	elif [ "1" == $( Is_Error_Pattern_Matched "$SZLINE" "rsync: failed to connect to " " No route to host (113)" ) ]; then
		iError=41
		Debug_Info ERROR_NO_ROUTE_TO_HOST
	elif [ "1" == $( Is_Error_Pattern_Matched "$SZLINE" "rsync: failed to connect to " " Invalid argument (22)" ) ]; then
		iError=41
		Debug_Info ERROR_NO_ROUTE_TO_HOST
	elif [ "1" == $( Is_Error_Pattern_Matched "$SZLINE" "ssh: connect to host " ": No route to host" ) ]; then
		iError=41
		Debug_Info ERROR_NO_ROUTE_TO_HOST
	elif [ "1" == $( Is_Error_Pattern_Matched "$SZLINE" "rsync: failed to connect to " " Connection refused (111)" ) ]; then
		iError=42
		Debug_Info ERROR_NO_REMOTE_SERVICE
	elif [ "1" == $( Is_Error_Pattern_Matched "$SZLINE" "ssh: connect to host " ": Connection refused" ) ]; then
		iError=42
		Debug_Info ERROR_NO_REMOTE_SERVICE
	elif [ "1" == $(Is_Prefix_Matched "$SZLINE" "@ERROR: auth failed on module") ]; then
		iError=43
		Debug_Info ERROR_AUTH_FAILED
	elif [ "1" == $(Is_Prefix_Matched "$SZLINE" "@ERROR: Unknown module") ]; then
		iError=44
		Revise_Error_Log_In_First;
		Debug_Info ERROR_PATH_NOT_FOUND
	elif [ "1" == $(Is_Prefix_Matched "$SZLINE" "@ERROR: chroot failed") ]; then
		iError=44
		Revise_Error_Log_In_First;
		Debug_Info ERROR_PATH_NOT_FOUND
	elif [ "1" == $( Is_Prefix_Matched "$SZLINE" "rsync: read errors mapping" ) ]; then
		iError=52
		Debug_Info ERROR_SOURCE_IO_ERROR
	elif [ "1" == $( Is_Error_Pattern_Matched "$SZLINE" "rsync: rsync_xal_get: " " Input/output error (5)" ) ]; then
		iError=144
		Debug_Info ERROR_HANDLE_ACL_ERROR
	elif [ "1" == $( Is_Error_Pattern_Matched "$SZLINE" "rsync: " " Input/output error (5)" ) ]; then
		iError=44
		Debug_Info ERROR_PATH_NOT_FOUND
	elif [ "1" == $( Is_Error_Pattern_Matched "$SZLINE" "rsync: " " No such file or directory (2)" ) ]; then
		iError=44
		Debug_Info ERROR_PATH_NOT_FOUND
	elif [ "1" == $( Is_Error_Pattern_Matched "$SZLINE" "rsync: " " Connection reset by peer (104)" ) ]; then
		# bug 112665 add function to parse reason
		Get_Reason_Connection_Reset_By_Peer "$1"
		local _iReasonError=$?
		if [ x"$_iReasonError" != x"0" ]; then
			iError=$_iReasonError
		elif [ x"$RR_result" == x"12" ]; then
			if is_remote2local; then
				iError=45
			elif [ "$RR_MODE" == "0" ] || [ "$RR_MODE" == "1" ]; then
				iError=53
			else
				iError=45
			fi
		else
			iError=45
			Debug_Info ERROR_CONN_RESET
		fi
	elif [ "1" == $( Is_Prefix_Matched "$SZLINE" "rsync error: error in socket IO (code 10)" ) ]; then
		iError=46
		Debug_Info ERROR_SOCKET_IO_TIMEOUT
	elif [ "1" == $( Is_Error_Pattern_Matched "$SZLINE" "io timeout after " " exiting" ) ]; then
		iError=46
		Debug_Info ERROR_CONN_TIMEOUT
	elif [ "1" == $(Is_Prefix_Matched "$SZLINE" "file has vanished: ") ]; then
		iError=47
		Debug_Info ERROR_FILE_NOT_EXISTED
	elif [ "1" == $(Is_Prefix_Matched "$SZLINE" "rsync: cancelled by system") ]; then
		iError=56
		Debug_Info ERROR_OPER_SYS_CANCELLED
	elif [ "1" == $(Is_Prefix_Matched "$SZLINE" "rsync: cancelled by user") ]; then
		iError=48
		Debug_Info ERROR_OPER_USER_CANCELLED
	elif [ "1" == $(Is_Prefix_Matched "$SZLINE" "rsync error: received SIGINT, SIGTERM, or SIGHUP") ]; then
		iError=48
		Debug_Info ERROR_OPER_USER_CANCELLED
	elif [ "1" == $(Is_Prefix_Matched "$SZLINE" "ERROR: ${ERROR_DESC[49]}") ]; then
		iError=49
		Debug_Info ERROR_SOURCE_PATH_NOT_FOUND
	elif [ "1" == $( Is_Error_Pattern_Matched "$SZLINE" "rsync: " "Read-only file system (30)" ) ]; then
		# bug 115736 revise qts log from rsync error log
		iError=54
		ERROR_DESC[54]=`echo $SZLINE | sed 's/Read-only file system (30)/Read-only or read\/delete file system (30)/g'`
		Debug_Info ERROR_READ_ONLY_FILE_SYSTEM
	elif [ "1" == $( Is_Contain_Substring "$szLine" "ACLs are not supported" ) ]; then
		iError=55
		Debug_Info ERROR_ACLS_NOT_SUPPORT
	elif [ "1" == $(Is_Prefix_Matched "$SZLINE" "ERROR: ${ERROR_DESC[57]}") ]; then
		iError=57
		Debug_Info ERROR_SOURCE_ENCRYPTED_LOCKED
	elif [ "1" == $(Is_Prefix_Matched "$SZLINE" "ERROR: ${ERROR_DESC[58]}") ]; then
		iError=58
		Debug_Info ERROR_DEST_PATH_NOT_FOUND
	elif [ "1" == $(Is_Prefix_Matched "$SZLINE" "ERROR: ${ERROR_DESC[59]}") ]; then
		iError=59
		Debug_Info ERROR_DEST_ENCRYPTED_LOCKED
	elif [ "1" == $( Is_Contain_Substring "$SZLINE" "No space left on device" ) ]; then
		iError=51
		Debug_Info ERROR_NO_SPACE
	else
		iError=1
		ERROR_DESC[1]="$SZLINE"
		NCLOG_MSGID_VARS[1]="Encountered error \"$SZLINE\""
		Debug_Info "ERROR_UNEXPECTED: $SZLINE"
	fi
}

stop_network()
{
	/etc/init.d/smb.sh stop &
	/etc/init.d/atalk.sh stop &
	/etc/init.d/ftp.sh stop &
}

start_network()
{
#	kenttest=0

	/etc/init.d/smb.sh start &
	/etc/init.d/atalk.sh start &
	/etc/init.d/ftp.sh start &
}

# Update the finished status, time, error code and log after remote backup finished.
# Update_Status $schedule $szUser $szAddr $szHost $LastRetry
Update_Status()
{
	if [ "80" == "$RR_result" ]; then
		iError=50
		Debug_Info "ERROR_JOB_ALREADY_RUNNING"
	else
		Parse_Error_Log_File "$SZF_ERROR"
	fi

	# handle finished with warning
	if [ "0" == "$iError" ]; then
		if [ "x${bTake_Snapshot_Fail}" = "xTRUE" ]; then
			iError=2
		fi
	fi

	# spec #20996 check shard folder lock status of source / dest path first
	if is_check_locked_errno ${iError} && is_locked_encrypted_storage "${Local_vol}"; then
		if is_remote2local; then
			sed -i "1s/^/ERROR: ${ERROR_DESC[59]}\n/" "$SZF_ERROR"
			iError=59
		else
			sed -i "1s/^/ERROR: ${ERROR_DESC[57]}\n/" "$SZF_ERROR"
			iError=57
		fi
	fi

	szName=`/sbin/getcfg "$1" "Name" -f "${RR_CONF}" -d ""`
	/sbin/setcfg "$1" "ErrorCode" "$iError" -f "${RR_CONF}"
	szFinishedTime=`/bin/date +%s`
	/sbin/setcfg "$1" "Finished Time" "$szFinishedTime" -f "${RR_CONF}"
	Debug_Info "iError[$iError] ERROR_DESC[$iError]"
	if [ "0" == "$iError" ] || [ "2" == "$iError" ] || [ "144" == "$iError" ]; then
		SetStatus "$1" "2"
		if [ "2" == "$iError" ]; then
			nclog $NC_LEVEL_WARN ${MSGID_JOB_FINISHED_WARNING} "$szName"
			if [ "x${RR_JOB_CUSTOM_FAILED}" == "xTRUE" ]; then
				nclog $NC_LEVEL_WARN ${MSGID_JOB_CUSTOM_FINISHED_WARNING} "$szName"
			fi
		else
			nclog $NC_LEVEL_INFO ${MSGID_JOB_FINISHED} "$szName"
			if [ "x${RR_JOB_CUSTOM_FINISHED}" == "xTRUE" ]; then
				nclog $NC_LEVEL_INFO ${MSGID_JOB_CUSTOM_FINISHED} "$szName"
			fi
		fi
	elif [ "48" == "$iError" ]; then
		SetStatus "$1" "2"
		nclog $NC_LEVEL_INFO ${MSGID_JOB_STOPPED_BY_USER} "$szName"
	elif [ "56" == "$iError" ]; then
		SetStatus "$1" "2"
		nclog $NC_LEVEL_INFO ${MSGID_JOB_STOPPED_BY_SYSTEM} "$szName"
	elif [ "50" == "$iError" ]; then
		SetStatus "$1" "2"
		nclog $NC_LEVEL_INFO ${MSGID_JOB_CANCELLED} "$szName"
	elif is_not_retry_errno ${iError} || [ "x${5}" == "xLASTRETRY" ]; then
		SetStatus "$1" "7"
		nclog $NC_LEVEL_ERROR ${MSGID_JOB_FAILED} "$szName" "$iError"
		if [ "x${RR_JOB_CUSTOM_FAILED}" == "xTRUE" ]; then
			nclog $NC_LEVEL_ERROR ${MSGID_JOB_CUSTOM_FAILED} "$szName" "$iError"
		fi
	else
		SetStatus "$1" "7"
		nclog $NC_LEVEL_ERROR ${MSGID_JOB_FAILED_RETRY} "$szName" "$iError"
	fi
}

Replace_Date()
{
	SZ_RETURN="$1"
	SZ_RETURN=${SZ_RETURN//\$YEAR/`/bin/date +%Y`}
	SZ_RETURN=${SZ_RETURN//\$MONTH/`/bin/date +%m`}
	SZ_RETURN=${SZ_RETURN//\$DAY/`/bin/date +%d`}
	SZ_RETURN=${SZ_RETURN//\$HOUR/`/bin/date +%H`}
	SZ_RETURN=${SZ_RETURN//\$MINUTE/`/bin/date +%M`}
	echo $SZ_RETURN
}

check_rsync_error()
{
	local ret=0
	out_msg=`cut -d ' ' -f 2 ${SZF_OUTPUT}`
	out_msg2=`/usr/bin/expr substr "$out_msg" 1 5`
	if [ $out_msg2 = "auth" ]; then
		ret=1
	fi
	if [ $out_msg2 = "Unknown" ]; then
		ret=2
	fi
	return $ret
}

auto_add_known_hosts()
{
	cat /root/.ssh/known_hosts | grep ${Remote_IP} &> /dev/null
	if [ $? != 0 ]; then
		Debug_Info "add known_host with ${Remote_IP}"
		/usr/bin/ssh -o "StrictHostKeyChecking no" -o PasswordAuthentication=no \
		  -o PubkeyAuthentication=no ${Remote_IP} &>/dev/null
	fi
}

modify_known_hosts()
{
	/bin/ps | /bin/grep ${Remote_IP} > "/tmp/rsync-tmp"
	/bin/grep /usr/bin/ssh "/tmp/rsync-tmp" >> /dev/null
	if [ $? != 0 ]; then
		/usr/bin/ssh-keygen -R ${Remote_IP} 1>/dev/null 2>/dev/null
	fi
}

# stop by system (hbs.sh stop)
function signal_handler_terminate()
{
	Debug_Info "Catch signal TERM/USR2"
	bAbort="1"
	bAbortBySys="1"
	iProcPid=`/sbin/getcfg "${RR_sche}" "PID" -f "${RR_CONF}" -d "0"`
	Debug_Info "try to kill rsync pid [$iProcPid]"
	if [ "$iProcPid" -gt 0 ]; then
		/bin/kill -TERM "$iProcPid"
		# RR_result => 140
	fi
}

# stop by rra
function signal_handler()
{
	Debug_Info "Catch signal"
	bAbort="1"
	iProcPid=`/sbin/getcfg "${RR_sche}" "PID" -f "${RR_CONF}" -d "0"`
	Debug_Info "try to kill rsync pid [$iProcPid]"
	if [ "$iProcPid" -gt 0 ]; then
		/bin/kill -9 "$iProcPid"
	fi
}

# --------------------------
# spec 17529
# snapshot function : begin
# --------------------------

# get the share folder real base path in the full path.
# example: /share/Public/test1 => ${2}=/share/CACHEDEV1_DATA ${3}=Public
#          /share/CACHEDEV1_DATA/Public/testcase => ${2}=/share/CACHEDEV1_DATA ${3}=Public
function get_share_volume_path() {
	local arr_link
	local arr_phys
	IFS=/ read -ra arr_link <<< "$1"
	num="${#arr_link[@]}"
	if [ "${num}" -le "2" ]; then
		# path too shot
		Debug_Info "@@ $FUNCNAME path too shot"
		return 1
	fi
	# skip ${arr_link[0]} that it is empty
	link_volume_path="/${arr_link[1]}/${arr_link[2]}"
	# example: link_volume_path[/share/Public] => tmp_path[CACHEDEV1_DATA/Public]
	#          link_volume_path[/share/CACHEDEV1_DATA/Public] => tmp_path[]
	tmp_path=`readlink ${link_volume_path} 2>/dev/null`
	if [ "$?" != 0 ]; then
		# is real path
		if [ "${arr_link[1]}" == "share" ]; then
			eval "${2}='/share/${arr_link[2]}'"
			if [ ! -z "${3}" ] && [ ! -z "${arr_link[3]}" ]; then
				eval "${3}='${arr_link[3]}'"
			fi
			return 0 #True
		fi
		Debug_Info "@@ not share volume path [${link_volume_path}]"
		return 1
	fi

	IFS=/ read -ra arr_phys <<< "$tmp_path"
	num="${#arr_phys[@]}"
	if [ "${num}" -ne "2" ]; then
		# path too shot
		Debug_Info "@@ $FUNCNAME path too shot"
		return 1
	fi
	eval "${2}='/share/${arr_phys[0]}'"
	if [ ! -z "${3}" ]; then
		eval "${3}='${arr_phys[1]}'"
	fi
	return 0
}

function is_encrypt_shared_folder() {
	local _volume_path
	local _shared_name

	get_share_volume_path "$1" _volume_path _shared_name
	_ret=$?
	if [ "$_ret" == "0" ]; then
		if [ -d "$_volume_path/.__eN__${_shared_name}" ]; then
			return 0 #True
		else
			_ret=1 # False
		fi
	fi
	return $_ret
}

function SU_Volume_Get_Vol_ID_By_Mount_Path() {
	_vol_id=`$RR_snapshot_exe get_volume_id $1`
	_ret=$?
	if [ "$_ret" != "0" ]; then
		return $_ret # False
	fi
	eval "$2='${_vol_id}'"
	return 0 # True
}

function SU_NAS_Snapshot_Volume_Is_Supported() {
	_msg=`$RR_snapshot_exe check_volume $1`
	_ret=$?
	Debug_Info "check_volume ret[$_ret] [$_msg]"
	if [ "$_ret" == "0" ]; then
		return 0 # True
	fi
	return 1 # False
}

function SU_NAS_Snapshot_Create_For_App()
{
	local _vol_id=$1
	local _conf_name="$2"
	local _pid=$3

	# stdout example 1:
	#     /dev/mapper/vg1-snap10001 /mnt/snapshot/1/10001 fuse ro,nosuid,nodev,allow_other,default_permissions 0 0
	#     10001
	# stdout example 2:
	#     /dev/mapper/vg1-snap10001 /mnt/snapshot/1/10001 fuse ro,nosuid,nodev,allow_other,default_permissions 0 0
	#     10001
	_msg_lines=(`$RR_snapshot_exe create_snapshot_for_app ${_vol_id} "${_conf_name}" ${_pid}`)
	if [ $? != 0 ] || [ "${#_msg_lines[@]}" == "0" ]; then
		Debug_Info "$FUNCNAME( $@ ) create_snapshot_for_app fail"
		return 1 # False
	fi

	if [ ${#_msg_lines[@]} -eq 1 ]; then
		eval "$4='${_msg_lines[0]}'"
	else
		eval "$4='${_msg_lines[${#_msg_lines[@]}-1]}'"
	fi
	return 0 # True
}

function SU_NAS_Snapshot_Mount() {
	# stdout example 1:
	#     /dev/mapper/vg1-snap10001 /mnt/snapshot/1/10001 ext4 ro,usrjquota=aquota.user,jqfmt=vfsv0,user_xattr,data=ordered,data_err=abort,delalloc,nopriv,nodiscard,noacl 0 0
	#     /mnt/snapshot/1/10001
	# stdout example 2:
	#     /mnt/snapshot/1/10001
	_msg_lines=(`$RR_snapshot_exe mount_snapshot $1`)
	if [ $? != 0 ] || [ "${#_msg_lines[@]}" == "0" ]; then
		return 1 # False
	fi

	if [ ${#_msg_lines[@]} -eq 1 ]; then
		eval "$2='${_msg_lines[0]}'"
	else
		eval "$2='${_msg_lines[${#_msg_lines[@]}-1]}'"
	fi
	return 0 # True
}

function SU_Create_Snapshot_By_Path() {
	local _job_local_path="$1"
	local _display_name=$2
	local _pid=$3

	$RR_snapshot_exe check_nas 1>/dev/null 2> /dev/null
	_ret=$?
	if [ $_ret != 0 ]; then
		Debug_Info "Not supported in this model."
		return 1 # False
	fi

	if is_encrypt_shared_folder "$_job_local_path"; then
		Debug_Info "$_job_local_path : Is encryption shared folder."
		return 2
	fi
	if [ ! -d "$_job_local_path" ]; then
		Debug_Info "$_job_local_path : No such file or directory."
		return 3
	fi

	_volume_path=""
	_vol_id=""
	_snapshot_id=""
	_snapshot_mntpath=""

	get_share_volume_path "$_job_local_path" _volume_path
	_ret=$?
	if [ "$_ret" != "0" ]; then
		Debug_Info "@@ get_share_volume_path [${_job_local_path}] fail($_ret)"
		return 4 # False
	fi
	Debug_Info "volume_path [${_volume_path}]"

	SU_Volume_Get_Vol_ID_By_Mount_Path "$_volume_path" _vol_id
	_ret=$?
	if [ "$_ret" != "0" ]; then
		Debug_Info "@@ Volume_Get_Vol_ID_By_Mount_Path : fail($_ret)"
		return 5 # False
	fi
	Debug_Info "vol_id [${_vol_id}]"

	SU_NAS_Snapshot_Volume_Is_Supported $_vol_id
	_ret=$?
	if [ "$_ret" != "0" ]; then
		Debug_Info "NAS_Snapshot_Volume_Is_Supported : not support ($_ret)"
		return 6 # False
	fi

	SU_NAS_Snapshot_Create_For_App $_vol_id "${_display_name}" $_pid _snapshot_id;
	_ret=$?
	if [ "$_ret" != "0" ]; then
		Debug_Info "@@ NAS_Snapshot_Create_For_App : fail($_ret)"
		return 7 # False and should as finished with warning. bug132297
	fi
	Debug_Info "snapshot_id [${_snapshot_id}]"

	SU_NAS_Snapshot_Mount ${_snapshot_id} _snapshot_mntpath
	_ret=$?
	if [ "$_ret" != "0" ] || [ -z "$_snapshot_mntpath" ]; then
		Debug_Info "@@ NAS_Snapshot_Mount: fail($_ret)"
		return 8 # False and should as finished with warning. bug132297
	fi
	Debug_Info "snapshot_mntpath [${_snapshot_mntpath}], local_path[${_job_local_path}]"

	# Wait for FileTiers FTFS FUSE overlay to settle after snapshot
	# FTFS magic: 0xB550CA10 = 3041476112
	_fs_id=$(stat -f -c '%t' "$_job_local_path" 2>/dev/null)
	if [ "x${_fs_id}" = "xb550ca10" ]; then
		Debug_Info "FTFS detected on ${_job_local_path} (fs_id=${_fs_id}), sleeping 6 seconds for readiness"
		/bin/sleep 6
	fi

	_tmp_path="${_job_local_path#/*/*/}" # Remove "/share/CACHEDEV1_DATA/" directory components

	eval "$4='${_snapshot_id}'" #snapshot_id
	eval "$5='${_snapshot_mntpath}/${_tmp_path}'" #local path in the snapshot
	return 0 # True
}

function SU_Expire_Snapshot_By_ID() {
	local _snapshot_id=$1
	_msg_lines=`$RR_snapshot_exe set_snapshot_expire $_snapshot_id 1`
	if [ $? != 0 ]; then
		return $? # False
	fi
	return 0 # True
}

function SU_Snapshot_Set_PID() {
	local _snapshot_id=$1
	local _pid=$2
	_msg_lines=`$RR_snapshot_exe set_snapshot_pid $_snapshot_id $_pid`
	if [ $? != 0 ]; then
		return $? # False
	fi
	return 0 # True
}

# the flow to take a snapshot in the job processing
function Job_Take_Snapshot() {
	Debug_Info "$FUNCNAME( $@ ) ..."
	local _display_name="$1" # max length 64 chars

	RR_local_path_real="$RR_local_path" #default
	if is_remote2local; then
		Debug_Info "use path [${RR_local_path_real}]"
		return 0
	fi

	if [ "x${RR_opt_snapshot}" = "xTRUE" ]; then
		RR_snapshot_id="0"

		SU_Create_Snapshot_By_Path "$RR_local_path" "$_display_name" $iPid RR_snapshot_id RR_local_path_real
		_ret=$?
		if [ "$_ret" != "0" ]; then
			Debug_Info "not take snapshot (${_ret})"
			if [ "$_ret" -gt "6" ]; then # bug132297
				return ${_ret} # False
			fi
		else
			Debug_Info "take snapshot id[$RR_snapshot_id]"
		fi
		Debug_Info "use path [${RR_local_path_real}]"
	else
		Debug_Info "disable snapshot [${RR_opt_snapshot}]"
	fi
	Debug_Info "$FUNCNAME( $@ ) done ..."
	return 0 # True
}

function Job_Set_Rsync_Pid() {
	if [ "x${RR_opt_snapshot}" = "xTRUE" ] && [ "x${RR_snapshot_id}" != "x0" ] ; then
		if ! SU_Snapshot_Set_PID ${RR_snapshot_id} ${RR_process_pid}; then
			Debug_Info "set snapshot \"$RR_snapshot_id\" pid \"$RR_process_pid\" fail($?)"
		fi
	fi
}

# --------------------------
# snapshot function : end
# --------------------------

# --------------------------------
# encrypted storage function
# --------------------------------
function is_locked_encrypted_storage() {
	Debug_Info "$FUNCNAME( $@ ) ..."
	. $QPKG_INSTALL_PATH/rr2/scripts/set_py3_env.sh 1>/dev/null
	# --> decide PY or PYC
	if [ -e "$QPKG_INSTALL_PATH/rr2/bin/cli/get_local.pyc" ]; then
		local PY_ENTRY="$QPKG_INSTALL_PATH/rr2/bin/cli/get_local.pyc"
	else
		local PY_ENTRY="$QPKG_INSTALL_PATH/rr2/bin/cli/get_local.py"
	fi

	if [ ! -f "${PY_ENTRY}" ]; then #
		Debug_Info "$FUNCNAME( $@ ) missing get local py"
		return 1 # False
	fi

	ret_lock=`python3 $PY_ENTRY -share_lock "$1"`
	if [ "x$?" = "x0" ]; then
		if [ "x$ret_lock" = "x1" ]; then
			Debug_Info "$FUNCNAME( $@ ) locked"
			return 0 # True
		elif [ "x$ret_lock" = "x0" ]; then
			Debug_Info "$FUNCNAME( $@ ) unlocked"
			return 1 # False
		else
			Debug_Info "$FUNCNAME( $@ ) @@ unexcept value [$ret_lock]"
			return 1 # False
		fi
	else
		Debug_Info "$FUNCNAME( $@ ) @@ run get_local.sh failed"
		return 1 # False
	fi
}

function Job_Pre_Check() {
	Debug_Info "$FUNCNAME( $@ ) ..."
	local _vol_path="$1"

	# spec #20996 check shard folder lock status of source path first
	if is_locked_encrypted_storage "$_vol_path"; then
		if is_remote2local; then
			echo "ERROR: ${ERROR_DESC[59]}" > "$SZF_ERROR"
		else
			echo "ERROR: ${ERROR_DESC[57]}" > "$SZF_ERROR"
		fi
		return 1
	fi
	Debug_Info "$FUNCNAME( $@ ) done ..."
	return 0 # True
}

# --------------------------------
# encrypted storage function : end
# --------------------------------


function QTS_Hero_Extattr_Routine() {
	Debug_Info "current RR_EXTAAR: [${RR_EXTAAR}]"
	[ -z "${RR_EXTAAR}" ] && return

	# Detect is remote qts hero
	if [ "x${RR_opt_SSH}" = "xTRUE" ]; then
		RemoteFS=$(${RSYNC_CMD} \
				--sever-mode=${RR_MODE} \
				-e "${RR_com_ssh} -o LogLevel=ERROR" \
				--rsync-path='/mnt/ext/opt/HybridBackup/bin/rsync' \
				--timeout=${xsTimeout} --enc_password="${EnPasswd}" \
				"${UserName}"@[${Remote_IP}]::"#..QNAP.Query.File.System:${RR_remote_path}" | tr -d '\n')
		_ret=$?
	else
		RemoteFS=$(${RSYNC_CMD} \
				--sever-mode=${RR_MODE} --port=${RR_PORT} \
				--timeout=${xsTimeout} --enc_password="${EnPasswd}" \
				"${UserName}"@[${Remote_IP}]::"#..QNAP.Query.File.System:${RR_remote_path}" | tr -d '\n')
		_ret=$?
	fi

	if [ "${_ret}" != "0" ]; then
		Debug_Info "Receiver use old acl rule"
		return
	fi

	if [ -f "/etc/QTS_ZFS" ]; then
		Debug_Info "Local QuTShero"
		if [ "$RemoteFS" == "zfs" ]; then
			Debug_Info "QuTShero active sync from QuTShero"
			RR_EXTAAR="--acls-hero2hero -X"
		else
			Debug_Info "Different platform, skip sync ACL"
			RR_EXTAAR=""
		fi
	else
		Debug_Info "Local QTS"
		if is_remote2local; then
			if [ "$RemoteFS" == "zfs" ]; then
				Debug_Info "Different platform, skip sync ACL"
				RR_EXTAAR=""
			else
				# keep use -A -X
				Debug_Info "QTS active sync from QTS"
			fi
		else # oneway sync
			if [ "$RemoteFS" == "zfs" ]; then
				Debug_Info "QTS oneway sync to QuTShero"
				RR_EXTAAR="--acls-qts2hero -X"
			else
				# keep use -A -X
				Debug_Info "QTS oneway sync to QTS"
			fi
		fi
	fi
	Debug_Info "RR_EXTAAR as: ${RR_EXTAAR}"
}

function _file_rotation()
{
	_file_path=$1
	_moved=0
	[ x"$_file_path" == "x" ] && return
	[ x"$_file_path" == x"/dev/null" ] && return

	for i in 3 2 1; do # keep max .4 logs
		if [ -f "${_file_path}.${i}" ]; then
			mv "${_file_path}."{$i,$((i+1))}
			_moved=1
		fi
	done
	if [ -f "${_file_path}" ]; then
		mv "${_file_path}" "${_file_path}".1
	elif [ "$_moved" == "1" ]; then
		touch "${_file_path}".1
	fi
}

function Debug_Log_Rotation()
{
	_file_rotation "${SZF_OUTPUT}"
	_file_rotation "${SZF_ERROR_TMP}"
	_file_rotation "${SHELL_OUTPUT}"
	_file_rotation "${RSYNC_LOGFILE}"
}

function Net_Adapter_NcLog()
{
	if [ "${RR_BIND_MODE}" == "auto" ]; then
		nclog $NC_LEVEL_INFO ${MSGID_JOB_NET_ADAPTER} "$szName" "206" "${RR_BIND_DISPNAME}"
	elif [ "${RR_BIND_MODE}" == "default" ]; then
		nclog $NC_LEVEL_INFO ${MSGID_JOB_DEFAULT_ROUTE} "$szName"
	else
		nclog $NC_LEVEL_INFO ${MSGID_JOB_NET_ADAPTER} "$szName" "205" "${RR_BIND_DISPNAME}"
	fi
}

function is_remote2local()
{
	[ "${RR_DIRECTION}" == "remote2local" ] && return 0
	return 1
}

# 20171103 HBS 2.1v4 word proofread
service_type="Sync"
app_name="Hybrid Backup Sync"

# Exit if there is no schedule specified.
RR_sche=${1}
[ "x${RR_sche}" = "x" ] && exit -1

LOGFN_PREFIX="$1" # HBS2 use job session name, HBS3 use UUID from config file
RR_opt_hbsv3=""

# setup user name, ip addres and host name from parameters
szUser="System"
szAddr="127.0.0.1"
szHost="localhost"

if [ $# -ge 2 ]; then # for HBS3 $2 is full path of rsync_schedule.conf
	if [ ! -f "$2" ]; then
		echo "$2: No such rsync job config file."
		exit -1
	else
		RR_CONF="$2"
		if [ -z "$3" ]; then
			RR_UUID=`/sbin/getcfg "${RR_sche}" UUID -f "${RR_CONF}" -d ""`
		else
			RR_UUID="$3"
		fi
		if [ -z "${RR_UUID}" ]; then
			echo "$2: No such key \"UUID\"."
			exit -1
		fi
		LOGFN_PREFIX="${RR_UUID}"
	fi
	RR_opt_hbsv3="--schedule-conf=${RR_CONF} --hbs3-id=${RR_UUID}"
else
	# 20190308 hbs3 #36
	exit -1
fi

# Exit if I am not the first instance launched.
iPid="$$"
iPrevPid=`/sbin/getcfg "$RR_sche" "ScriptPID" -f "${RR_CONF}" -d "0"`
if [ "$iPrevPid" -gt 0 ]; then
	/bin/kill -0 "$iPrevPid" 2> /dev/null
	if [ "$?" -eq "0" ]; then
		szName=`/sbin/getcfg "$RR_sche" "Name" -f "${RR_CONF}" -d ""`
		nclog $NC_LEVEL_INFO ${MSGID_JOB_STARTED} "$szName" # show the started message
		nclog $NC_LEVEL_INFO ${MSGID_JOB_CANCELLED} "$szName" # show the cancelled prev runing job
		exit 0
	fi
fi
/sbin/setcfg "$RR_sche" "ScriptPID" "$iPid" -f "${RR_CONF}"
/sbin/setcfg "$RR_sche" "ErrorCode" "0" -f "${RR_CONF}"

bAbort=0
bAbortBySys=0
trap 'signal_handler' SIGINT
trap 'signal_handler_terminate' SIGTERM
trap 'signal_handler_terminate' SIGUSR2

NOW_TIME=`/bin/date +'%Y%m%d%H%M'`

. /etc/init.d/functions

FindDefVol
_ret=$?
if [ $_ret != 0 ]; then
	echo "Default volume not found."
	SZD_LOGS="/etc/logs/rsync" # this folder is /mnt/HDA_ROOT/.logs/
	SZF_ERROR="$SZD_LOGS/rsync-err-${LOGFN_PREFIX}" # job err log
	SZF_ERROR_TMP="$SZD_LOGS/rsync-tmp-${LOGFN_PREFIX}"
	SHELL_OUTPUT="/dev/null"
	SZF_OUTPUT="/dev/null"
	RSYNC_LOGFILE="/dev/null"
else
	[ ! -d "${QPKG_INSTALL_PATH}" ] && exit 2
	SZF_ERROR_TMP_PATH="${QPKG_INSTALL_PATH}/data/system/nas"
	[ -d "${SZF_ERROR_TMP_PATH}/${LOGFN_PREFIX}" ] || /bin/mkdir -p "${SZF_ERROR_TMP_PATH}/${LOGFN_PREFIX}"
	SZF_ERROR="$SZF_ERROR_TMP_PATH/${LOGFN_PREFIX}/rsync-err" # job err log
	SZF_ERROR_TMP="$SZF_ERROR_TMP_PATH/${LOGFN_PREFIX}/rsync-stderr"

	# 20170704 Matt add for enable the rsync verbose log
	if [ "$STDOUT_LOG" == "yes" ]; then
		SHELL_OUTPUT="${SZF_ERROR_TMP_PATH}/${LOGFN_PREFIX}/shell-stdout"
	else
		SHELL_OUTPUT="/dev/null"
	fi
fi

/bin/mkdir $SZD_LOGS 1>/dev/null 2>/dev/null
/bin/rm -f $SZF_ERROR 1>/dev/null 2>/dev/null
/bin/touch $SZF_ERROR 1>/dev/null 2>/dev/null

# QTSHBS00-1051
_RR_JOB_DEBUG=$(/sbin/getcfg "${RR_sche}" "DebugLog" -f ${RR_CONF} -d "FALSE" -u)
[ "x${_RR_JOB_DEBUG}" = "xTRUE" ] && DEBUG=yes

SZF_OUTPUT="${SZF_ERROR_TMP_PATH}/${LOGFN_PREFIX}/rsync-stdout"
RSYNC_LOGFILE="${SZF_ERROR_TMP_PATH}/${LOGFN_PREFIX}/rsync-logfile"
Debug_Log_Rotation

echo "" > $SHELL_OUTPUT
Debug_Info "use config: $RR_CONF"
RR_name=`/sbin/getcfg "${RR_sche}" Name -f "${RR_CONF}" -d ""`
#[ "x${RR_name}" = "x" ] && exit -1
Remote_IP=`/sbin/getcfg "${RR_sche}" "Remote IP" -f ${RR_CONF} -d "0.0.0.0"`
#[ "x${Remote_IP}" = "x0.0.0.0" ] && exit -1
Remote_IP=`/sbin/get_domain_ip ${Remote_IP}`

Local_vol=`/sbin/getcfg "${RR_sche}" "Local Volume" -f ${RR_CONF} -d ""`
Local_path=`/sbin/getcfg "${RR_sche}" "Local Path" -f ${RR_CONF} -d ""`
SMB_path=`/sbin/getcfg "${Local_vol}" "path" -f ${SMB_CONF} -d ""`
RR_PORT=`/sbin/getcfg "${RR_sche}" "Rsync Port" -f ${RR_CONF} -d "873"`

if [ "x${Local_path}" = "x/" ]
then
	RR_local_path="${SMB_path}"
else
	RR_local_path="${SMB_path}${Local_path}"
fi

Remote_vol=`/sbin/getcfg "${RR_sche}" "Remote Volume" -f ${RR_CONF} -d ""`
Remote_path=`/sbin/getcfg "${RR_sche}" "Remote Path" -f ${RR_CONF} -d ""`
# QTSHBS00-4195
#Remote_path=$(Replace_Date "$Remote_path")
if [ "x${Remote_path}" = "x/" ]
then
	RR_remote_path="${Remote_vol}"
else
	RR_remote_path="${Remote_vol}${Remote_path}"
fi

# Verify User/Pwd through samba  /  added by KenChen 20060418
UserName=`/sbin/getcfg "${RR_sche}" "User Name" -f ${RR_CONF} -d ""`
EnPasswd=`/sbin/getcfg "${RR_sche}" "EnPassword" -f ${RR_CONF} -d ""`

RR_options="--info=progress --stats " # QTSHBS00-1421
RR_opt_WanAccel=`/sbin/getcfg ${RR_sche} "WanAccel" -f ${RR_CONF} -d "FALSE" -u`
if [ "x${RR_opt_WanAccel}" = "xTRUE" ]; then
	_bbr=$(/mnt/ext/opt/HybridBackup/bin/qts_bbr check)
	[ "${_bbr}" == "0" ] && RR_opt_WanAccel="FALSE"
fi
RR_opt_QWanTag=`/sbin/getcfg ${RR_sche} "QWanTag" -f ${RR_CONF} -d "FALSE" -u`
#20171016 spec 7331 snapshot enable / disable
RR_opt_snapshot=`/sbin/getcfg "${RR_sche}" "Snapshot" -f ${RR_CONF} -d "TRUE" -u`
RR_opt_sparse=`/sbin/getcfg "${RR_sche}" "Sparse" -f ${RR_CONF} -d "FALSE" -u`
[ "x${RR_opt_sparse}" = "xTRUE" ] && RR_options=${RR_options}"--sparse "
RR_opt_compress=`/sbin/getcfg "${RR_sche}" "Compressed" -f ${RR_CONF} -d "FALSE" -u`
[ "x${RR_opt_compress}" = "xTRUE" ] && RR_options=${RR_options}"--compress "
RR_opt_sync=`/sbin/getcfg "${RR_sche}" "Delete Extra" -f "${RR_CONF}" -d "FALSE" -u`
[ "x${RR_opt_sync}" = "xTRUE" ] && RR_options=${RR_options}"--delete "
RR_opt_inc=`/sbin/getcfg ${RR_sche} "Incremental" -f ${RR_CONF} -d "FALSE" -u`
[ "x${RR_opt_inc}" = "xFALSE" ] && RR_options=${RR_options}"--whole-file -I "
RR_opt_stopnet=`/sbin/getcfg ${RR_sche} "Stop Net" -f ${RR_CONF} -d "FALSE" -u`

RR_opt_SSH=`/sbin/getcfg "${RR_sche}" "Use of SSH" -f ${RR_CONF} -d "FALSE" -u`
RR_SSH_PORT=`/sbin/getcfg "${RR_sche}" "SSH PORT" -f ${RR_CONF} -d "22"`
RR_com_ssh="/usr/bin/ssh -o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -l ${UserName} -p ${RR_SSH_PORT}"
RR_EXTAAR=
[ x`/sbin/getcfg "${RR_sche}" "EXATTR" -f ${RR_CONF} -d "FALSE" -u` = "xTRUE" ] && RR_EXTAAR="-A -X"
RR_EXATTR_SERVER_SUPPORT="1"
RR_ARCHIVE="-a"
RR_opt_perm=`/sbin/getcfg "${RR_sche}" "PERM" -f ${RR_CONF} -d "TRUE" -u`
[ x${RR_opt_perm} != "xTRUE" ] && RR_ARCHIVE="-a --no-perms --chmod=ugo=rwX"

RR_MODE=`/sbin/getcfg "${RR_sche}" "RSYNC MODE" -f ${RR_CONF} -d "1" -u`
# spec19770
RR_JOB_CUSTOM_FINISHED=$(/sbin/getcfg "${RR_sche}" "CustomEventSuccess" -f ${RR_CONF} -d "FALSE" -u)
RR_JOB_CUSTOM_FAILED=$(/sbin/getcfg "${RR_sche}" "CustomEventFailed" -f ${RR_CONF} -d "FALSE" -u)
# QTSHBS00-1101
RR_BIND_DISPNAME=`/sbin/getcfg "${RR_sche}" "BIND DISPNAME" -f ${RR_CONF} -d "Deafult Route"`
RR_BIND_MODE=`/sbin/getcfg "${RR_sche}" "BIND MODE" -f ${RR_CONF} -d "auto"`
RR_BIND_IFNAME=`/sbin/getcfg "${RR_sche}" "BIND IFNAME" -f ${RR_CONF} -d ""`
RR_BIND_ADDR=`/sbin/getcfg "${RR_sche}" "BIND ADDR" -f ${RR_CONF} -d ""`
# QTSHBS00-1099
RR_DIRECTION=`/sbin/getcfg "${RR_sche}" "DIRECTION" -f ${RR_CONF} -d "local2remote"`

if [ ! -z "${RR_BIND_ADDR}" ]; then
	if [ "x${RR_opt_SSH}" = "xTRUE" ]; then
		RR_com_ssh="${RR_com_ssh} -b ${RR_BIND_ADDR}"
	else
		RR_options=${RR_options}"--address=${RR_BIND_ADDR} "
	fi
	Debug_Info "use adapter [${RR_BIND_DISPNAME}][${RR_BIND_ADDR}]"
fi

[ "x${RR_opt_stopnet}" = "xTRUE" ] && stop_network

RR_snapshot_exe="/sbin/snapshot_util" # spec 17529
if [ ! -f "${RR_snapshot_exe}" ] ; then # for old firmware use qpkg build-in version
	RR_snapshot_exe="${QPKG_INSTALL_PATH}/bin/snapshot_util"
	if [ ! -f "${RR_snapshot_exe}" ] ; then
		_rsync_bin=`readlink /usr/bin/rsync.hbs 2>/dev/null`
		_rsync_bin_dir=`dirname $_rsync_bin`
		RR_snapshot_exe="${_rsync_bin_dir}/snapshot_util"
		Debug_Info "RR_snapshot_exe [${RR_snapshot_exe}]"
		if [ ! -f "${RR_snapshot_exe}" ] ; then
			echo "No snapshot_util file" >> "$SZF_ERROR"
			Update_Status $RR_sche $szUser $szAddr $szHost
			/sbin/setcfg "$RR_sche" "ScriptPID" "0" -f "${RR_CONF}"
			exit -1
		fi
	fi
fi
RR_snapshot_id="0"
RR_process_pid="0"

# set status to "Replicating"
szName=`/sbin/getcfg "$RR_sche" "Name" -f "${RR_CONF}" -d ""`
nclog $NC_LEVEL_INFO ${MSGID_JOB_STARTED} "$szName"
Net_Adapter_NcLog

if [ "x${SMB_path}" = "x" ] || [ ! -d "${RR_local_path}" ]; then
	if is_remote2local; then
		echo "ERROR: ${ERROR_DESC[58]}" > "$SZF_ERROR"
	else
		echo "ERROR: ${ERROR_DESC[49]}" > "$SZF_ERROR"
	fi
	if [ "x${SMB_path}" = "x" ]; then # bug 131683
		echo "changdir \"/share/${Local_vol}${Local_path}\" failed: No such file or directory" >> "$SZF_ERROR"
	else
		echo "changdir \"${RR_local_path}\" failed: No such file or directory" >> "$SZF_ERROR"
	fi
	Update_Status $RR_sche $szUser $szAddr $szHost
	/sbin/setcfg "$RR_sche" "ScriptPID" "0" -f "${RR_CONF}"
	exit 0
fi

#/sbin/setcfg "${RR_sche}" Status 9 -f ${RR_CONF}
SetStatus "${RR_sche}" "9"

LOG_FILE_NAME="/tmp/${RR_name}_`/bin/date +'%Y%m%d%H%M'`.log"
touch "${LOG_FILE_NAME}"

# spec 20996
Job_Pre_Check "${Local_vol}"
if [ "$?" != "0" ]; then
	Update_Status $RR_sche $szUser $szAddr $szHost
	/sbin/setcfg "$RR_sche" "ScriptPID" "0" -f "${RR_CONF}"
	exit 0
fi

iRetry=0
xsTimeout=`/sbin/getcfg "${RR_sche}" "Timeout" -f "${RR_CONF}" -d "600"`
xnRetry=`/sbin/getcfg "${RR_sche}" "RetryTimes" -f "${RR_CONF}" -d "5"`
xsInterval=`/sbin/getcfg "${RR_sche}" "RetryIntervals" -f "${RR_CONF}" -d "60"`

RR_options=${RR_options}"--qnap-bwlimit " # bug 115045
if [ -f "${RSYNC_DEF_CONF}" ]; then
	_opt=$(/sbin/getcfg "Extend" "Option" -f "${RSYNC_DEF_CONF}" -d "")
	Debug_Info "Extend Option: ${_opt}"
	if [ "x{$_opt}" != "x" ]; then
		RR_options=${RR_options}"${_opt} "
	fi

	_user=$(/sbin/getcfg "Extend" "User" -f "${RSYNC_DEF_CONF}" -d "TRUE" -u)
	if [ "${_user}" == "FALSE" ]; then
		RR_options=${RR_options}"--no-u "
	fi

	_group=$(/sbin/getcfg "Extend" "Group" -f "${RSYNC_DEF_CONF}" -d "TRUE" -u)
	if [ "${_group}" == "FALSE" ]; then
		RR_options=${RR_options}"--no-g "
	fi

	_chmod=$(/sbin/getcfg "Extend" "Chmod" -f "${RSYNC_DEF_CONF}" -d "")
	if [ "x${_chmod}" != "x" ]; then
		RR_options=${RR_options}"--chmod=${_chmod} "
	fi

	_devices=$(/sbin/getcfg "Extend" "Devices" -f "${RSYNC_DEF_CONF}" -d "FALSE" -u)
	if [ "${_devices}" == "FALSE" ]; then
		RR_options=${RR_options}"--no-devices "
	fi

	_specials=$(/sbin/getcfg "Extend" "Specials" -f "${RSYNC_DEF_CONF}" -d "FALSE" -u)
	if [ "x${_specials}" == "FALSE" ]; then
		RR_options=${RR_options}"--no-specials "
	fi
else
	RR_options=${RR_options}"--no-D "
fi

case ${RR_MODE} in
  0|1)
	if [ "x${RR_opt_SSH}" = "xTRUE" ]; then
		if [ "${RR_MODE}" == "1" ] && [ "${UserName}" != "admin" ] ; then
			# Probe to support rate limit
			${RSYNC_CMD} --sever-mode=${RR_MODE} --dry-run \
				--rsync-path="${RSYNC_CMD} --config=/etc/rsyncd.conf" \
				-e "${RR_com_ssh}" --qnap-ssh-bwlimit --timeout=${xsTimeout} --enc_password="${EnPasswd}" \
				"${UserName}"@[${Remote_IP}]:: 1>/dev/null 2>/dev/null
			if [ "x$?" = "x0" ]; then
				RR_options=${RR_options}"--rsync-path=${RSYNC_CMD} "
				RR_options=${RR_options}"--qnap-ssh-bwlimit "
			fi
		else
			# spec 15032 use qpkg buildin rsync first
			# example: for debug rsync server log
			#--rsync-path="/mnt/ext/opt/HybridBackup/bin/rsync --log-file=~/ssh-rsync-logfile-${NOW_TIME}.log"
			message=$(${RSYNC_CMD} --sever-mode=${RR_MODE} --dry-run \
				-e "${RR_com_ssh} -o LogLevel=ERROR" --timeout=${xsTimeout} --enc_password="${EnPasswd}" \
				--rsync-path="${RSYNC_CMD}" \
				"${UserName}"@[${Remote_IP}]::"#" 2>&1)
			if [ "x$?" = "x0" ] || [ "1" == $( Is_Contain_Substring "${message}" "Unknown command" ) ]; then
				tmp_ssh_dry_run_rsync_path="--rsync-path='${RSYNC_CMD}'"
				RR_options=${RR_options}"${tmp_ssh_dry_run_rsync_path} "
			fi
			Debug_Info "dryrun: ${message}"
			# bug 226271: for compatible the job that upgrade from HBSv2.1
			${RSYNC_CMD} --sever-mode=${RR_MODE} --dry-run ${tmp_ssh_dry_run_rsync_path} \
				-e "${RR_com_ssh}" --qnap-ssh-bwlimit --timeout=${xsTimeout} --enc_password="${EnPasswd}" \
				"${UserName}"@[${Remote_IP}]:: 1>/dev/null 2>/dev/null
			if [ "x$?" = "x0" ]; then
				RR_options=${RR_options}"--qnap-ssh-bwlimit "
			fi
		fi
	fi

	QTS_Hero_Extattr_Routine;

	RR_opt_server_mode="--sever-mode=${RR_MODE}"
	if [ "x${RR_opt_SSH}" = "xTRUE" ] && [ "${RR_MODE}" == "1" ] && [ "${UserName}" != "admin" ] ; then
		RR_remote_uri="${UserName}"@[${Remote_IP}]:"/share/${RR_remote_path}"
	else
		RR_remote_uri="${UserName}"@[${Remote_IP}]::"${RR_remote_path}"
		if [ "x${RR_opt_SSH}" = "xTRUE" ] && [ "${RR_MODE}" == "1" ] && [ "${UserName}" == "admin" ] ; then
			message=$(${RSYNC_CMD} --sever-mode=${RR_MODE} --dry-run \
				-e "${RR_com_ssh} -o LogLevel=ERROR" --timeout=${xsTimeout} --enc_password="${EnPasswd}" \
				--rsync-path="${RSYNC_CMD}" \
				"${RR_remote_uri}" 2>&1)
			# If it fails, switch to ES-NAS mode
			# QTSHBS00-6406 Some NAS rsync paths are not located at /mnt/ext/opt/HybridBackup/bin/rsync.
			if [ "x$?" != "x0" ] && [ "1" != $( Is_Contain_Substring "${message}" "No such file or directory" ) ]; then
				Debug_Info "${message}"
				Debug_Info "dry run failed, CMD: ${RSYNC_CMD} --list-only -e "${RR_com_ssh} -o LogLevel=ERROR" --timeout=${xsTimeout} --enc_password="${EnPasswd}" "${RR_remote_uri}" "
				RR_opt_server_mode=""
				RR_remote_uri="${UserName}"@[${Remote_IP}]:"${RR_remote_path}"
			fi
		fi
	fi
	;;

  2) # spec 12735 TS to ES
	RR_opt_server_mode=""
	# bug 115036 directory name has space or special char
	Escape_RR_remote_path=$(printf %q "${RR_remote_path}")
	RR_remote_uri="${UserName}"@[${Remote_IP}]:"${Escape_RR_remote_path}"
	# avoid "Host key verification failed"
	auto_add_known_hosts
	;;

  3) # NAS-base
	RR_opt_server_mode="--sever-mode=0"
	RR_remote_uri="${UserName}"@[${Remote_IP}]::"${RR_remote_path}"
	;;

  4) # Rsync-compatible
	RR_opt_server_mode="--sever-mode=0"
	if [ "x${RR_opt_SSH}" = "xTRUE" ]; then
		if [ ${RR_remote_path:0:1} != "/" ]; then
			RR_remote_path_with_slash="/${RR_remote_path}"
		fi
		RR_remote_uri="${UserName}"@[${Remote_IP}]:"${RR_remote_path_with_slash}"

		message=$(${RSYNC_CMD} --list-only \
				-e "${RR_com_ssh} -o LogLevel=ERROR" --timeout=${xsTimeout} --enc_password="${EnPasswd}" \
				"${RR_remote_uri}/" 2>&1)
		# If it fails, switch to NAS-based mode
		if [ "x$?" != "x0" ] ; then
			Debug_Info "${message}"
			Debug_Info "dry run failed, CMD: ${RSYNC_CMD} --list-only -e "${RR_com_ssh} -o LogLevel=ERROR" --timeout=${xsTimeout} --enc_password="${EnPasswd}" "${RR_remote_uri}/" "
			RR_remote_uri="${UserName}"@[${Remote_IP}]::"${RR_remote_path}"
		fi
	else
		RR_remote_uri="${UserName}"@[${Remote_IP}]::"${RR_remote_path}"
	fi
	;;

  *)
	exit 1
	;;
esac

Debug_Info "Direction: ${RR_DIRECTION}, RURI: \"${RR_remote_uri}\""
Debug_Info "Options: ${RR_ARCHIVE} ${RR_EXTAAR} ${RR_options}"

if [ "$DEBUG" = "yes" ]; then
	RR_opt_debug_progress="--debug-progress=1 " #--debug=all4
	RR_opt_verbose="-vvvv"
else
	RR_opt_debug_progress=""
	if is_remote2local; then
		RR_opt_verbose="-vv"
	else
		RR_opt_verbose=""
	fi
fi

while [ $iRetry -le "$xnRetry" ]
do
	Remote_IP=`/sbin/getcfg "${RR_sche}" "Remote IP" -f ${RR_CONF} -d "0.0.0.0"`
	Remote_IP=`/sbin/get_domain_ip ${Remote_IP}`

	if [ "$bAbort" -eq "0" ]; then
		bTake_Snapshot_Fail="FALSE"
		Job_Take_Snapshot "Replication-${LOGFN_PREFIX}";
		if [ "$?" != "0" ]; then
			bTake_Snapshot_Fail="TRUE"
			NCLOG_MSGID_VARS[0]="%V002%"
		fi
		#Tom_change, for bug#62847, 2015/06/18
		szStartTime=`/bin/date +%s`
		/sbin/setcfg "$RR_sche" "Start Time" "$szStartTime" -f "${RR_CONF}"		
		#/sbin/setcfg "$RR_sche" "Finished Time" "$szStartTime" -f "${RR_CONF}"
		#--/

		# double check source path
		if [ ! -d "${RR_local_path_real}" ]; then
			echo "ERROR: ${ERROR_DESC[49]}" > "$SZF_ERROR"
			echo "changdir \"${RR_local_path_real}\" failed: No such file or directory" >> "$SZF_ERROR"
			Update_Status $RR_sche $szUser $szAddr $szHost
			/sbin/setcfg "$RR_sche" "ScriptPID" "0" -f "${RR_CONF}"
			exit
		fi

		# QTSHBS00-1099
		if is_remote2local; then
			RR_src_uri="${RR_remote_uri}/"
			RR_dest_uri="${RR_local_path_real}/"
		else
			RR_src_uri="${RR_local_path_real}/"
			RR_dest_uri="${RR_remote_uri}/"
		fi

		RR_process_pid=0
		if [ "x${RR_opt_SSH}" = "xTRUE" ]; then
			LC_ALL=en_US.UTF-8 ${RSYNC_CMD} -H ${RR_ARCHIVE} ${RR_EXTAAR} \
				${RR_opt_server_mode} \
				${RR_opt_debug_progress} \
				-e "${RR_com_ssh}" \
				${RR_options} \
				--log-file="${RSYNC_LOGFILE}" \
				--exclude='.streams/' \
				--exclude=':2eDS_Store'\
				--exclude='.AppleDB/' \
				--exclude='.AppleDesktop/' \
				--exclude='.AppleDouble/' \
				--exclude='.digest/' \
				--exclude='.@__thumb/' \
				--exclude='@Recycle/' \
				--exclude='@Recently-Snapshot/' \
				--exclude='.@__qini/' \
				--exclude='.@upload_cache/' \
				--schedule="$RR_sche" ${RR_opt_hbsv3} \
				--enc_password="${EnPasswd}" \
				--timeout=${xsTimeout} \
				"${RR_src_uri}" "${RR_dest_uri}" \
				${RR_opt_verbose} 1>${SZF_OUTPUT} 2>"$SZF_ERROR_TMP" &
		else
			[ "x${RR_opt_WanAccel}" = "xTRUE" ] && \
				RR_options=${RR_options}"--sockopts=TCP_CONGESTION=bbr " && \
				Debug_Info "Enable WanAccel"
			[ "x${RR_opt_QWanTag}" = "xTRUE" ] && \
				RR_options=${RR_options}"--sockopts=IP_TOS=4 " && \
				Debug_Info "Enable QWanTag"

			LC_ALL=en_US.UTF-8 ${RSYNC_CMD} -H ${RR_ARCHIVE} ${RR_EXTAAR} \
				${RR_opt_server_mode} \
				${RR_opt_debug_progress} \
				${RR_options} \
				--log-file="${RSYNC_LOGFILE}" \
				--exclude='.streams/' \
				--exclude=':2eDS_Store'\
				--exclude='.AppleDB/' \
				--exclude='.AppleDesktop/' \
				--exclude='.AppleDouble/' \
				--exclude='.digest/' \
				--exclude='.@__thumb/' \
				--exclude='@Recycle/' \
				--exclude='@Recently-Snapshot/' \
				--exclude='.@__qini/' \
				--exclude='.@upload_cache/' \
				--schedule="$RR_sche" ${RR_opt_hbsv3} \
				--enc_password="${EnPasswd}" \
				--timeout=${xsTimeout} \
				--port=${RR_PORT} \
				"${RR_src_uri}" "${RR_dest_uri}" \
				${RR_opt_verbose} 1>${SZF_OUTPUT} 2>"$SZF_ERROR_TMP" &
		fi

		RR_process_pid=$!
		if [ -n "${RR_process_pid}" ]; then
			Job_Set_Rsync_Pid;
			
			# Check if the parent PID is still alive, otherwise stop waiting.
			if [ "1" != "$PPID" ] && [ -d "/proc/$PPID" ]; then
				Debug_Info "Parent process with PID $PPID is alive."
			else
				Debug_Info "Parent process with PID $PPID is not alive."
				iProcPid=`/sbin/getcfg "${RR_sche}" "PID" -f "${RR_CONF}" -d "0"`
				Debug_Info "try to kill rsync pid [$iProcPid]"
				/bin/kill -TERM "$iProcPid"
			fi
			
			Debug_Info "script[${iPid}] wait rsync[${RR_process_pid}] ..."
			wait ${RR_process_pid}  # wait rsync process done
			RR_result=$?
			Debug_Info "pid [${RR_process_pid}] done, RET[${RR_result}] SS_ID[${RR_snapshot_id}]"
		else
			echo "rsync: program failed to execute" > "$SZF_ERROR"
			Debug_Info "@@ rsync: program failed to execute"
		fi
		/sbin/setcfg "$RR_sche" "PID" "0" -f "${RR_CONF}"
		# report errors to log
		Filter_Log_Msg "$SZF_ERROR_TMP" "$SZF_ERROR"
	else
		RR_result="143"
	fi

	if [ "1" == "$bAbortBySys" ] && [ "143" != "$RR_result" ]; then
		echo "rsync: cancelled by system" > "$SZF_ERROR"
	elif [ "143" == "$RR_result" ]; then 	# when rsyncRR.sh catch signal by kill -15 (SIGTERM)
		echo "rsync: cancelled by user" > "$SZF_ERROR"
	elif [ "140" == "$RR_result" ] || [ "137" == "$RR_result" ] || [ "130" == "$RR_result" ] ; then
		# rsyncRR.sh got rsync kill by -9 (SIGKILL) or -2 (SIGINT)
		echo "rsync: cancelled by system" > "$SZF_ERROR"
	fi

	# report_error: set the $iError, send log
	if [ "$iRetry" -ge "$xnRetry" ]; then
		Update_Status "$RR_sche" "$szUser" "$szAddr" "$szHost" "LASTRETRY"
	else
		Update_Status "$RR_sche" "$szUser" "$szAddr" "$szHost"
	fi

	# 20171024 keep last rsync job stderr log
	Echo_Time "RR_result [${RR_result}] iRetry[$((iRetry))] iError[${iError}]" >> "$SZF_ERROR_TMP"
	[ -f "${LOG_FILE_NAME}" -a "$DEBUG" = "no" ] && /bin/rm "${LOG_FILE_NAME}"

	# Don't retry on the following cases.
	if is_not_retry_errno ${iError}; then
		break
	fi

	# Increment $iRetry, Sleep a while before the next try.
	iRetry=$(($iRetry+1))
	/sbin/setcfg "$RR_sche" "Retry" "$iRetry" -f "${RR_CONF}"
	Debug_Info "Beging retry[$iRetry] maxTimes[$xnRetry] interval[$xsInterval]"
	if [ $iRetry -le "$xnRetry" ] && [ "$xsInterval" -gt 0 ]; then
		iInterval=$xsInterval
		while [ $iInterval -gt 0 ]
		do
			if [ "$bAbort" -ne "0" ]; then
				break
			fi
			/sbin/setcfg "$RR_sche" "RemainTime" "$iInterval" -f "${RR_CONF}"
			
			if [ $iInterval -gt 4 ]; then
				iInterval=$(($iInterval-5))
				/bin/sleep 5
			else
				iInterval=$(($iInterval-1))
				/bin/sleep 1
			fi
		done
		/sbin/setcfg "$RR_sche" "RemainTime" "$iInterval" -f "${RR_CONF}"
	fi
done

[ "x${RR_opt_stopnet}" = "xTRUE" ] && start_network

# Clear PID and ScriptPID before exit.
/sbin/setcfg "$RR_sche" "PID" "0" -f "${RR_CONF}"
/sbin/setcfg "$RR_sche" "ScriptPID" "0" -f "${RR_CONF}"

Debug_Info "### End"