#!/bin/sh
#
# $Id$
#
#                       RESTRICTED RIGHTS LEGEND
# Use, duplication, or disclosure by the U.S. Government is subject to
# restrictions as set forth in sub-paragraph (c)(1)(ii) of the Rights in
# Technical Data and Computer Software clause in DFARS 252.227-7013.
#
# RSF-1 (c) Copyright High-Availability.Com Ltd, 1995-2014. All Rights Reserved.
#
#                       High-Availability.com Ltd
#                        Suite B, Pentland House
#                         Village Way, Wilmslow
#                           Cheshire  SK9 2GH
#                            United  Kingdom
#
#                   http://www.high-availability.com
#
#                   For UK support call 0844 736 1974
#                   Outside UK call +44 800 756 1198
#                (or email support@high-availability.com)
#
# Script:       S20zfs
#
# Description:  ZFS file system import/export support. Known environment
#               options are:
#                           zfs_export_fail_reboot=y default is no
#
# Platform:     Unix
#
# Author:       High-Availability.Com Ltd / Paul.Griffiths-Todd
#
. /opt/HAC/bin/rsf.sh

#
# Script name used in dated_echo.
script="`basename $0`"

#
# Environment setup. The service name should mirror that of the ZFS pool
# so it can be picked up here and used in the import/export commands.
# The service name can be hard coded (changing the "service_name" parameter
# below), however, for portability reasons this should be avoided.
#
service=${RSF_SERVICE:-"service_name"}

#
# To support Alternate root import mount point set this variable to ="true"
# Support also needs to be enabled in rsf_python3/rsf_service/Add.py - search
# for NEX-18569 and uncomment the Alt root part.
SUPPORT_ZPOOL_ALTROOT="false"
if [ "${PROP_TRUENAS}" = "${BOOL_STR_TRUE}" ]; then
    dated_echo "TN system. Enabling alternative root mount point"
    SUPPORT_ZPOOL_ALTROOT="true"
fi

#
# Number of threads to use with stmfha restore
STMFHA_THREAD_COUNT=8
PROP_STMFHA_THREADS_STRIPPED=`echo "${PROP_STMFHA_THREADS}" | sed -e 's/ //g' -e 's/[a-z]//g' -e 's/[A-Z]//g'`
if match "x${PROP_STMFHA_THREADS_STRIPPED}" "x" ; then
    dated_echo "Invalid setting (empty value) in PROP_STMFHA_THREADS - using default value of ${STMFHA_THREAD_COUNT}"
elif ! match "${PROP_STMFHA_THREADS_STRIPPED}" '[0-9]*' ; then
    dated_echo "Invalid setting (non-numeric) in PROP_STMFHA_THREADS: ${PROP_STMFHA_THREADS_STRIPPED} - using default value of ${STMFHA_THREAD_COUNT}"
else
    STMFHA_THREAD_COUNT=${PROP_STMFHA_THREADS_STRIPPED}
fi
dated_echo "Thread count for STMFHA COMSTAR LUN import: ${STMFHA_THREAD_COUNT}"

#
# Number of backups to keep.
STMFHA_BACKUP_COUNT=20
PROP_COMSTAR_BACKUP_COUNT_STRIPPED=`echo ${PROP_COMSTAR_BACKUP_COUNT} | sed -e 's/ //g' -e 's/[a-z]//g' -e 's/[A-Z]//g'`
if match "x${PROP_COMSTAR_BACKUP_COUNT_STRIPPED}" "x" ; then
    dated_echo "Invalid setting (empty value) in PROP_COMSTAR_BACKUP_COUNT - using default value of ${STMFHA_BACKUP_COUNT}"
elif ! match "${PROP_COMSTAR_BACKUP_COUNT_STRIPPED}" '[^0-9]*' ; then
    dated_echo "Invalid setting (non-numeric) in PROP_COMSTAR_BACKUP_COUNT: ${PROP_COMSTAR_BACKUP_COUNT} - using default value of ${STMFHA_BACKUP_COUNT}"
else
    STMFHA_BACKUP_COUNT=${PROP_COMSTAR_BACKUP_COUNT_STRIPPED}
fi
dated_echo "Backup count for STMFHA: ${STMFHA_BACKUP_COUNT}"

#
# Various files/flags
VAR_ZPOOL_IMPORT_POINT="/opt/HAC/RSF-1/etc/.zpool_import_point"
export VAR_ZPOOL_IMPORT_POINT

if [ -f /opt/HAC/RSF-1/etc/.no_zfs_cache_save -o "${PROP_USE_ZFS_CACHE}" = "${BOOL_STR_FALSE}" ] ; then
    NOZFSCACHEFILE="${BOOL_STR_TRUE}"
    rm -rf ${CACHEDIR} >/dev/null 2>&1
    if [ "${PROP_USE_ZFS_CACHE}" = "${BOOL_STR_FALSE}" ] ; then
        dated_echo "Note! ZFS cache file option is turned off because of ${PROP_USE_ZFS_CACHE} in rsf configuration database" WARN
    else
        dated_echo "Note! ZFS cache file option is turned off because of /opt/HAC/RSF-1/etc/.no_zfs_cache_save" WARN
    fi
else
    NOZFSCACHEFILE="${BOOL_STR_FALSE}"
    #
    # Check for cache file directory and create if necessary.
    #
    if [ ! -d "${CACHEDIR}" ] ; then
        dated_echo "creating cache file directory ${CACHEDIR}"
        rm -rf ${CACHEDIR} >/dev/null 2>&1
        mkdir ${CACHEDIR}
    fi
fi

if [ "${PROP_DTRACE_IS_BROKEN}" = "${BOOL_STR_TRUE}" ] ; then
    dated_echo "Note! prop_dtrace_is_broken is True - not using dtrace wrapper around import/export"
fi

#
#
STMF_PURGE="/opt/HAC/RSF-1/bin/purge-stmf.sh"
ZPOOL_LOG="/tmp/zpool-cmd.$$"
rm -f ${ZPOOL_LOG}

#
#
ZPOOL_IMPORT_POINT="/volumes"
export ZPOOL_IMPORT_POINT

if [ -f ${VAR_ZPOOL_IMPORT_POINT} ] ; then
    ZPOOL_IMPORT_POINT=`cat ${VAR_ZPOOL_IMPORT_POINT}`
fi

#
# Only implement alternate root if configured so.
if [ "${SUPPORT_ZPOOL_ALTROOT}" = "true" ] ; then
    # NEX-18569: grab altroot from DB.
    dated_echo "New altroot detection mode. Reading mount point from database."
    DB_IMPORT_POINT=`/opt/HAC/RSF-1/bin/rsfcdb sg_mp ${service}`
    ZPOOL_IMPORT_POINT=${DB_IMPORT_POINT%${service}}
    case ${ZPOOL_IMPORT_POINT} in
        ?*/) ZPOOL_IMPORT_POINT=${ZPOOL_IMPORT_POINT%/}
    esac
    dated_echo "Detected mount point: ${ZPOOL_IMPORT_POINT}"
    export ZPOOL_IMPORT_POINT
    # NEX-18569 end
fi

#
# Standard script arguments:
#
#  $1 - State which should either be start or stop
#  $2 - Optional attempt number, default 1, or that passed in (RSF-1 increments this value when attempting multiple times)
#
State=$1
attempt=${2:-'3'}

if [ "${PROP_COMSTAR_SUPPORT}" = "${BOOL_STR_TRUE}" ]; then
    #
    # Permissions check on STMFHA...
    dated_echo "Using STMFHA binary: ${PROG_STMFHA}"
    if [ -f "${PROG_STMFHA}" -a ! -x "${PROG_STMFHA}" ] ; then
        chmod +x "${PROG_STMFHA}"
    fi
fi

if [ "${PROP_ZPOOL_THREADS_OPTION_T}" = "${BOOL_STR_TRUE}" ] && [ ${PROP_ZPOOL_THREADS_NUMBER} -gt 1 ]; then
    THREAD_OPT="-t ${PROP_ZPOOL_THREADS_NUMBER}"
    if [ ${State} = "start" ]; then
        import_export="import"
    else
        import_export="export"
    fi
    dated_echo "Using zpool ${import_export} parallel mount option."
    dated_echo "Max threads set to ${PROP_ZPOOL_THREADS_NUMBER}"
else
    THREAD_OPT=
fi

#
# Check if we have option C enabled, and if so, check we are Nexenta V4
if [ "${PROP_ZPOOL_EXPORT_OPTION_C}" = "${BOOL_STR_TRUE}" ] ; then
    if ! is_nexenta_4 && ! is_nef ; then
        dated_echo "Warning: PROP_ZPOOL_EXPORT_OPTION_C set to TRUE, but Nexenta 4 not detected; disabling setting."
        PROP_ZPOOL_EXPORT_OPTION_C="${BOOL_STR_FALSE}"

        ${CDB_EXEC} ${CDB_FLAGS} -r 'S20zfs->option_c_check' update prop_zpool_export_option_c ${BOOL_STR_FALSE}
    fi
fi

#
# Usage: scan_mounts <mount table filename> <mount point to find>
scan_mounts()
{
    scan_mounts_result=0
    while read scan_mounts_arg_1 scan_mounts_arg_2 scan_mounts_arg_345 ; do
        case "$scan_mounts_arg_2" in
            $2) echo "$scan_mounts_arg_1 $scan_mounts_arg_2 $scan_mounts_arg_345"
                scan_mounts_result=1
                ;;
        esac
    done <"$1"
    return $scan_mounts_result
}

###################################################################################
#
# Function:    sync_zfs_cache()
#
# Description:
#              update archive cache file from live one
#
# Args:
#              from cache file
#              to cache file
#
sync_zfs_cache()
{
    FROM=$1
    TO=$2

    #
    # copy the cache file.
    if [ -f ${FROM} ] ; then
        dated_echo "Updating saved cache file: ${FROM} ==> ${TO}"
        rm -f ${TO}.old ${TO}.new
        cp ${FROM} ${TO}.new
        if [ -f ${TO} ] ; then
            mv ${TO} ${TO}.old
        fi
        mv ${TO}.new ${TO}
    fi
}

###################################################################################
#
# Function:    run_pool_hook()
#
# Description:
#              Cluster hook is called in several key places and provides an extension
#              interface to allow customised external commands to be run at certain
#              stages during service start/stop.
#
# Args:
#              phase   - string defining either starting or stopping: import/export
#              stage   - string defining stage of precess: before/after/failure.
#              service - the service currently being operated on.
#
# Return:
#              exit code from the hook.
#
run_pool_hook()
{
    #
    # Check it's not the default value of NULL
    if [ "${PROP_POOL_HOOK}" = "${BOOL_STR_NULL}" ] ; then
        return
    fi

    #
    # Check for correct number of arguments.
    if [ $# -ne 3 ] ; then
        dated_echo "Assertion failed - run_pool_hook called with incorrect number of args: $*" ERROR
        return
    fi

    PHASE=$1
    STAGE=$2
    SERVICE=$3

    if [ -f "${PROP_POOL_HOOK}" ] ; then
        if [ ! -x "${PROP_POOL_HOOK}" ] ; then
            chmod +x "${PROP_POOL_HOOK}"
        fi

        dated_echo "Running pool hook: ${PROP_POOL_HOOK} ${PHASE} ${STAGE} ${SERVICE}"
        pfexec ${PROP_POOL_HOOK} "${PHASE}" "${STAGE}" "${SERVICE}"
    fi

    return $?
}

###################################################################################
#
# Function:    write_broken_log()
#
# Description:
#              Update the log message file when a service goes broken.
#              ZPOOL_LOG is the output from zpool import/export.
#              NEX-14392 9th Aug 2022: removed erroneous output from dtrace.
#
# Args:
#              Log message
#
# Return:
#              Nothing
#
write_broken_log()
{
    if [ -f "${ZPOOL_LOG}" ] ; then
        #
        # Output details to the log file.
        sed '/^dtrace:/d' "${ZPOOL_LOG}"
        if [ -d "${BROKEN_LOG}" ] ; then
            dated_echo "Saving zpool error to ${BROKEN_LOG}/${service}.log" ERROR
            sed  '/^dtrace:/d' ${ZPOOL_LOG} >> ${BROKEN_LOG}/${service}.log
        fi
        rm -f ${ZPOOL_LOG}
    fi
}

###################################################################################
#
# Function:    nef_write_broken_log()
#
# Description:
#              Same function as above to preserve the original for NEF
#              Update the log message file when a service goes broken.
#              ZPOOL_LOG is the output from zpool import/export.
#              NEX-14392 9th Aug 2022: removed erroneous output from dtrace.
#
# Args:
#              Log message
#
# Return:
#              Nothing
#
nef_write_broken_log()
{
    if [ -f "${ZPOOL_LOG}" ] ; then
        #
        # Output details to the log file.
        sed '/^dtrace:/d' "${ZPOOL_LOG}"
        if [ -d "${BROKEN_LOG}" ] ; then
            dated_echo "Saving zpool error to ${BROKEN_LOG}/${service}.log" ERROR
            echo "<b>`date`</b><br />" > ${BROKEN_LOG}/${service}.log
            echo "${1} - saved zpool command output is:" >> ${BROKEN_LOG}/${service}.log
            echo "" >> ${BROKEN_LOG}/${service}.log
            echo "==============================================================" >> ${BROKEN_LOG}/${service}.log
            sed  '/^dtrace:/d' ${ZPOOL_LOG} >> ${BROKEN_LOG}/${service}.log
            echo "==============================================================" >> ${BROKEN_LOG}/${service}.log
        fi
        rm -f ${ZPOOL_LOG}
    fi
}

###################################################################################
#
# Function:    check_comstar_sync()
#
# Description:
#              Wait for COMSTAR to sync iSCSI when out of sync
#              Times out if sync does not complete in 15 seconds
#
# Args:
#              Service to check
#
# Return:
#              Nothing
#
check_comstar_sync()
{
    CNT=0
    TIMEOUT=30
    SYNC_FILE="${ZPOOL_IMPORT_POINT}/${1}/.comstar/config-out-of-sync"
    SYNC_DONE_FILE="${ZPOOL_IMPORT_POINT}/${1}/.comstar/config-sync-done"

    dated_echo "COMSTAR: starting sync check."

    while [ -f "${SYNC_FILE}" -a ! -f "${SYNC_DONE_FILE}" ] ; do
        if [ ${CNT} -eq 0 -o `expr ${CNT} % 5` -eq 0 ] ; then
            dated_echo "COMSTAR: sync check, timeout in `expr ${TIMEOUT} - ${CNT}` seconds."
        fi

        CNT=`expr ${CNT} + 1`
        if [ ${CNT} -gt ${TIMEOUT} ] ; then
            dated_echo "COMSTAR: sync timeout (${TIMEOUT} reached), continuing."
            break
        fi
        sleep 1
    done

    if [ ${CNT} -le ${TIMEOUT} ] ; then
        dated_echo "COMSTAR: sync complete, continuing." SUCCESS
    fi
}

###################################################################################
#
# Function:    do_init()
#
# Description:
#              Call init if zfs_export_fail_reboot is set to Y or y
#
# Args:
#              none
#
# Return:
#              0 if init called
#              1 if no option specified
#
do_init()
{
    OPTION=${opt_zfs_export_fail_reboot:-"n"}

    case "${OPTION}" in

        [Yy]*)
            dated_echo "${service} zfs_export_fail_reboot in effect - rebooting appliance" ERROR
            notify_watchers "${LOG_CRIT}" "RSF_SERVICE_SHUTDOWN" "service=${service} resource=ZPOOL zfs_export_fail_reboot=${opt_zfs_export_fail_reboot}"
            sleep 4
            uadmin 2 1
            return 0
            ;;

        *)
            return 1
            ;;

    esac
}

###################################################################################
#
# Function:    pool_is_imported()
#
# Description:
#              Test if the given pool is mounted locally
#
# Args:
#              zpool to test
#              zpool GUID (optional)
#
# Return:
#              0 - pool is imported
#              1 - another pool is imported with the same name
#              2 - pool is imported with another name
#              3 - pool is not imported
#
pool_is_imported()
{
    pool_name=$1
    pool_guid=$2

    if [ "${pool_guid}" = "GUIDREPLACEME" ] ; then
        HOSTNAME=`hostname`
        pool_guid=`/opt/HAC/RSF-1/bin/rsfcdb read ${pool_name}_${HOSTNAME}_guid`
    fi

    #
    # First read the GUID from the imported pool.
    # If this matches then the pool is imported OK
    #
    guid_from_pool=`zpool get -H -o value guid ${pool_name} 2>/dev/null`
    pool_name_imported=$?

    #
    # Special case first for when we don't have a supplied GUID
    #
    if [ -z "${pool_guid}" ]; then
        #
        # We can only give a yes or no answer in this case
        #
        if [ ${pool_name_imported} -eq 0 ]; then
            return 0
        else
            return 3
        fi
    fi

    if [ "${guid_from_pool}" = "${pool_guid}" ]; then
        #
        # Pool is imported - name and GUID are OK
        #
        return 0
    fi

    #
    # There are now 3 options:
    #   a) Another pool is imported with the same name
    #   b) Pool is imported with another name
    #   c) Pool is not imported (by name or GUID)
    #

    if [ ${pool_name_imported} -eq 0 ]; then
       #
       # Found pool name with wrong GUID (option a)
       #
       dated_echo "Another pool is imported using the same name - ${pool_name} (${guid_from_pool})" ERROR
       return 1
    fi

    #
    # Try to find imported pool name using the given GUID
    # SG-1321:
    # zpool get requires a pool name in Solaris.
    for zpool_list_name in `zpool list -Ho name`
    do
        name_from_guid=$(zpool get -H -o value,name guid ${zpool_list_name} | sed -n "$(printf 's/^%s[\t ]\{1,\}//p' "${pool_guid}")")
        if [ -n "${name_from_guid}" ]; then
            #
            # Pool imported with wrong name (option b)
            #
            dated_echo "Pool ${pool_name} (${pool_guid}) is imported with a different name (${name_from_guid})" ERROR
            return 2
        fi
    done
    #
    # SG-1321 replaces this:
    #name_from_guid=$(zpool get -H -o value,name guid | sed -n "$(printf 's/^%s[\t ]\{1,\}//p' "${pool_guid}")")
    #if [ -n "${name_from_guid}" ]; then
    #    dated_echo "Pool ${pool_name} (${pool_guid}) is imported with a different name (${name_from_guid})" ERROR
    #    return 2
    #fi

    #
    # We now know there is no pool currently imported with the given name or GUID
    # Pool must be exported (option c)
    #
    return 3
}

###################################################################################
#
# Function:    set_paths()
#
# Description:
#              set up a number of paths dependent on pool name
#
# Args:
#              zpool name
#
# Return:
#              nothing
set_paths()
{
    CURRENT_ZPOOL=$1

    if [ -z "${CURRENT_ZPOOL}" ] ; then
        CACHEFILE="none"
        CACHELIVEFILE="none"
        ZPOOLCACHEVERSION="none"
    else
        CACHEFILE="${CACHEDIR}/${CURRENT_ZPOOL}.${CACHE_SUFFIX}"
        CACHELIVEFILE="${CACHEDIR}/${CURRENT_ZPOOL}.${CACHE_LIVE_SUFFIX}"
        ZPOOLCACHEVERSION="${CACHEDIR}/${CURRENT_ZPOOL}.zpool_version"
    fi
    export CACHEFILE CACHELIVEFILE ZPOOLCACHEVERSION

    dated_echo "CACHEFILE:         ${CACHEFILE}"
    dated_echo "CACHELIVEFILE:     ${CACHELIVEFILE}"
    dated_echo "ZPOOLCACHEVERSION: ${ZPOOLCACHEVERSION}"
}

###################################################################################
#
# Function:    save_zpool_version()
#
# Description:
#              save the zpool version
#
# Args:
#              zpool name
#
# Return:
#              nothing
save_zpool_version()
{
    return

    CURRENT_ZPOOL=$1

    if [ ! -z "${CURRENT_ZPOOL}" -a "${NOZFSCACHEFILE}" != "${BOOL_STR_TRUE}" -a -d "${CACHEDIR}" ] ; then
        #
        # Specify the e option here to signify that we are not using standard cache file.
        zdb -ue "${CURRENT_ZPOOL}" | grep -i version | sed 's/[^0-9]*//g' > "${ZPOOLCACHEVERSION}" &
        dated_echo "Saving zpool version to ${ZPOOLCACHEVERSION}"
    else
        dated_echo "Not saving zpool version - cache file option turned off."
    fi
}

###################################################################################
#
# Function:    check_zpool_version()
#
# Description:
#              check the zpool version is the same as we have previously saved
#
# Args:
#              zpool name
#
# Return:
#              nothing
check_zpool_version()
{
    return

    CURRENT_ZPOOL=$1

    #
    # Grab zpool version.
    ZPOOL_VERSION=`zdb -ue ${CURRENT_ZPOOL} | grep -i version | sed 's/[^0-9]*//g'`
    dated_echo "Pool ${CURRENT_ZPOOL} is zpool version ${ZPOOL_VERSION}"

    if [ -f "${ZPOOLCACHEVERSION}" ] ; then
        SAVED_ZPOOL_VERSION=`cat ${ZPOOLCACHEVERSION}`
        if [ ! -z "${SAVED_ZPOOL_VERSION}" -a ! -z "${ZPOOL_VERSION}" ] ; then
            if [ "${SAVED_ZPOOL_VERSION}" -ne "${ZPOOL_VERSION}" ] ; then
                dated_echo "Saved zpool version, ${SAVED_ZPOOL_VERSION}, and that reported by zpool for pool ${CURRENT_ZPOOL}, ${ZPOOL_VERSION}, differ - clearing cache" WARN
                rm -f "${CACHEDIR}/${CURRENT_ZPOOL}.${CACHE_SUFFIX}" "${CACHEDIR}/${CURRENT_ZPOOL}.${CACHE_LIVE_SUFFIX}" "${SAVED_ZPOOL_VERSION}"
            else
                dated_echo "Saved zpool version, ${SAVED_ZPOOL_VERSION}, same as version reported by zpool, ${ZPOOL_VERSION}, cache file considered valid"
            fi
        fi
    else
        dated_echo "No saved zpool version; clearing cache"
        rm -f "${CACHEDIR}/${CURRENT_ZPOOL}.${CACHE_SUFFIX}" "${CACHEDIR}/${CURRENT_ZPOOL}.${CACHE_LIVE_SUFFIX}" "${SAVED_ZPOOL_VERSION}"
    fi

}

###################################################################################
#
# Function:    clear_zpool_cache()
#
# Description:
#              remove the cached zpool files
#
# Args:
#              zpool name
#
# Return:
#              nothing
clear_zpool_cache()
{
    CURRENT_ZPOOL=$1

    if [ "${NOZFSCACHEFILE}" = "${BOOL_STR_TRUE}" ] ; then
        dated_echo "Not using zpool cache files: NOZFSCACHEFILE is ${NOZFSCACHEFILE}"

        if [ ! -z "${CURRENT_ZPOOL}" ] ; then
            dated_echo "Clearing zpool cache files for pool: ${CURRENT_ZPOOL}"
            rm -rf "${CACHEFILE}" "${CACHELIVEFILE}" "${ZPOOLCACHEVERSION}"
        fi
        set_paths
    fi
}

###################################################################################
#
# Function:    clean_mountpoint()
#
# Description:
#              remove meta data files from mountpoint so import does not fail with
#              metadata.
#
# Args:
#              zpool to test
#
# Return:      0 - all ok
#              1 - non metadata files exist; cannot remove (could be customer files)
#
clean_mountpoint()
{
    VOLDIR="${ZPOOL_IMPORT_POINT}/${1}"
    MAPDIR="${VOLDIR}/.mapping/"
    RSFDIR="${VOLDIR}/.rsf-cluster/"
    CNFDIR="${VOLDIR}/.config"
    ABORT=1
    OK=0

    #
    # It's mounted so do not attempt to tidy metadata.
    if pool_is_imported ${1}; then
        return ${ABORT}
    fi

    if [ -f "${VOLDIR}" ] ; then
        MVDIR=${VOLDIR}_`date +%Y.%m.%d-%H:%M:%S`
        dated_echo "### File as mount point detected, moving out of the way, as ${VOLDIR} to ${MVDIR}" WARN
        mv ${VOLDIR} ${MVDIR}
    fi

    #
    # Test volume import directory is actually a directory.
    # If it does not exist then ignore
    if [ ! -d "${VOLDIR}" ] ; then
        return ${OK}
    fi

    #
    # Check and remove metadata directory for map manager.
    if [ -d "${MAPDIR}" ] ; then
        dated_echo "### removing unused ${MAPDIR} on volume ${1} import point"
        rm -rf ${MAPDIR}
    fi

    #
    # Check and remove metadata directory for saved config.
    if [ -d "${CNFDIR}" ] ; then
        dated_echo "### removing unused ${CNFDIR} on volume ${1} import point"
        rm -rf ${CNFDIR}
    fi

    #
    # Check and remove metadata directory for cluster info.
    if [ -d "${RSFDIR}" ] ; then
        dated_echo "### removing unused ${RSFDIR} on volume ${1} import point"
        rm -rf ${RSFDIR}
    fi

    #
    # Finally - remove the mount point if required.
    if [ -d "${VOLDIR}" ] ; then
        #
        # Check for any other extra files.
        count=`ls -a ${VOLDIR}/ | wc -l`

        #
        # To be safe
        if [ -z "${count}" ] ; then
            count=2
        fi

        if [ ${count} -gt 2 ]; then
            MVDIR=${VOLDIR}_`date +%Y.%m.%d-%H:%M:%S`
            dated_echo "### Data still in mount point, moving directory ${VOLDIR} to ${MVDIR}" WARN
            if ! mv ${VOLDIR} ${MVDIR}; then
                dated_echo "Failed to move directory!" ERROR
                return ${ABORT}
            fi
        fi
    fi
    return ${OK}
}


###################################################################################
#
# Function:    zpool_decrypt()
#
# Description:
#              Wrapper around the ZFS load keys and ZFS mount
#
# Args:
#              First arg should be the pool name
#
# Return:
#
zpool_decrypt()
{
    OK=0
    ER=1

    if [ $# -ne 1 ] ; then
        dated_echo "Assertion failed - zfs_decrypt called with incorrect number of args: $*" ERROR
        return ${ER}
    else
        EZPOOL=${1}
    fi

    encryption="off"
    for i in $(zfs list -rH ${EZPOOL} | awk '{print $1}')
    do
        cmd="zfs get -H -o value encryption ${i}"
	output=`${cmd}`
	if [ $? -ne 0 ] ; then
	    dated_echo "ZFS get encryption type failed status ${zstat}"
	    dated_echo "Failing encryption cmd: ${cmd}"
	    return ${OK} # no point failing
	fi

	if [ "${output}" != "off" ] ; then
	    encryption="${output}"
	    break
	fi
    done

    if [ "${encryption}" = "off" ] ; then
        dated_echo "ZFS encryption not enabled on pool/datasets: ${EZPOOL}"
    else
        dated_echo "ZFS encryption type is: ${output}"
        dated_echo "Loading ZFS keys from key files"
        zfs load-key -a
        dated_echo "Mounting encrypted ZFS datasets in ${EZPOOL}:"
        for i in $(zfs list -rH ${EZPOOL} | awk '{print $1}')
        do
            dated_echo "  mounting ${i}"
            zfs mount ${i}
        done
    fi

    return ${OK}
}

###################################################################################
#
# Function:    zpool_import()
#
# Description:
#              Wrapper around the zpool import command
#              Gives timing information and prints the command before running it
#
# Args:
#              Any args to be passed to zpool import
#
# Return:      Passes return code from zpool import command
#
zpool_import()
{
    DTRACE="eval"
    using_dtrace="false"
    if [ "${PROP_ZPOOL_DTRACE_IMPORT_EXPORT}" = "${BOOL_STR_TRUE}" ]; then
        mkdir -p ${DTRACE_LOGDIR}
        DTRACE_LOG="${DTRACE_LOGDIR}/zpool-import-trace-${ZPOOL}-`date +%Y-%m-%d-%H-%M`.log"
        rm -f "${DTRACE_LOG}"
        if [ -f ${DTRACE_DIR}/spa_import.d ]; then
            DTRACE="dtrace -s ${DTRACE_DIR}/spa_import.d -o ${DTRACE_LOG} -Z -c"
            using_dtrace="true"
        elif [ -f ${RSF_DTRACE_DIR}/spa_import.d ]; then
            DTRACE="dtrace -s ${RSF_DTRACE_DIR}/spa_import.d -o ${DTRACE_LOG} -Z -c"
            using_dtrace="true"
        fi
    fi

    IMPORT_TIME="unknown"
    if [ "${SUPPORT_ZPOOL_ALTROOT}" = "true" ] ; then
        cmd="zpool import -R ${ZPOOL_IMPORT_POINT} $*"
    else
        cmd="zpool import $*"
    fi
    dated_echo "Zpool import command is: ${cmd}"
    [ -x ${PROG_RSFSYSEVENTD} ] && ${PROG_RSFSYSEVENTD} --request nef_zpool_import --immediate-delivery --batch-mode --zpool "${ZPOOL}" --operation-state "started"
    START_TIME=`${DATE}`
    ${DTRACE} "${cmd}" >${ZPOOL_LOG} 2>&1
    zstat=$?
    if [ "${using_dtrace}" = "true" -a -f ${ZPOOL_LOG} ]; then
        # dtrace does not pass through the zpool import return code, so we have to grab it from the dtrace log
        zstat=`awk "${DTRACE_CODE_EXTRACTOR}" ${DTRACE_LOG}`
    fi
    IMPORT_TIME=`expr \`${DATE}\` - ${START_TIME}`
    [ -x ${PROG_RSFSYSEVENTD} ] && ${PROG_RSFSYSEVENTD} --request nef_zpool_import --immediate-delivery --batch-mode --zpool "${ZPOOL}" --operation-state "complete"
    cat ${ZPOOL_LOG}
    if [ -f "${DTRACE_LOG}" ] ; then
        cat "${DTRACE_LOG}"
    fi
    if [ ${zstat} -ne 0 ] ; then
        dated_echo "Zpool import failed status ${zstat}, in ${IMPORT_TIME} seconds" WARN
    else
        dated_echo "Zpool import completed status ${zstat}, in ${IMPORT_TIME} seconds" SUCCESS
    fi

    if [ "${PROP_ZPOOL_DTRACE_IMPORT_EXPORT}" = "${BOOL_STR_TRUE}" ]; then
        if [ ${zstat} -eq 0 ]; then
            if [ ${IMPORT_TIME} -gt ${PROP_ZPOOL_DTRACE_NEEDED_TIMEOUT_SECONDS} ]; then
                dated_echo "Zpool import took more than ${PROP_ZPOOL_DTRACE_NEEDED_TIMEOUT_SECONDS} seconds."
            else
                rm -f ${DTRACE_LOG}
            fi
        fi
        if [ -f ${DTRACE_LOG} ]; then
            dated_echo "Please see ${DTRACE_LOG} for more information"
            #
            # If the zpool import worked but took too long, we may want to halt now for debugging
            #
            if [ ${zstat} -eq 0 ]; then
                service_halt ${service} "timeout"
            fi
        fi
    fi

    return ${zstat}
}

###################################################################################
#
# Function:    zpool_export()
#
# Description:
#              Wrapper around the zpool export command
#              Gives timing information and prints the command before running it
#
# Args:
#              Any args to be passed to zpool export
#
# Return:      Passes return code from zpool export command
#
zpool_export()
{
    DTRACE="eval"
    using_dtrace="false"
    if [ "${PROP_ZPOOL_DTRACE_IMPORT_EXPORT}" = "${BOOL_STR_TRUE}" ]; then
        mkdir -p ${DTRACE_LOGDIR}
        DTRACE_LOG="${DTRACE_LOGDIR}/zpool-export-trace-${ZPOOL}-`date +%Y-%m-%d-%H-%M`.log"
        rm -f "${DTRACE_LOG}"
        if [ -f ${DTRACE_DIR}/spa_export.d ]; then
            DTRACE="dtrace -s ${DTRACE_DIR}/spa_export.d -o ${DTRACE_LOG} -Z -c"
            using_dtrace="true"
        elif [ -f ${RSF_DTRACE_DIR}/spa_export.d ]; then
            DTRACE="dtrace -s ${RSF_DTRACE_DIR}/spa_export.d -o ${DTRACE_LOG} -Z -c"
            using_dtrace="true"
        fi
    fi

    EXPORT_TIME="unknown"
    cmd="zpool export $*"
    dated_echo "Command is: ${cmd}"
    START_TIME=`${DATE}`
    [ -x ${PROG_RSFSYSEVENTD} ] && ${PROG_RSFSYSEVENTD} --request nef_zpool_export --immediate-delivery --batch-mode --zpool "${ZPOOL}" --operation-state "started"
    ${DTRACE} "${cmd}" >${ZPOOL_LOG} 2>&1
    zstat=$?
    if [ "${using_dtrace}" = "true" -a -f ${ZPOOL_LOG} ]; then
        # dtrace does not pass through the zpool import return code, so we have to grab it from the dtrace log
        zstat=`awk "${DTRACE_CODE_EXTRACTOR}" ${DTRACE_LOG}`
    fi
    EXPORT_TIME=`expr \`${DATE}\` - ${START_TIME} `
    [ -x ${PROG_RSFSYSEVENTD} ] && ${PROG_RSFSYSEVENTD} --request nef_zpool_export --immediate-delivery --batch-mode --zpool "${ZPOOL}" --operation-state "complete"
    cat ${ZPOOL_LOG}
    if [ -f "${DTRACE_LOG}" ] ; then
        cat "${DTRACE_LOG}"
    fi
    if [ ${zstat} -ne 0 ] ; then
        dated_echo "Zpool export failed status ${zstat}, in ${EXPORT_TIME} seconds" WARN
    else
        dated_echo "Zpool export completed status ${zstat}, in ${EXPORT_TIME} seconds" SUCCESS
    fi

    if [ "${PROP_ZPOOL_DTRACE_IMPORT_EXPORT}" = "${BOOL_STR_TRUE}" ]; then
        if [ ${zstat} -eq 0 ]; then
            if [ ${EXPORT_TIME} -gt ${PROP_ZPOOL_DTRACE_NEEDED_TIMEOUT_SECONDS} ]; then
                dated_echo "Zpool export took more than ${PROP_ZPOOL_DTRACE_NEEDED_TIMEOUT_SECONDS} seconds."
            else
                rm -f ${DTRACE_LOG}
            fi
        fi
        if [ -f ${DTRACE_LOG} ]; then
            dated_echo "Please see ${DTRACE_LOG} for more information"
            #
            # If the zpool export worked but took too long, we may want to halt now for debugging
            #
            if [ ${zstat} -eq 0 ]; then
                service_halt ${service} "timeout"
            fi
        fi
    fi

    return ${zstat}
}


###################################################################################
#
# Function:    zfs_import()
#
# Description:
#              method to handle import of ZFS pool
#
# Args:
#              zpool to import
#              force import (string true/false)
#
# Return:      0  - all OK
#              >0 - error occured from zpool import
#
zfs_import()
{
    ZPOOL=""
    FORCE="-f"
    MISSING_DEVICES=""
    OFFLINING=""

    if [ $# -ne 1 ] ; then
        dated_echo "Assertion failed - zfs_import called with incorrect number of args: $*" ERROR
        return 1
    else
        ZPOOL=$1
    fi

    FAILMODE=`${HAC_PYTHON} ${PROG_RSFPYUTILS} --getZpoolFailmode ${ZPOOL} 2>/dev/null`
    if [ -n "${FAILMODE}" ]; then
        FAILMODE="-o failmode=${FAILMODE}"
    fi

    rm -rf /opt/HAC/RSF-1/etc/volume-cache-live/${ZPOOL}.${CACHE_LIVE_SUFFIX} >/dev/null 2>&1

    #
    #
    [ -f /opt/HAC/RSF-1/etc/.import_missing_devices.${ZPOOL} ] && OFFLINING="`cat /opt/HAC/RSF-1/etc/.import_missing_devices.${ZPOOL}`"

    #
    # Check for missing devices -m option.
    if [ -z "${PROP_ZPOOL_IMPORT_OPTION_M}" ] ; then
        dated_echo "### missing property: PROP_ZPOOL_IMPORT_OPTION_M, setting to false"
        PROP_ZPOOL_IMPORT_OPTION_M="${BOOL_STR_FALSE}"
    fi
    dated_echo "zfs_import: ignore missing devices -m: ${PROP_ZPOOL_IMPORT_OPTION_M}"

    if [ "${PROP_ZPOOL_IMPORT_OPTION_M}" = "${BOOL_STR_TRUE}" ] ; then
        MISSING_DEVICES="-m"
    fi

    if [ -n "${POOL_GUID}" ] ; then
        ZPOOL_ID=${POOL_GUID}
        # If ZPOOL_ID is "GUIDREPLACEME" this is a shared nothing cluster
        # and we need to get the GUID from the cluster database
        if [ "${ZPOOL_ID}" = "GUIDREPLACEME" ] ; then
            HOSTNAME=`hostname`
            ZPOOL_ID=`/opt/HAC/RSF-1/bin/rsfcdb read ${ZPOOL}_${HOSTNAME}_guid`
        fi
    else
        ZPOOL_ID=${ZPOOL}
    fi

    if [ "${NOZFSCACHEFILE}" = "${BOOL_STR_TRUE}" ] ; then
        CACHE_LIVE_OPT="-o cachefile=none"
    else
        CACHE_LIVE_OPT="-o cachefile=${CACHELIVEFILE}"
    fi

    S3_ARG=
    if current_pool_is_s3; then
        dated_echo "Detected S3 pool (${POOL_NAME})"
        S3_ARG="-d /var/run/S3/${S3_ACCOUNT}-`hostname`/${S3_BUCKET}"
    fi

    ZPOOL_ARGS="${CACHE_LIVE_OPT} ${FAILMODE} ${FORCE} ${OFFLINING} ${MISSING_DEVICES} ${THREAD_OPT} ${ZPOOL_ID}"

    #
    # SG-1086: if Awanstor import by -d /dev/gpt and by name (not guid, -d does not support it)
    AWANSTOR_ARG=
    if [ "${PROP_AWANSTOR}" = "${BOOL_STR_TRUE}" ]; then
        dated_echo "Awanstor system. Importing pool with -d /dev/gpt"
        AWANSTOR_ARG="-d /dev/gpt"
        ZPOOL_ARGS="${FAILMODE} ${FORCE} ${OFFLINING} ${MISSING_DEVICES} ${THREAD_OPT} ${ZPOOL}"
    fi

    #
    # Handle the case where we have no cache file or it is turned off.
    # In this case we just use the CACHELIVEFILE directory  to create
    # the cache which is then moved over.
    if [ ! -f "${CACHEFILE}" -o "${NOZFSCACHEFILE}" = "${BOOL_STR_TRUE}" -o "${PROP_AWANSTOR}" = "${BOOL_STR_TRUE}" ] ; then
        dated_echo "No saved cachefile found or cachefile use disabled in database"
        dated_echo "Issuing zpool import"
        zpool_import ${AWANSTOR_ARG} ${S3_ARG} ${ZPOOL_ARGS}
        zstat=$?
    else
        if [ "${PROP_BSD_MPATH_ENABLED}" = "${BOOL_STR_TRUE}" ]; then
            dated_echo "Using import path of /dev/multipath for BSD multipathing support"
            ZPOOL_ARGS="-d /dev/multipath ${ZPOOL_ARGS}"
            zpool_import ${ZPOOL_ARGS}
            zstat=$?
        else
            #
            # Cache file detected - import using this cachefile.
            dated_echo "Saved cache file detected as ${CACHEFILE}; issuing zpool import with cachefile option"
            zpool_import -c ${CACHEFILE} ${ZPOOL_ARGS}
            zstat=$?
            #
            # Fail could be because of invalid saved cache file.
            if [ ${zstat} -ne 0 ] ; then
                if pool_is_imported ${ZPOOL}; then
                    dated_echo "Pool ${ZPOOL} imported but zpool import command exited non-zero (${zstat})" ERROR
                else
                    dated_echo "Saved cache file import failed; importing without cachefile" WARN
                    rm -f ${CACHEFILE}
                    zpool_import ${S3_ARG} ${ZPOOL_ARGS}
                    zstat=$?
                fi
            fi
        fi
    fi

    #
    # ZFS encryption
    # First we need to know if encryption is enabled
    zpool_decrypt ${ZPOOL}

    #
    # Check the zpool import did the mount correctly and re-issue if not.
    maxretries=4
    zfs get -H -o value mounted ${ZPOOL} | grep no >/dev/null 2>&1
    if [ $? -eq 0 ] ; then
        dated_echo "Filesystem ${ZPOOL} reported as not mounted by ZFS" ERROR

        #
        # May want to halt at this point for debugging
        # This depends on a DB property and will probably be a no-op
        #
        service_halt ${service}

        dated_echo "Re-issuing mount request (max ${maxretries} times)"

        cnt=1
        while [ ${cnt} -le ${maxretries} ]
        do
            zfs mount -a
            mntcode=$?
            zfs share -a
            shrcode=$?

            #
            # We're going to have to assume here that if the mount worked then the
            # share also did.
            MNTVALUE=`zfs get -H -o value mounted ${ZPOOL}`
            dated_echo "attempt ${cnt}: mounted pool value is '${MNTVALUE}', mountall returned ${mntcode} and shareall returned ${shrcode}"

            if [ "${MNTVALUE}" = "no" ] ; then
                dated_echo "Re-mount attempt ${cnt} failed" WARN
                cnt=`expr ${cnt} + 1`
                sleep 1
            else
                zstat=0
                dated_echo "Re-mount succeeded at attempt ${cnt}"
                break
            fi
        done
    fi

    if [ ${zstat} -eq 0 ]; then
        notify_watchers "${ZFS_INFO}" "RSF_SERVICE_STARTUP" "service=${service} resource=ZPOOL pool=${ZPOOL} status=OK"
    else
        notify_watchers "${ZFS_CRIT}" "RSF_SERVICE_STARTUP" "service=${service} resource=ZPOOL pool=${ZPOOL} status=FAILED"
    fi

    return ${zstat}
}

###################################################################################
#
# Function:    zfs_export()
#
# Description:
#              method to handle export of ZFS pool
#
# Args:
#              zpool to export
#              force export (string true/false)
#
# Return:      0  - all OK
#              >0 - error occured from zpool export
#
zfs_export()
{
    ZPOOL=""

    if [ $# -ne 1 ] ; then
        dated_echo "Assertion failed - zfs_export called with incorrect number of args: $*" ERROR
        return 1
    else
        ZPOOL=$1
    fi

    if [ -z "${PROP_ZPOOL_EXPORT_OPTION_C}" ] ; then
        dated_echo "### missing property: PROP_ZPOOL_EXPORT_OPTION_C, setting to false"
        PROP_ZPOOL_EXPORT_OPTION_C="${BOOL_STR_FALSE}"
    fi
    dated_echo "zfs_export: export cache save option: ${PROP_ZPOOL_EXPORT_OPTION_C}"

    if [ "${PROP_ZPOOL_EXPORT_OPTION_C}" = "${BOOL_STR_FALSE}" ] ; then
        sync_zfs_cache ${CACHELIVEFILE} ${CACHEFILE}
        ZPOOL_EXPORT_SAVE_CACHE=
    else
        dated_echo "Issuing zpool export with ${ZPOOL_EXPORT_SAVE_CACHE} cache option."
        ZPOOL_EXPORT_SAVE_CACHE="-c"
    fi

    #
    # NEX-19354.
    # Set the failmode back to what it was before clustering. This is
    # also done when removing a service/pool from a cluster, but if that
    # service is not running anywhere when it is destroyed (i.e. the pool
    # is exported), then the pool will be left with failmode=panic set,
    # hence the need to set the mode each time we export the pool.
    ORIG_FAILMODE=`${HAC_PYTHON} ${PROG_RSFPYUTILS} --getPreClusterFailmode ${ZPOOL} 2>/dev/null`
    [ -n "${ORIG_FAILMODE}" ] && zpool set failmode=${ORIG_FAILMODE} ${ZPOOL}

    zpool_export "${THREAD_OPT} ${ZPOOL_EXPORT_SAVE_CACHE} ${ZPOOL}"
    zstat=$?

    #
    # Check for non zero exit code.
    if [ ${zstat} -ne 0 ] ; then
        dated_echo "Forcibly exporting ${ZPOOL} ZFS pool"

        zpool_export "${ZPOOL_EXPORT_SAVE_CACHE} -f ${THREAD_OPT} ${ZPOOL}"
        zstat=$?

        if [ ${zstat} -ne 0 ] ; then
            #
            # Set failmode back to panic.
            FAILMODE=`${HAC_PYTHON} ${PROG_RSFPYUTILS} --getZpoolFailmode ${ZPOOL} 2>/dev/null`
            if [ -n "${FAILMODE}" ]; then
                zpool set failmode=${FAILMODE} ${ZPOOL}
            fi
            #
            # Tell all listeners of the failure.
            notify_watchers "${ZFS_CRIT}" "RSF_SERVICE_SHUTDOWN" "service=${service} resource=ZPOOL pool=${ZPOOL} status=FAILED"
            run_pool_hook export failure ${ZPOOL}
            write_broken_log "Failed to forcibly export ${ZPOOL}"
            do_init
            curl -k X POST -H 'Content-Type: application/json' -d '{"svcname":"'"${service}"'", "message":"S20zfs: Failed to forcibly export "'"${ZPOOL}"'"}' https://localhost:13514/ScriptAlert
            rc_exit ${service} ${RSF_ABORT}
        fi
    fi

    notify_watchers "${ZFS_INFO}" "RSF_SERVICE_SHUTDOWN" "service=${service} resource=ZPOOL pool=${ZPOOL} status=OK"

    #
    # Export worked and cache file sync option set on.
    if [ ${zstat} -eq 0 -a "${PROP_ZPOOL_EXPORT_OPTION_C}" = "${BOOL_STR_TRUE}" ]; then
        dated_echo "Export complete - saving live cache ${CACHELIVEFILE} to ${CACHEFILE}" SUCCESS
        sync_zfs_cache ${CACHELIVEFILE} ${CACHEFILE}
    fi

    return ${zstat}
}

stmfha_lock()
{
    if [ "`uname -s`" != "SunOS" ] ;  then
        dated_echo "stmfha_lock(): not required for non SunOS hosts"
        return
    fi

    if ! [ "${PROP_COMSTAR_SUPPORT}" = "${BOOL_STR_TRUE}" ]; then
        dated_echo "stmfha_lock(): COMSTAR support disablad - nothing to do"
        return
    fi

    WITH_LOCK=${PRODUCT_BIN}/with_lock
    LOCK_TIMEOUT=5
    LOCK_FILE=$1

    dated_echo "Locking file ${LOCK_FILE}"
    #
    # Use with_lock to lock the file, but we want to do some work
    # while it's locked, so put with_lock in the background.
    #
    # with_lock will block until the file lock is established. If
    # it reaches its timeout, it will exit without locking anything
    # and we'll have to operate without file locks - still want to
    # avoid broken_safe!
    #
    # The following heredoc defines the child process that is
    # kicked off by with_lock.
    #
    ${WITH_LOCK} -t ${LOCK_TIMEOUT} ${LOCK_FILE} /bin/sh -s ${LOCK_FILE} $$ ${service} ${script}-child <<-"EOF" &
	LOCKFILE=$1
	PARENT_PID=$2
	#
	# Variables to make dated_echo work
	#
	service=$3
	script=$4
	#
	# Child PID
	#
	PID=$$

	finish()
	{
	    rm -f ${LOCKFILE}
	    dated_echo "Child process ${PID} exiting"
	    exit 0
	}

	#
	# When we (the child) get the USR1 signal, exit.
	# When we exit, the lock will disappear
	#
	trap finish USR1

	#
	# Sourcing rsf.sh can be slow - avoid that
	# delay until after the trap has been set up
	#
	DB_PROPS_DEFINED=true
	. /opt/HAC/bin/rsf.sh

	dated_echo "Child process ${PID} established to hold lock on ${LOCKFILE}"

	#
	# Always check the existence of the parent process and
	# exit if it goes away. We don't want file locks hanging
	# round if the parent exits without sending USR1.
	#
	while [ -d /proc/${PARENT_PID} -a -f ${LOCKFILE} ]; do
	    #
	    # Print my PID to lock file to allow parent
	    # process to send signals to the child
	    #
	    echo ${PID} > ${LOCKFILE}
	    #
	    # Now put the child process to sleep. We should
	    # exit when the parent tells us to, but in the
	    # event that the parent has died, we need to be
	    # able to clean up the child process, so don't
	    # make the sleep too long.
	    #
	    sleep 5
	done

	#
	# Child has left the main loop without a signal
	#  - parent must have died
	#
	finish
	EOF
    #
    # Back in the parent proc now - remember to send
    # USR1 to the child to unlock the lock file
    #

    if [ $? -eq 255 ]; then
        dated_echo "Timed out waiting for file lock on ${LOCK_FILE} - continuing anyway" WARN
    fi
}

stmfha_unlock()
{
    if [ "`uname -s`" != "SunOS" ] ;  then
        dated_echo "stmfha_unlock(): not required for non SunOS hosts"
        return
    fi

    if ! [ "${PROP_COMSTAR_SUPPORT}" = "${BOOL_STR_TRUE}" ]; then
        dated_echo "stmfha_unlock(): COMSTAR support disablad - nothing to do"
        return
    fi

    #
    # If the stmfha lock file is locked, there will be a
    # child process running, holding the lock. If we kill
    # that process, with_lock will unlock the file and exit
    #
    LOCK_FILE=$1
    UNLOCK_TIMEOUT=50

    child_pid=`cat ${LOCK_FILE}` 2>/dev/null
    if [ -z "${child_pid}" ]; then
        dated_echo "Lock file ${LOCK_FILE} was not locked" WARN
        return
    fi

    #
    # Child process should exit when it catches USR1
    #
    dated_echo "Sending USR1 signal to child process ${child_pid}"
    kill -USR1 ${child_pid}

    t=0
    while [ ${t} -lt ${UNLOCK_TIMEOUT} -a -d /proc/${child_pid} ]; do
        t=`expr ${t} + 1`
        sleep 0.1
    done

    if [ ${t} -eq ${UNLOCK_TIMEOUT} ]; then
        #
        # Forcefully kill the child
        #
        dated_echo "Timed out waiting for stmfha lock child ${child_pid}" ERROR
        rm -f ${LOCK_FILE}
        kill -9 ${child_pid}
    else
        dated_echo "stmfha lock child ${child_pid} has exited OK"
    fi
}

#
# ZFS Logging
#
# All notifies are sent either ZFS_INFO or ZFS_CRIT, the second argument will be
# the action and the third is service=<servicename> (i.e. the pool name).
#
# Possible action messages are:
#  IMPORT          - an import is being attempted
#  IMPORT_FORCE    - a forced import is being attempted
#  IMPORT_FAILED   - the forced import failed
#  IMPORT_COMPLETE - the import completed
#
#  EXPORT          - an export is being attempted
#  EXPORT_FORCE    - a forced export is being attempted
#  EXPORT_FAILED   - the forced export failed
#  EXPORT_COMPLETE - the export completed
#
#  If a pool is already imported then no ZFS import notification message is sent.
#  On an unimported pool, a ZFS_INFO message is sent with the first argument of
#  IMPORT, if this fails then ZFS_INFO is sent with IMPORT_FORCE and a forced
#  import is attempted. If the forced import fails then a ZFS_CRIT IMPORT_FAILED
#  message is sent, otherwise if it succeeds then a ZFS_INFO IMPORT_COMPLETE
#  message is sent.
#
#  The same sequence of events is performed for zfs exports but with EXPORT
#  replacing the IMPORT string.
#
case "${State}" in

'start')
        #
        # Workaround: kill svc-stmfproxy prior to failover
        # Because the stmfproxy is run by rsfpmon the kill will invoke an
        # automatic restart.
        if [ -f "${STMFPROXY_CONFIG}" ] ; then
            stmfproxy_pid=`pgrep svc-stmfproxy`
            dated_echo "Killing svc-stmfproxy pid:${stmfproxy_pid}"
            /usr/bin/pkill svc-stmfproxy
            /usr/bin/kill -0 ${stmfproxy_pid} >/dev/null 2>&1 && dated_echo "Failed to kill svc-stmfproxy during service start"
        fi

        while get_next_pool_vars
        do
            dated_echo "${POOL_NAME}: GUID = ${POOL_GUID}"
            set_paths ${POOL_NAME}
            dated_echo "Importing ${POOL_NAME} ZFS pool"
            clear_zpool_cache ${POOL_NAME}

            if [ "${ZPOOL_ID}" = "GUIDREPLACEME" ] ; then
                HOSTNAME=`hostname`
                POOL_GUID=`/opt/HAC/RSF-1/bin/rsfcdb read ${POOL_NAME}_${HOSTNAME}_guid`
            fi


            if ! clean_mountpoint ${POOL_NAME} ; then
                dated_echo "### Import directory not clean - import could fail" WARN
                dated_echo "### Please manually clean directory - attempting import" WARN
            fi

            pool_is_imported ${POOL_NAME} ${POOL_GUID}
            case $? in
            0)
                dated_echo "ZFS pool ${POOL_NAME} was already imported" WARN
                FAILMODE=`${HAC_PYTHON} ${PROG_RSFPYUTILS} --getZpoolFailmode ${POOL_NAME} 2>/dev/null`
                [ -n "$FAILMODE" ] && zpool set failmode=${FAILMODE} ${POOL_NAME}
                ;;
            1|2)
                curl -k X POST -H 'Content-Type: application/json' -d '{"svcname":"'"${service}"'", "message":"S20zfs: Imported pool found with same name or GUID as "'"${POOL_NAME}"'"}' https://localhost:13514/ScriptAlert
                rc_exit ${service} ${RSF_ABORT} "Imported pool found with same name or GUID as ${POOL_NAME}"
                ;;
            3)
                #
                # Lock stmfha-<pool>.lck to tell stmfha not to run any backups
                # of this pool until we have finished importing it
                #
                stmfha_lock ${PRODUCT_RUN}/stmfha-${POOL_NAME}.lck

                run_pool_hook import before ${POOL_NAME}

                # check_zpool_version ${POOL_NAME}
                zfs_import ${POOL_NAME}
                zstat=$?

                if [ ${zstat} -ne 0 ] ; then
                    write_broken_log "Zpool import of ${POOL_NAME} failed with errors (exit code ${zstat})"
                    run_pool_hook import failure ${POOL_NAME}
                    #
                    # Unlock stmfha lock file and abort
                    #
                    stmfha_unlock ${PRODUCT_RUN}/stmfha-${POOL_NAME}.lck
                    curl -k X POST -H 'Content-Type: application/json' -d '{"svcname":"'"${service}"'", "message":"S20zfs: Zpool import of "'"${POOL_NAME}"'" failed with errors (exit code "'"${zstat})"'"}' https://localhost:13514/ScriptAlert
                    rc_exit ${service} ${RSF_ABORT}
                else
                    if [ "${PROP_ZPOOL_EXPORT_OPTION_C}" = "${BOOL_STR_FALSE}" ] ; then
                        sync_zfs_cache ${CACHELIVEFILE} ${CACHEFILE}
                    fi
                    save_zpool_version ${POOL_NAME}

                    #
                    # Import succeeded - run STMFHA and notify watchers
                    if [ "`uname -s`" = "SunOS" ] ;  then
                        if [ "${PROP_COMSTAR_SUPPORT}" = "${BOOL_STR_TRUE}" ] && [ -x "${PROG_STMFHA}" ] ; then
                            STMFHA_START_TIME=`${DATE}`
                            dated_echo "Running comstar mapping restore"
                            ${PROG_STMFHA} -c ${STMFHA_THREAD_COUNT} restore ${POOL_NAME}
                            STMFHA_RESTORE_TIME=`expr \`${DATE}\` - ${STMFHA_START_TIME} `
                            dated_echo "Comstar mapping restored in ${STMFHA_RESTORE_TIME} seconds" SUCCESS
                        fi
                    fi

                    run_pool_hook import after ${POOL_NAME}
                fi

                #
                # Unlock stmfha lock file
                #
                stmfha_unlock ${PRODUCT_RUN}/stmfha-${POOL_NAME}.lck
                ;;
            esac
        done

        rm -f ${ZPOOL_LOG}

        rc_exit ${service} ${RSF_OK}
        ;;

'stop')
    # Shared nothing cluster? Then do not export the pool
    SHARED_NOTHING=`/opt/HAC/RSF-1/bin/rsfcdb read shared_nothing_cluster`

    if [ "`uname -s`" = "SunOS" ] ;  then
        if [ -x "${PROG_STMFHA}" ] ; then
            dated_echo "Starting stmfha check-sync as a background process"
            ${PROG_STMFHA} check-sync -m &
        fi
    fi

    while get_next_pool_vars
    do
        set_paths ${POOL_NAME}

        mntpoint=`zfs get mountpoint ${POOL_NAME}|awk '$1 !~ /NAME/ {print $3}'`

        if [ -z "${mntpoint}" ] ; then
            dated_echo "ZFS pool ${POOL_NAME} appears not to be mounted on ${mntpoint}!"
        else
            dated_echo "Exporting ${POOL_NAME} ZFS pool mounted on ${mntpoint}"
        fi

        zpool status -x ${POOL_NAME} >/dev/null 2>&1

        if [ $? -eq 1 ] ; then
            dated_echo "${POOL_NAME} ZFS pool was already exported" WARN
        else
            #
            # Check if we are destroying the cluster and if so
            # set the -f flag to force LUN removal in ALUA mode.
            DESTROY="${ZPOOL_IMPORT_POINT}/${POOL_NAME}/.destroy"

            if [ -f  "${DESTROY}" ] ; then
                STOP_STMFHA_ARGS="-f "
                export STOP_STMFHA_ARGS
                dated_echo "Note: volume is being removed from cluster control and therefore ALUA LU's will be removed"
            else
                STOP_STMFHA_ARGS=""
                export STOP_STMFHA_ARGS
            fi

            #
            # Once done with, remove the destroy flag
            rm -f ${DESTROY}

            #
            # Lock stmfha-<pool>.lck to tell stmfha not to run any backups
            # of this pool until we have finished exporting it
            #
            stmfha_lock ${PRODUCT_RUN}/stmfha-${POOL_NAME}.lck

            if [ "${PROP_COMSTAR_SUPPORT}" = "${BOOL_STR_TRUE}" ]; then
                if [ "`uname -s`" = "SunOS" ] ;  then
                    STMFHA_START_TIME=`${DATE}`
                    #
                    # Call STMFHA before export
                    if [ -x "${PROG_STMFHA}" ] ; then
                        dated_echo "Running stmfha ${STOP_STMFHA_ARGS}backup --remove-after-backup"
                        ${PROG_STMFHA} -B ${STMFHA_BACKUP_COUNT} ${STOP_STMFHA_ARGS} backup --remove-after-backup ${POOL_NAME}
                    fi
                fi

                #
                # Normally null - for non Solaris machine this is the only thing that gets
                # run in this loop - probably not needed - but we'll leave for now.
                run_pool_hook export before ${POOL_NAME}

                #
                # Check for old LU's - STMF_PURGE is only shipped with Solaris packages.
                if [ "`uname -s`" = "SunOS" ] ;  then
                    if [ -x "${STMF_PURGE}" ] ; then
                        LUS=`${STMF_PURGE} -l -x ${POOL_NAME}`
                        if [ $? -eq 0 ] ; then
                            cnt=1
                            dated_echo "${POOL_NAME}: logical units still exist on volume - attempting forcible removal of:" WARN
                            echo ${LUS}
                            while [ ${cnt} -le 30 ] ; do
                                ${STMF_PURGE} -d ${POOL_NAME}
                                cnt=`expr $cnt + 1`
                                ${STMF_PURGE} -l -x ${POOL_NAME} >/dev/null 2>&1
                                if [ $? -eq 1 ] ; then
                                    break
                                fi
                                sleep 1
                            done
                        else
                            dated_echo "${POOL_NAME}: no iSCSI logical units need removing"
                        fi

                        ${STMF_PURGE} -l -x ${POOL_NAME}
                        if [ $? -eq 0 ] ; then
                            dated_echo "${POOL_NAME}: unable to delete logical units from volume - export may fail with device busy" WARN
                        fi
                    else
                        dated_echo "${POOL_NAME}: ${STMF_PURGE} is missing or is not executable, cannot check iSCSI logical units" ERROR
                    fi

                    STMFHA_REMOVE_TIME=`expr \`${DATE}\` - ${STMFHA_START_TIME} `
                    dated_echo "Comstar mapping removed in ${STMFHA_REMOVE_TIME} seconds" SUCCESS
                fi
            fi

            if [ "${SHARED_NOTHING}" = "true" ] ; then
                # We do not export and we do not run anything else
                dated_echo "Shared nothing cluster - neither exporting the pool nor running any hooks"
                rc_exit ${service} ${RSF_OK}
            else
                zfs_export ${POOL_NAME}
            fi

            #
            # Unlock stmfha lock file
            #
            stmfha_unlock ${PRODUCT_RUN}/stmfha-${POOL_NAME}.lck

            #
            # Export OK so clean mount point.
            if ! clean_mountpoint ${POOL_NAME} ; then
                dated_echo "clean_mountpoint() failed - pool could still be imported! Aborting..." ERROR
                curl -k X POST -H 'Content-Type: application/json' -d '{"svcname":"'"${service}"'", "message":"S20zfs: clean_mountpoint() failed - pool could still be imported! Aborting..."}' https://localhost:13514/ScriptAlert
                rc_exit ${service} ${RSF_ABORT}
            fi

            #
            # Export succeeded - delete mount point and notify watchers.
            # Mount point is deleted to prevent files/dirs being created
            # within it that subsequently will cause zfs import to fail.
            if [ ! -z "${mntpoint}" -a -d "${mntpoint}" ] ; then
                dated_echo "Removing mount directory: ${mntpoint}"

                if ! rmdir "${mntpoint}" ; then
                    dated_echo "Failed to remove mount directory!" ERROR
                    if is_nef ; then
                        dated_echo "Checking /etc/mnttab for mount directory: ${mntpoint}"
                        scan_mounts /etc/mnttab "$mntpoint"
                        if [ $? -eq 0 ] ; then
                            dated_echo "No entry in /etc/mnttab for mount directory"
                        else
                            dated_echo "Running fuser -cu on mount directory"
                            fuser -cu ${mntpoint}
                        fi
                    fi
                    curl -k X POST -H 'Content-Type: application/json' -d '{"svcname":"'"${service}"'", "message":"S20zfs: Problem with mount directory, please check the logs"}' https://localhost:13514/ScriptAlert
                    rc_exit ${service} ${RSF_ABORT}
                fi
            fi

            run_pool_hook export after ${POOL_NAME}

            clear_zpool_cache ${POOL_NAME}

            if [ "${NOZFSCACHEFILE}" != "${BOOL_STR_TRUE}" ] ; then
                save_zpool_version ${POOL_NAME}
            fi
        fi
    done

        rm -f ${ZPOOL_LOG}
        rc_exit ${service} ${RSF_OK}
        ;;

'check')
        online_pools=
        offline_pools=
        while get_next_pool_vars; do
                if pool_is_imported $POOL_NAME; then
                        online_pools="${online_pools} $POOL_NAME"
                        echo "ONLINE: ZFS pool $POOL_NAME"
                else
                        offline_pools="${offline_pools} $POOL_NAME"
                        echo "OFFLINE: ZFS pool $POOL_NAME"
                fi
        done

        if [ -n "${online_pools}" -a -n "${offline_pools}" ]; then
                retval=${RSF_CHECK_PARTIAL}
        elif [ -n "${online_pools}" ]; then
                retval=${RSF_CHECK_ONLINE}
        elif [ -n "${offline_pools}" ]; then
                retval=${RSF_CHECK_OFFLINE}
        else
                retval=${RSF_CHECK_NORESRC}
        fi

        exit ${retval}
        ;;

*)      echo "usage: $0 [start|stop|check] <attempts>"
        rc_exit ${service} ${RSF_WARN}
        ;;

esac
