Commit 67ec36c3 by Eric Coissac

Initial commit of the LECA Bash Library

parents
File added
File added
File added
File added
File added
# LECA Bash library
#
# The LECA bash library provides a set of function used for helping
# development of bash script mainly to write job script on the luke
# cluster
#
# The files from the LECA Bash library must be sourced from your main script
#
#
THIS_DIR="$(dirname ${BASH_SOURCE[0]})"
source "${THIS_DIR}/utils.sh"
include stack
newstack __LECABASHLIB_CLOBBERSTACK__
function setnoclobber() {
set -o noclobber
}
function setclobber() {
set +o noclobber
}
function getnoclobber() {
set -o | awk '($1 ~ /noclobber/) {print $2}'
}
function pushclobber() {
pushvalue __LECABASHLIB_CLOBBERSTACK__ $(getnoclobber)
setclobber
}
function pushnoclobber() {
pushvalue __LECABASHLIB_CLOBBERSTACK__ $(getnoclobber)
setnoclobber
}
function popclobber() {
local state=$(popvalue __LECABASHLIB_CLOBBERSTACK__)
if [[ "${state}"]]
}
function popnoclobber() {
popclobber
}
# LECA Bash library
#
# The LECA bash library provides a set of function used for helping
# development of bash script mainly to write job script on the luke
# cluster
#
# The files from the LECA Bash library must be sourced from your main script
#
#
# Manages the clobber/noclobber property of the bash shell
# ========================================================
#
# When the noclobber mode is on you cannot substitute an existing file by a
# empty one using a unix redirection.
#
# Functions provided:
#
# - setnoclobber
# Sets the noclobber mode on.
# You cannot erase an existing file by a redirection
#
# - setclobber
# Sets the noclobber mode off.
# You can erase an existing file by a redirection
#
# - getnoclobber
# Returns the noclobber status as a string
# - on: the noclobber mode is on
# - off: the noclobber mode is off
#
# - pushclobber
# Saves the current noclobber mode on a stack and sets it to 'off'
#
# - pushnoclobber
# Saves the current noclobber mode on a stack and sets it to 'on'
#
# - popclobber
# Restores the last pushed noclobber mode
#
# - popnoclobber
# Alias for the popclobber function
#
THIS_DIR="$(dirname ${BASH_SOURCE[0]})"
source "${THIS_DIR}/utils.sh"
include stack
newstack __LECABASHLIB_CLOBBERSTACK__
function setnoclobber() {
set -o noclobber
}
function setclobber() {
set +o noclobber
}
function getnoclobber() {
set -o | awk '($1 ~ /noclobber/) {print $2}'
}
function pushclobber() {
pushvalue __LECABASHLIB_CLOBBERSTACK__ $(getnoclobber)
setclobber
}
function pushnoclobber() {
pushvalue __LECABASHLIB_CLOBBERSTACK__ $(getnoclobber)
setnoclobber
}
function popclobber() {
local state=$(popvalue __LECABASHLIB_CLOBBERSTACK__)
if [[ "$?" == "0"]]; then
if [[ "${state}" == "on" ]]; then
setnoclobber
else
setclobber
fi
else
return 1
fi
return 0
}
function popnoclobber() {
popclobber
return $?
}
#
# LECA Bash library
#
# The LECA bash library provides a set of function used for helping
# development of bash script mainly to write job script on the luke
# cluster
#
# The files from the LECA Bash library must be sourced from your main script
#
#
# Provides functions to facilitate downloading of files
# =====================================================
#
# Provided function:
#
# - download <URL> [USERNAME] [PASSWORD]
# Downloads the file specified by URL.
# If specifed the second (USERNAME) and third (PASSWORD) parametters allow
# to indicate a login and password for a protected connection
#
# - downloadandcheck <URL> <MD5URL> <DESTINATION> [USERNAME] [PASSWORD]
# Downloads the file specified by URL and check file integrity.
# The MD5 checksum file URL is indicated using the MD5URL parametters.
# You must specified a destination filename to store the downloaded data
# using the DESTINATION parametter. The MD5 file is downloaded under the
# name DESTINATION.md5
# If specifed the fourth (USERNAME) and fifth (PASSWORD) parametters allow
# to indicate a login and password for a protected connection
#
# On error the function has a return status set to 1, 0 otherwise.
#
# - setmaxretry <MAXRETRY>
# If downloading failed the downloadandcheck function can retry to download
# the file several times before a complete failure. This function allows to
# define the maximum of time the downloadandcheck tries to redownload before
# exiting in error
#
#
THIS_DIR="$(dirname ${BASH_SOURCE[0]})"
source "${THIS_DIR}/utils.sh"
if onluke; then
module load curl/7.24.0_gcc-4.6.2
module load openssl/1.0.0f_gcc-4.6.2
fi
CURL="curl -L"
LECA_MAX_RETRY=3
function download() {
logdebug "Function called with $# arguments"
logdebug "$@"
local URL="${1}"
shift
local LINK="${CURL} ${URL}"
if [[ ! -z $1 && -z $2 ]]; then
logdebug "Download with username ${1}"
local LINK="${CURL} --user ${1} ${URL}"
shift
fi
if [[ ! -z $1 && ! -z $2 ]]; then
logdebug "Download with username ${1} and password ${2}"
local LINK="${CURL} --user ${1}:${2} ${URL}"
shift
shift
fi
loginfo "Downloading URL : ${URL}"
${LINK}
loginfo "URL : ${URL} -> Done."
}
function setmaxretry() {
LECA_MAX_RETRY="$1"
}
function downloadandcheck() {
local URL="${1}"
shift
local MD5="${1}"
shift
local TARGETFILE="${1}"
shift
if [[ ! -z "${1}" ]]; then
local DOWLOADUSER="${1}"
shift
fi
if [[ ! -z "${1}" ]]; then
local PASSWORD="${1}"
shift
fi
local RETRY=0
local MD5_BAD=1
while (( MD5_BAD > 0 )) && (( RETRY < LECA_MAX_RETRY )); do
download "${URL}" "${DOWLOADUSER}" "${PASSWORD}" > "${TARGETFILE}"
download "${MD5}" "${DOWLOADUSER}" "${PASSWORD}" > "${TARGETFILE}.md5"
MD5LOCAL=$(md5sum "${TARGETFILE}" | awk '{print $1}')
MD5ORIGI=$(awk '{print $1}' "${TARGETFILE}.md5")
if [[ "$MD5LOCAL" == "$MD5ORIGI" ]]; then
MD5_BAD=0
else
MD5_BAD=1
fi
RETRY=$((RETRY+1))
if (( MD5_BAD > 0 )); then
logwarning "The two MD5 are different : $MD5LOCAL and $MD5ORIGI"
fi
done
if (( MD5_BAD > 0 )); then
logerror "The two MD5 are different after $RETRY retries: "
logerror " $MD5LOCAL and $MD5ORIGI"
return 1
else
loginfo "${TARGETFILE} downloaded"
loginfo " and checked with MD5 $MD5ORIGI"
return 0
fi
}
#
# LECA Bash library
#
# The LECA bash library provides a set of function used for helping
# development of bash script mainly to write job script on the luke
# cluster
#
# The files from the LECA Bash library must be sourced from your main script
#
#
# Provides a set of variables relative to the LECA environement
# =============================================================
#
# The lecaluke module provides a set of variable defining emplacement
# where LECA ressources are stored on the LUKE cluster.
THIS_DIR="$(dirname ${BASH_SOURCE[0]})"
source "${THIS_DIR}/utils.sh"
# Where is located the LECA_ENVIRONEMENT
LECA_ENVIRONEMENT=/nfs_scratch/LECA_ENVIRONEMENT
# Lock files are stored in a common place
LOCK_DIR="${LECA_ENVIRONEMENT}/locks"
# All the data shared among users can be stored here
DATA_DIR="${LECA_ENVIRONEMENT}/data"
# All the molecular public databases downloaded for users on the luke cluster
# will be downloaded here
BIODATA_DIR="${DATA_DIR}/biodatabase"
# Main directory for the NCBI taxonomy
TAXONOMY_DIR="${BIODATA_DIR}/taxonomy"
#
# LECA Bash library
#
# The LECA bash library provides a set of function used for helping
# development of bash script mainly to write job script on the luke
# cluster
#
# The files from the LECA Bash library must be sourced from your main script
#
#
# Logging facilities for bash
# ===========================
#
# Provides the following functions:
#
# openlogfile <FILENAME>
# Redirected all logging to the file specified by FILENAME.
# If the file already exists, new logs are appened at the end of the file.
# If the file does not exist it is created
#
# closelogfile
# Closes the current logfile and redirect the logging to stderr.
#
# logdebug <MESSAGE>
# Writes message as a debug level log to the current log file.
#
# loginfo <MESSAGE>
# Writes message as an info level log to the current log file.
#
# logwarning <MESSAGE>
# Writes message as a warning level log to the current log file.
#
# logerror <MESSAGE>
# Writes message as an error level log to the current log file.
#
# setloglevel <LEVEL>
# Set the current logging level to the level specified by LEVEL.
# Level is DEBUG, INFO, WARNING or ERROR.
# A log is recorded only when it is submitted with a level
# greater or equal to the current logging level.
#
# All the log will be done on file descriptor 3
# By default file descriptor 3 is redirected to stderr (2)
# By default the logging level is set to WARNING.
# Therefore only WARNING and ERROR messages will be actually logged.
THIS_DIR="$(dirname ${BASH_SOURCE[0]})"
source "${THIS_DIR}/utils.sh"
LOG_DEBUG_LEVEL=1
LOG_INFO_LEVEL=2
LOG_WARNING_LEVEL=3
LOG_ERROR_LEVEL=4
LOG_LEVEL=2
exec 3>&2
#####
#
# The logging function
function logdebug() {
if (( LOG_LEVEL <= LOG_DEBUG_LEVEL )); then
echo `date +'%Y-%m-%d %H:%M:%S'` "[INFO ] $$ -- $*" 1>&3
fi
}
function loginfo() {
if (( LOG_LEVEL <= LOG_INFO_LEVEL )); then
echo `date +'%Y-%m-%d %H:%M:%S'` "[INFO ] $$ -- $*" 1>&3
fi
}
function logerror() {
if (( LOG_LEVEL <= LOG_ERROR_LEVEL )); then
echo `date +'%Y-%m-%d %H:%M:%S'` "[ERROR ] $$ -- $*" 1>&3
fi
}
function logwarning() {
if (( LOG_LEVEL <= LOG_WARNING_LEVEL )); then
echo `date +'%Y-%m-%d %H:%M:%S'` "[WARNING] $$ -- $*" 1>&3
fi
}
function setloglevel () {
LOG_LEVEL=$(indirect "LOG_${1}_LEVEL")
}
function openlogfile() {
exec 3>> "$1"
LOGFILE="$1"
}
function closelogfile() {
if [[ ! -z "${LOGFILE}" ]]; then
exec 3>&-
exec 3>&2
LOGFILE=""
fi
}
# LECA Bash library
#
# The LECA bash library provides a set of function used for helping
# development of bash script mainly to write job script on the luke
# cluster
#
# The files from the LECA Bash library must be sourced from your main script
#
#
THIS_DIR="$(dirname ${BASH_SOURCE[0]})"
source "${THIS_DIR}/utils.sh"
if onluke; then
include lecaluke
else
LOCK_DIR="/tmp"
fi
LECA_LOCK_DELAY=2
function lockfile() {
echo "${LOCK_DIR}/__${1}__.lecalock"
}
function getlock() {
local name=$(lockfile "$1")
shift
notgetlock=1
while (( notgetlock )); do
notgetlock=$?
if (( notgetlock )); then
sleep "${LECA_LOCK_DELAY}"
fi
done
cat "${TAG}" >
}
# LECA Bash library
#
# The LECA bash library provides a set of function used for helping
# development of bash script mainly to write job script on the luke
# cluster
#
# The files from the LECA Bash library must be sourced from your main script
#
#
# Provides function for manipulating a stack of values
# ====================================================
#
# Provides function:
#
# - newstack <STACKNAME>
# Creates a new stack named STACKNAME
#
# - stacksize <STACKNAME>
# Returns the count of element stored in the stack
#
# - pushvalue <STACKNAME> <VALUE>
# Adds the value VALUE at the end of the stack STACKNAME
#
# - popvalue <STACKNAME>
# Pops out the last pushed value of the stack.
# If the stack is already empty the function returns nothing
# and the return status is set to 1 instead of 0 usually
#
THIS_DIR="$(dirname ${BASH_SOURCE[0]})"
source "${THIS_DIR}/utils.sh"
function newstack() {
eval declare -a "${1}"
}
function stacksize() {
eval "echo \${#${1}[@]}"
}
function pushvalue() {
eval ${1}[$(stacksize ${1})]=${2}
}
function popvalue() {
local s=$(stacksize ${1})
if (( s==0 )); then
return 1
fi
local valcmd="echo \${${1}[\$(( s - 1 ))]}"
eval $valcmd
local valcmd="unset ${1}[\$(( s - 1 ))]"
eval $valcmd
return 0
}
# LECA Bash library
#
# The LECA bash library provides a set of function used for helping
# development of bash script mainly to write job script on the luke
# cluster
#
# The files from the LECA Bash library must be sourced from your main script
#
#
if [[ -z "${LECALIB_UTIL_SH}" ]]; then
LECALIB_UTIL_SH=1
THIS_DIR="$(dirname ${BASH_SOURCE[0]})"
source "${THIS_DIR}/utils.sh"
LECALIB_DIR="${THIS_DIR}"
function upper() {
echo $1 | tr "a-z" "A-Z"
}
function lower() {
echo $1 | tr "A-Z" "a-l"
}
function indirect() {
eval echo \$${1}
}
function include() {
local MODULE=$(upper $1)
local LOADED_MODULE_VAR="LECALIB_${MODULE}_SH"
local LOADED_MODULE=$(indirect $LOADED_MODULE_VAR)
if [[ -z "${LOADED_MODULE}" ]]; then
source "${LECALIB_DIR}/${1}.sh"
eval $LOADED_MODULE_VAR=1
fi
}
function onluke() {
[[ $(hostname | sed -E 's/^(....).*$/\1/') == "luke" ]]
}
function onfroggy() {
[[ $(hostname | sed -E 's/^(......).*$/\1/') == "froggy" ]]
}
include logging
if onluke; then
loginfo Script run on the LUKE cluster
source ${LECALIB_DIR}/../etc/bashrc
fi
fi
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment