Patch few bugs

README.md

@@ -14,9 +14,49 @@ For activating the *LECA Bash Library* you must source the `util.sh` bash script
 	> source $LECABASHHOME/utils.sh
 ```
 
+----------------------------------------------------------
 
+# Modules provided by the *LECA Bash Library*
 
-# modules
+- **[atexit](#atexit)**     : for automatically execute task at the end of the execution
+- **[clobber](#clobber)**   : for managing the clobber/noclobber status
+- **[download](#download)** : for managing download from FTP and Web including MD5 checks
+- **[ifs](#ifs)**           : for managing the *input field separator*
+- **[lecaluke](#lecaluke)** : declare variable related for the LECA computers in LUKE
+- **[logging](#logging)**   : adds functionnalities for managing log files and log levels
+- **[mutex](#mutex)**       : manages concurent access to resources through locks and semaphores
+- **[sets](#sets)**         : manages *set* of values 
+- **[stacks](#stacks)**     : manages *stacks* of values
+- **[tempdir](#tempdir)**   : manages the creation of temporary directories and their erasements
+- **[timeout](#timeout)**   : provides a function for limiting the maximum execution time of a command
+
+----------------------------------------------------------
+
+# Detailed description of modules
+
+## atexit
+
+### Activating the module
+
+```{shell}
+	> source $LECABASHHOME/utils.sh
+	> include atexit
+```
+
+### Functions provided:
+
+
+#### registeratexit
+
+Allows to register commands that will be automatically executed at the end
+of the script execution.
+
+```{shell}
+	> include atexit
+	> registeratexit "echo this is the end of the script"
+```
+
+----------------------------------------------------------
 
 ## clobber
 
@@ -64,6 +104,10 @@ You can erase an existing file by a redirection
 
 #### getnoclobber
 
+```{shell} 
+    getnoclobber
+```
+
 Returns the noclobber status as a string
 
 -  `on`: the noclobber mode is on
@@ -74,30 +118,460 @@ Returns the noclobber status as a string
 	> getnoclobber
 	off
 ```
+
 #### pushclobber
-  
+
+```{shell} 
+    pushclobber
+```
+
 Saves the current noclobber mode on a stack and sets it to 'off'
 
 #### pushnoclobber
-  
+ 
+```{shell} 
+    pushnoclobber
+```
+
 Saves the current noclobber mode on a stack and sets it to 'on'
 
 #### popclobber
   
+```{shell} 
+    popclobber
+```
+
 Restores the last pushed noclobber mode
 
 #### popnoclobber
-  
-Alias for the popclobber function. Using one of the both commands `popnoclobber` or `popclobber` leads to the same action, restoring the last pushed noclober mode
 
+```{shell} 
+    popnoclobber
+```
 
+Alias for the `popclobber` function. Using one of the both commands `popnoclobber` or `popclobber` leads to the same action, restoring the last pushed noclober mode
+
+----------------------------------------------------------
 
 ## download
 
+### Activating the module
+
+```{shell}
+	> source $LECABASHHOME/utils.sh
+	> include download
+```
+
+### Functions provided:
+
+#### download
+
+```{shell} 
+    download <URL> [USERNAME] [PASSWORD]
+```
+
+Downloads the file specified by `URL`.
+If specifed the second (`USERNAME`) and third (`PASSWORD`) parametters allow
+to indicate a login and password for a protected connection
+
+
+#### downloadandcheck
+
+```{shell} 
+    downloadandcheck <URL> <MD5URL> <DESTINATION> [USERNAME] [PASSWORD]
+```
+
+Downloads the file specified by `URL` and check file integrity.
+The MD5 checksum file URL is indicated using the `MD5URL` parametters.
+You must specified a destination filename to store the downloaded data
+using the `DESTINATION` parametter. The MD5 file is downloaded under the
+name DESTINATION.md5
+If specifed the fourth (`USERNAME`) and fifth (`PASSWORD`) parametters allow
+to indicate a login and password for a protected connection
+
+On error the function has a return status set to *1*, *0* otherwise.
+
+
+
+#### setmaxretry
+
+```{shell} 
+    setmaxretry <MAXRETRY>
+```
+
+If downloading failed the downloadandcheck function can retry to download
+the file several times before a complete failure. This function allows to
+define the maximum of time the downloadandcheck tries to redownload before
+exiting in error.
+
+----------------------------------------------------------
+
+
+
+## ifs
+
+### Activating the module
+
+```{shell}
+	> source $LECABASHHOME/utils.sh
+	> include ifs
+```
+
+### Functions provided:
+
+#### pushifs
+
+```{shell} 
+    pushifs <NEWIFS>
+```
+
+#### popifs
+
+```{shell} 
+    popifs
+```
+
+
+
+----------------------------------------------------------
+
 ## lecaluke
 
+The lecaluke module provides a set of variable defining emplacement
+where LECA ressources are stored on the LUKE cluster.
+
+
+### Activating the module
+
+```{shell}
+	> source $LECABASHHOME/utils.sh
+	> include lecaluke
+```
+
+### Functions provided:
+
+#### latesttaxonomy
+
+```{shell} 
+    latesttaxonomy
+```
+
+### Defined variables
+
+#### LECA_ENVIRONEMENT
+
+#### LOCK_DIR
+
+#### DATA_DIR
+
+#### BIODATA_DIR
+
+#### TAXONOMY_DIR
+
+#### TREMBL_DIR
+
+#### SWISSPROT_DIR
+
+----------------------------------------------------------
+
 ## logging
 
+The *logging* module define a set of function allowing to log on the standard error
+or into a file messages.
+
+All the log will be done on file descriptor 3
+By default file descriptor 3 is redirected to stderr (2)
+By default the logging level is set to `WARNING`.
+Therefore only `WARNING` and `ERROR` messages will be actually logged.
+
+
+### Activating the module
+
+```{shell}
+	> source $LECABASHHOME/utils.sh
+	> include logging
+```
+
+### Functions provided:
+
+#### logdebug
+
+```{shell} 
+    logdebug <MESSAGE>
+```
+
+Writes `MESSAGE` as a debug level log to the current log file.
+
+#### loginfo
+
+```{shell} 
+    loginfo <MESSAGE>
+```
+
+Writes `MESSAGE` as a info level log to the current log file.
+
+
+#### logerror
+
+```{shell} 
+    logerror <MESSAGE>
+```
+
+Writes `MESSAGE` as a error level log to the current log file.
+
+
+#### logwarning
+
+```{shell} 
+    logwarning <MESSAGE>
+```
+
+Writes `MESSAGE` as a warning level log to the current log file.
+
+
+#### setloglevel
+
+```{shell} 
+    setloglevel <LEVEL>
+```
+
+Set the current logging level to the level specified by `LEVEL`.
+Level is `DEBUG`, `INFO`, `WARNING` or `ERROR`.
+A log is recorded only when it is submitted with a level
+greater or equal to the current logging level.
+
+#### openlogfile
+
+```{shell} 
+    openlogfile <FILENAME>
+```
+
+Redirected all logging to the file specified by `FILENAME`.
+If the file already exists, new logs are appened at the end of the file.
+If the file does not exist it is created
+
+#### closelogfile
+
+```{shell} 
+    closelogfile
+```
+
+Closes the current logfile and redirect the logging to stderr.
+
+
+----------------------------------------------------------
+
+
 ## mutex
 
-## stack
\ No newline at end of file
+### Activating the module
+
+```{shell}
+	> source $LECABASHHOME/utils.sh
+	> include mutex
+```
+
+### Functions provided:
+
+#### getlock
+
+```{shell} 
+    getlock <LOCKNAME> [global]
+```
+
+#### releaselock
+
+```{shell} 
+    releaselock <LOCKNAME> [global]
+```
+
+
+#### getsemaphore
+
+```{shell} 
+    getsemaphore <SEMAPHORENAME> <MAXCOUNT> [global]
+```
+
+#### releasesemaphore
+
+```{shell} 
+    releasesemaphore <SEMAPHORENAME> [global]
+```
+
+#### releaseoldlock
+
+```{shell} 
+    releaseoldlock [MINUTES]
+```
+
+### Variable provided
+
+#### LOCK_DIR
+
+#### LECA_LOCK_DELAY
+
+#### LECA_LOCK_MAXRETRY
+
+----------------------------------------------------------
+
+## sets
+
+### Activating the module
+
+```{shell}
+	> source $LECABASHHOME/utils.sh
+	> include sets
+```
+
+### Functions provided:
+
+#### newset
+
+```{shell} 
+    newset <SETNAME>
+```
+
+#### setsize
+
+```{shell} 
+    setsize <SETNAME>
+```
+
+#### setaddvalue
+
+```{shell} 
+    setsize <SETNAME> <VALUE>
+```
+
+#### setdelvalue
+
+```{shell} 
+    setdelvalue <SETNAME> <VALUE>
+```
+
+#### setcontains
+
+```{shell} 
+    setcontains <SETNAME> <VALUE>
+```
+
+#### setunion
+
+```{shell} 
+    setunion <SETNAME1> <SETNAME2> <DESTINATION>
+```
+
+
+#### setintersec
+
+```{shell} 
+    setintersec <SETNAME1> <SETNAME2> <DESTINATION>
+```
+
+
+----------------------------------------------------------
+
+
+## stack
+
+### Activating the module
+
+```{shell}
+	> source $LECABASHHOME/utils.sh
+	> include stack
+```
+
+### Functions provided:
+
+#### newstack
+
+```{shell}
+    newstack <STACKNAME>
+```
+
+Creates a new stack named `STACKNAME`.
+
+#### stacksize
+
+```{shell}
+    stacksize <STACKNAME>
+```
+
+Returns the count of element stored in the stack.
+
+#### pushvalue
+
+```{shell}
+    pushvalue <STACKNAME> <VALUE>
+```
+
+Adds the value VALUE at the end of the stack `STACKNAME`.
+
+
+#### popvalue
+
+```{shell}
+    popvalue <STACKNAME> <VARIABLE>
+```
+
+Pops out the last pushed value of the stack and store it into the `VARIABLE`.
+If the stack is already empty the function returns nothing
+and the return status is set to *1* instead of *0* usually.
+
+
+
+----------------------------------------------------------
+
+
+## tempdir
+
+### Activating the module
+
+```{shell}
+	> source $LECABASHHOME/utils.sh
+	> include tempdir
+```
+
+### Functions provided:
+
+#### availabledisk
+
+```{shell}
+    availabledisk <DIRPATH>
+```
+
+#### registertmpdir
+
+```{shell}
+    registertmpdir <DIRPATH>
+```
+
+#### tempdirectory
+
+```{shell}
+    tempdirectory <VARIABLE> [MINSIZE]
+```
+
+
+----------------------------------------------------------
+
+
+## timeout
+
+### Activating the module
+
+```{shell}
+	> source $LECABASHHOME/utils.sh
+	> include timeout
+```
+
+### Functions provided:
+
+#### timeoutcmd
+
+```{shell}
+    timeoutcmd <SECONDE> <UNIX COMMAND>
+```
+
+
+

lecaluke.sh

@@ -17,27 +17,51 @@
 include logging
 logdebug "Load lecaluke package"
 
+include tempdir
 
-# Where is located the LECA_ENVIRONEMENT
-LECA_ENVIRONEMENT=/nfs_scratch/LECA_ENVIRONEMENT
+if onluke; then
 
-# Lock files are stored in a common place
-LOCK_DIR="${LECA_ENVIRONEMENT}/locks"
+	logdebug Script run on the LUKE cluster
 
-# All the data shared among users can be stored here
-DATA_DIR="${LECA_ENVIRONEMENT}/data"
+	# Register scratch_dir as place where to create temp directory
+	registertmpdir $LOCAL_SCRATCH_DIR
+	registertmpdir $SHARED_SCRATCH_DIR
 
-# All the molecular public databases downloaded for users on the luke cluster
-# will be downloaded here
-BIODATA_DIR="${DATA_DIR}/biodatabase"
+	# Where is located the LECA_ENVIRONEMENT
+	LECA_ENVIRONEMENT=/nfs_scratch/LECA_ENVIRONEMENT
 
-# Main directory for the NCBI taxonomy
-TAXONOMY_DIR="${BIODATA_DIR}/taxonomy"
+	# Load the LECA envoronement
+	. ${LECA_ENVIRONEMENT}/etc/bashrc
 
-TREMBL_DIR="${BIODATA_DIR}/trembl"
+	# Lock files are stored in a common place
+	LOCK_DIR="${LECA_ENVIRONEMENT}/locks"
 
-SWISSPROT_DIR="${BIODATA_DIR}/swissprot"
+	# All the data shared among users can be stored here
+	DATA_DIR="${LECA_ENVIRONEMENT}/data"
 
-function latest_taxonomy() {
-  echo "${TAXONOMY_DIR}/$(ls -ltr "${BIODATA_DIR}/taxonomy" | awk '/^d/ {print $NF}' | tail -1)"
-}
+	# All the molecular public databases downloaded for users on the luke cluster
+	# will be downloaded here
+	BIODATA_DIR="${DATA_DIR}/biodatabase"
+
+	# Main directory for the NCBI taxonomy
+	TAXONOMY_DIR="${BIODATA_DIR}/taxonomy"
+
+	TREMBL_DIR="${BIODATA_DIR}/trembl"
+
+	SWISSPROT_DIR="${BIODATA_DIR}/swissprot"
+
+	function latest_taxonomy() {
+	  echo "${TAXONOMY_DIR}/$(ls -ltr "${BIODATA_DIR}/taxonomy" | awk '/^d/ {print $NF}' | tail -1)"
+	}
+
+else
+	logwarning Script is not running on the LUKE cluster
+	if onfroggy ; then
+		logdebug Script run on the FROGGY cluster
+		# Register scratch_dir as place where to create temp directory
+		registertmpdir $LOCAL_SCRATCH_DIR
+		registertmpdir $SHARED_SCRATCH_DIR
+	else
+		logwarning Script is not running on the FROGGY cluster
+	fi
+fi

mutex.sh

@@ -12,9 +12,7 @@
 include logging
 logdebug "Load mutex package"
 
-if onluke; then
-  include lecaluke
-else
+if [[ -z "$LOCK_DIR" ]] ; then
   LOCK_DIR="${HOME}/lecalock"
 fi
 
@@ -29,11 +27,11 @@ include clobber
 LECA_LOCK_DELAY=2
 LECA_LOCK_MAXRETRY=0
 
-function lockfile() {
+function __lockfile__() {
   echo "${LOCK_DIR}/__${1}__.lecalock"
 }
 
-function locktag() {
+function __locktag__() {
   echo "$(hostname).$$"
 }
 
@@ -48,12 +46,12 @@ function getlock() {
   fi
 
   if [[ "${mode}" == "global" ]]; then
-    name=$(lockfile "${name}")
+    name=$(__lockfile__ "${name}")
   else
-    name=$(lockfile "${name}.$(whoami)")
+    name=$(__lockfile__ "${name}.$(whoami)")
   fi
 
-  local TAG=$(locktag)
+  local TAG=$(__locktag__)
   local retry=0
 
   local notgetlock=1
@@ -97,12 +95,12 @@ function releaselock() {
   fi
 
   if [[ "${mode}" == "global" ]]; then
-    name=$(lockfile "${name}")
+    name=$(__lockfile__ "${name}")
   else
-    name=$(lockfile "${name}.$(whoami)")
+    name=$(__lockfile__ "${name}.$(whoami)")
   fi
 
-  local TAG=$(locktag)
+  local TAG=$(__locktag__)
   local LOCK=""
 
   if [[ -f "${name}" ]]; then
@@ -135,7 +133,7 @@ function getsemaphore() {
     mode="global"
   fi
 
-  local TAG=$(locktag)
+  local TAG=$(__locktag__)
   local retry=0
   local notgetlock=1
 
@@ -145,9 +143,9 @@ function getsemaphore() {
 	  while (( notgetlock > 0 )) && (( i < maxcount )) ; do
 	  
 		if [[ "${mode}" == "global" ]]; then
-		  name=$(lockfile "${semname}.@${i}")
+		  name=$(__lockfile__ "${semname}.@${i}")
 		else
-		  name=$(lockfile "${semname}.$(whoami).@${i}")
+		  name=$(__lockfile__ "${semname}.$(whoami).@${i}")
 		fi
 			  
 		pushnoclobber
@@ -187,9 +185,9 @@ function releasesemaphore() {
   fi
 
   if [[ "${mode}" == "global" ]]; then
-	  name=$(lockfile "${name}.@*")
+	  name=$(__lockfile__ "${name}.@*")
   else
-	  name=$(lockfile "${name}.$(whoami).@*")
+	  name=$(__lockfile__ "${name}.$(whoami).@*")
   fi
 
   toremove=$(ls -tr $name  2> /dev/null | head -1)

stacks.sh

@@ -41,7 +41,8 @@ function stacksize() {
 }
 
 function pushvalue() {
-  eval ${1}[$(stacksize ${1})]=${2}
+  p="${1}[$(stacksize ${1})]=\"${2}\""
+  eval "$p"
 }
 
 function popvalue() {

utils.sh

@@ -50,13 +50,7 @@ if [[ -z "${LECALIB_UTIL_SH}" ]]; then
 	}
 
 	function onfroggy() {
-		[[ $(hostname | sed -E 's/^(......).*$/\1/') == "froggy" ]]
+		[[ $(hostname | sed -E 's/^(....).*$/\1/') == "frog" ]]
 	}
 
-	include logging
-
-	if onluke; then
-		logdebug Script run on the LUKE cluster
-		source ${LECALIB_DIR}/../etc/bashrc
-	fi
 fi