download.sh 3.63 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
#
# LECA Bash library
#
# The LECA bash library provides a set of function used for helping
# development of bash script mainly to write job script on the luke
# cluster
#
# The files from the LECA Bash library must be sourced from your main script
#
#
# Provides functions to facilitate downloading of files
# =====================================================
#
# Provided function:
#
16 17 18 19
#    - download <URL> [USERNAME] [PASSWORD]
#      Downloads the file specified by URL.
#      If specifed the second (USERNAME) and third (PASSWORD) parametters allow
#      to indicate a login and password for a protected connection
20
#
21 22 23 24 25 26 27 28
#    - downloadandcheck <URL> <MD5URL> <DESTINATION> [USERNAME] [PASSWORD]
#      Downloads the file specified by URL and check file integrity.
#      The MD5 checksum file URL is indicated using the MD5URL parametters.
#      You must specified a destination filename to store the downloaded data
#      using the DESTINATION parametter. The MD5 file is downloaded under the
#      name DESTINATION.md5
#      If specifed the fourth (USERNAME) and fifth (PASSWORD) parametters allow
#      to indicate a login and password for a protected connection
29
#
30
#      On error the function has a return status set to 1, 0 otherwise.
31
#
32 33 34 35 36
#    - setmaxretry <MAXRETRY>
#      If downloading failed the downloadandcheck function can retry to download
#      the file several times before a complete failure. This function allows to
#      define the maximum of time the downloadandcheck tries to redownload before
#      exiting in error
37 38 39
#
#

40
include logging
Eric Coissac committed
41 42
include ifs

43 44
logdebug "Load download package"

Eric Coissac committed
45 46 47 48
if ! which curl; then
	if module 2> /dev/null; then
	  module load curl
	  module load openssl
49 50
	else
	  logerror "No curl command available"
Eric Coissac committed
51
	fi
52 53
fi

54 55 56 57 58 59 60 61 62 63
if ! which md5; then
	if ! which md5sum; then
		logerror "No md5 commands available"
	else
		LECABASHLIB_MD5SUM=md5sum
	fi
else
	LECABASHLIB_MD5SUM=md5
fi

64 65 66 67
CURL="curl -L"
LECA_MAX_RETRY=3

function download() {
Eric Coissac committed
68 69
  pushifs
  
70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92
  logdebug "Function called with $# arguments"
  logdebug "$@"
  local URL="${1}"
  shift

  local LINK="${CURL} ${URL}"

  if [[ ! -z $1 && -z $2 ]]; then
    logdebug "Download with username ${1}"
    local LINK="${CURL} --user ${1} ${URL}"
    shift
  fi

  if [[ ! -z $1 && ! -z $2 ]]; then
    logdebug "Download with username ${1} and password ${2}"
    local LINK="${CURL} --user ${1}:${2} ${URL}"
    shift
    shift
  fi

  loginfo "Downloading URL : ${URL}"
  ${LINK}
  loginfo "URL : ${URL} -> Done."
Eric Coissac committed
93 94
  
  popifs
95 96 97
}

function setmaxretry() {
98
  LECA_MAX_RETRY="$1"
99 100 101
}

function downloadandcheck() {
102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
  local URL="${1}"
  shift

  local MD5="${1}"
  shift

  local TARGETFILE="${1}"
  shift

  if [[ ! -z "${1}" ]]; then
    local DOWLOADUSER="${1}"
    shift
  fi

  if [[ ! -z "${1}" ]]; then
    local PASSWORD="${1}"
    shift
  fi

  local RETRY=0
  local MD5_BAD=1

  while (( MD5_BAD > 0 )) && (( RETRY < LECA_MAX_RETRY )); do
    download "${URL}" "${DOWLOADUSER}" "${PASSWORD}" > "${TARGETFILE}"
    download "${MD5}" "${DOWLOADUSER}" "${PASSWORD}" > "${TARGETFILE}.md5"

128
    MD5LOCAL=$(${LECABASHLIB_MD5SUM} "${TARGETFILE}" | awk '{print $1}')
129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152
    MD5ORIGI=$(awk '{print $1}' "${TARGETFILE}.md5")

    if [[ "$MD5LOCAL" == "$MD5ORIGI" ]]; then
      MD5_BAD=0
    else
      MD5_BAD=1
    fi

    RETRY=$((RETRY+1))

    if (( MD5_BAD > 0 )); then
      logwarning "The two MD5 are different : $MD5LOCAL and $MD5ORIGI"
    fi
  done

    if (( MD5_BAD > 0 )); then
      logerror "The two MD5 are different after $RETRY retries: "
      logerror "  $MD5LOCAL and $MD5ORIGI"
      return 1
    else
      loginfo "${TARGETFILE} downloaded"
      loginfo "   and checked with MD5 $MD5ORIGI"
      return 0
    fi
153
}