From 407fb2f77fce972092284a49abaf69ecdb4fb354 Mon Sep 17 00:00:00 2001 From: Lukas Macura Date: Thu, 1 Dec 2016 16:57:26 +0100 Subject: [PATCH] Repaired and reworked caching --- lib/cache.lib.sh | 97 ++++++++++++++++++++++++++++------------------- lib/ctrl.lib.sh | 70 ++++++++++++++++++++-------------- lib/plugin.lib.sh | 11 ++++-- lib/zaf.lib.sh | 3 ++ lib/zafcache | 14 ++----- zaf | 12 ++++++ 6 files changed, 125 insertions(+), 82 deletions(-) diff --git a/lib/cache.lib.sh b/lib/cache.lib.sh index 47bb93e..733a711 100644 --- a/lib/cache.lib.sh +++ b/lib/cache.lib.sh @@ -1,17 +1,23 @@ # Zaf cache related functions zaf_cache_init(){ - if [ -z "$ZAF_CACHE_DIR" ] || [ "$ZAF_CACHE_DIR" = "/tmp/zafc" ]; then + local files + local file + + if [ -z "$ZAF_CACHE_DIR" ] || [ "$ZAF_CACHE_DIR" = "/tmp/zafc/" ]; then ZAF_CACHE_DIR=${ZAF_TMP_DIR}/zafc mkdir -p $ZAF_CACHE_DIR chown $ZAF_FILES_UID $ZAF_CACHE_DIR >/dev/null 2>/dev/null fi if [ -w $ZAF_CACHE_DIR ]; then zaf_trc "Cache: Removing stale entries" + files=$(find $ZAF_CACHE_DIR/ -type f -name '*.lock' -mmin +1) + [ -n "$files" ] && rm -f $files (cd $ZAF_CACHE_DIR && find ./ -type f -name '*.info' -mmin +1 2>/dev/null | \ while read line ; do - rm -f $line $(basename $line .info) - done + file=$(basename $line .info) + [ "$line" -nt "$file" ] && { rm ${file}*; zaf_trc "rm ${file}*"; } + done ) else zaf_dbg "Cache dir $ZAF_CACHE_DIR is not accessible! Disabling cache." @@ -19,13 +25,12 @@ zaf_cache_init(){ } zaf_cache_clean(){ + local files + if [ -n "$ZAF_CACHE_DIR" ]; then zaf_wrn "Removing cache entries" - (cd $ZAF_CACHE_DIR && find ./ -type f -name '*.info' 2>/dev/null | \ - while read line ; do - rm -f $line $(basename $line .info) - done - ) + files=$(find $ZAF_CACHE_DIR/ -type f) + [ -n "$files" ] && rm -f $files else zaf_dbg "Cache dir not set. Disabling cache." fi @@ -37,6 +42,26 @@ zaf_cache_key(){ echo "$1" | (md5sum - ||md5) 2>/dev/null | cut -d ' ' -f 1 } +# Wait for lock +# $1 - key +zaf_cache_lock(){ + local lockfile + lockfile="${ZAF_CACHE_DIR}/${key}.lock" + + [ -f "$lockfile" ] && sleep 1 + [ -f "$lockfile" ] && return 1 + return 0 +} + +# Unlock cache key +# $1 - key +zaf_cache_unlock(){ + local lockfile + lockfile="${ZAF_CACHE_DIR}/${key}.lock" + + rm -f $lockfile +} + # Put object into cache # $1 key # $2 value @@ -46,24 +71,22 @@ zaf_tocache(){ local key local value local expiry - local lockfile local infofile local datafile key=$(zaf_cache_key "$1") - datafile=$ZAF_CACHE_DIR/$key - lockfile=$ZAF_CACHE_DIR/${key}.lock - infofile=$ZAF_CACHE_DIR/${key}.info + datafile=${ZAF_CACHE_DIR}/$key + infofile=${ZAF_CACHE_DIR}/${key}.info + + zaf_cache_lock "$key" || { zaf_wrn "Cache: Entry $1[$key] locked!"; return 1; } - [ -f $lockfile ] && return 1 # Key is locked, return - touch $lockfile rm -f $datafile $infofile echo "$2" >$datafile echo "$1" >$infofile expiry=$(zaf_date_add "$3") zaf_trc "Cache: Saving entry $1[$key,expiry=$expiry]" touch -m -d "$expiry" $infofile - rm -f $lockfile + zaf_cache_unlock $key } # Put object into cache from stdin and copy to stdout @@ -73,30 +96,24 @@ zaf_tocache_stdin(){ ! [ -w $ZAF_CACHE_DIR ] && { cat; return 1; } local key local expiry - local lockfile local infofile local datafile key=$(zaf_cache_key "$1") - datafile=$ZAF_CACHE_DIR/$key - lockfile=$ZAF_CACHE_DIR/${key}.lock - infofile=$ZAF_CACHE_DIR/${key}.info + datafile=${ZAF_CACHE_DIR}/$key + lockfile=${ZAF_CACHE_DIR}/${key}.lock + infofile=${ZAF_CACHE_DIR}/${key}.info - [ -f $lockfile ] && return 1 # Key is locked, return - touch $lockfile + zaf_cache_lock "$key" || { zaf_wrn "Cache: Entry $1[$key] locked!"; return 1; } rm -f $datafile $infofile cat >$datafile - if [ -s $datafile ]; then - expiry="$(zaf_date_add $2)" - echo "$1 [key=$key,expiry=$expiry]" >$infofile - zaf_trc "Cache: Saving entry $1[key=$key,expiry=$expiry]" - touch -m -d "$expiry" $infofile - cat $datafile - rm -f $lockfile - else - rm -f "$datafile" - fi + expiry="$(zaf_date_add $2)" + echo "$1 [key=$key,expiry=$expiry]" >$infofile + zaf_trc "Cache: Saving entry $1[key=$key,expiry=$expiry]" + touch -m -d "$expiry" $infofile + zaf_cache_unlock "$key" + cat $datafile } # Remove entry from cache @@ -106,15 +123,15 @@ zaf_cache_delentry(){ local key key=$(zaf_cache_key "$1") zaf_trc "Cache: removing $1($key) from cache" - rm -f "$ZAF_CACHE_DIR/$key*" + rm -f "${ZAF_CACHE_DIR}/$key*" } # List entries in cache zaf_cache_list(){ local i - ls $ZAF_CACHE_DIR/*info >/dev/null 2>/dev/null || return 1 + ls ${ZAF_CACHE_DIR}/*info >/dev/null 2>/dev/null || return 1 local key - for i in $ZAF_CACHE_DIR/*info; do + for i in ${ZAF_CACHE_DIR}/*info; do cat $i done } @@ -129,17 +146,17 @@ zaf_fromcache(){ local datafile key=$(zaf_cache_key "$1") - datafile=$ZAF_CACHE_DIR/$key - infofile=$ZAF_CACHE_DIR/${key}.info + datafile="${ZAF_CACHE_DIR}/${key}" + infofile="${ZAF_CACHE_DIR}/${key}.info" if [ -f $datafile ]; then - ! [ -f "$infofile" ] && { return 3; } + #zaf_cache_lock $key || return 3 + #zaf_cache_unlock $key if [ "$infofile" -nt "$datafile" ]; then zaf_trc "Cache: serving $1($key) from cache" - cat $datafile + cat "$datafile" || { ls -la "$datafile" >&2; zaf_err "auuu: $1"; } else - zaf_trc "Cache: removing old entry $1" - rm -f "$ZAF_CACHE_DIR/$key*" + #zaf_cache_delentry $key return 2 fi else diff --git a/lib/ctrl.lib.sh b/lib/ctrl.lib.sh index 4f9b68d..c769d76 100644 --- a/lib/ctrl.lib.sh +++ b/lib/ctrl.lib.sh @@ -38,7 +38,6 @@ zaf_ctrl_get_item_block() { }' } - # Get global plugin block body from stdin # $1 itemname zaf_ctrl_get_global_block() { @@ -77,15 +76,19 @@ zaf_block_get_option() { zaf_ctrl_get_global_option() { local ctrlvar local ctrlopt + local key - ctrlopt="ZAF_CTRL_$(zaf_stripctrl $2)" - eval ctrlvar=\$$ctrlopt - if [ -n "$ctrlvar" ]; then - zaf_dbg "Overriding control field $2 from env $ctrlopt($ctrlvar)" - echo $ctrlopt - else - zaf_ctrl_get_global_block <$1 | zaf_block_get_moption "$2" \ - || zaf_ctrl_get_global_block <$1 | zaf_block_get_option "$2" + key="$1-$2" + if ! zaf_fromcache $key; then + ctrlopt="ZAF_CTRL_$(zaf_stripctrl $2)" + eval ctrlvar=\$$ctrlopt + if [ -n "$ctrlvar" ]; then + zaf_dbg "Overriding control field $2 from env $ctrlopt($ctrlvar)" + echo $ctrlopt + else + zaf_ctrl_get_global_block <$1 | zaf_block_get_moption "$2" \ + || zaf_ctrl_get_global_block <$1 | zaf_block_get_option "$2" + fi | zaf_tocache_stdin "$key" "600" fi } @@ -96,16 +99,20 @@ zaf_ctrl_get_global_option() { zaf_ctrl_get_item_option() { local ctrlvar local ctrlopt + local key - ctrlopt="ZAF_CTRLI_$(zaf_stripctrl $2)_$(zaf_stripctrl $3)" - eval ctrlvar=\$$ctrlopt - if [ -n "$ctrlvar" ]; then - zaf_dbg "Overriding item control field $2/$3 from env $ctrlopt($ctrlvar)" - echo $ctrlopt - else - zaf_ctrl_get_item_block <$1 "$2" | zaf_block_get_moption "$3" \ - || zaf_ctrl_get_item_block <$1 "$2" | zaf_block_get_option "$3" - fi + key="i$1-$2-$3" + if ! zaf_fromcache "$key"; then + ctrlopt="ZAF_CTRLI_$(zaf_stripctrl $2)_$(zaf_stripctrl $3)" + eval ctrlvar=\$$ctrlopt + if [ -n "$ctrlvar" ]; then + zaf_dbg "Overriding item control field $2/$3 from env $ctrlopt($ctrlvar)" + echo $ctrlopt + else + zaf_ctrl_get_item_block <$1 "$2" | zaf_block_get_moption "$3" \ + || zaf_ctrl_get_item_block <$1 "$2" | zaf_block_get_option "$3" + fi | zaf_tocache_stdin "$key" "600" + fi } # Get external item specific option (single or multiline) @@ -115,15 +122,19 @@ zaf_ctrl_get_item_option() { zaf_ctrl_get_extitem_option() { local ctrlvar local ctrlopt + local key - ctrlopt="ZAF_CTRLI_$(zaf_stripctrl $2)_$(zaf_stripctrl $3)" - eval ctrlvar=\$$ctrlopt - if [ -n "$ctrlvar" ]; then - zaf_dbg "Overriding item control field $2/$3 from env $ctrlopt($ctrlvar)" - echo $ctrlopt - else - zaf_ctrl_get_extitem_block <$1 "$2" | zaf_block_get_moption "$3" \ - || zaf_ctrl_get_extitem_block <$1 "$2" | zaf_block_get_option "$3" + key="e$1-$2-$3" + if ! zaf_fromcache "$key"; then + ctrlopt="ZAF_CTRLI_$(zaf_stripctrl $2)_$(zaf_stripctrl $3)" + eval ctrlvar=\$$ctrlopt + if [ -n "$ctrlvar" ]; then + zaf_dbg "Overriding item control field $2/$3 from env $ctrlopt($ctrlvar)" + echo $ctrlopt + else + zaf_ctrl_get_extitem_block <$1 "$2" | zaf_block_get_moption "$3" \ + || zaf_ctrl_get_extitem_block <$1 "$2" | zaf_block_get_option "$3" + fi | zaf_tocache_stdin "$key" "600" fi } @@ -158,9 +169,11 @@ zaf_ctrl_sudo() { pdir="$3" plugin=$1 + sudo=$(zaf_ctrl_get_global_option $2 "Sudo" | zaf_far '{PLUGINDIR}' "${plugindir}") + [ -z "$sudo" ] || [ -z "$ZAF_SUDOERSD" ] && return ! [ -d "$ZAF_SUDOERSD" ] && { zaf_wrn "$ZAF_SUDOERSD nonexistent! Skipping sudo install!"; return 1; } zaf_dbg "Installing sudoers entry $ZAF_SUDOERSD/zaf_$plugin" - sudo=$(zaf_ctrl_get_global_option $2 "Sudo" | zaf_far '{PLUGINDIR}' "${plugindir}") + [ -z "$sudo" ] && return # Nothing to install if ! zaf_which sudo >/dev/null; then zaf_wrn "Sudo needed bud not installed?" @@ -186,9 +199,10 @@ zaf_ctrl_cron() { pdir="$3" plugin=$1 + cron=$(zaf_ctrl_get_global_option $2 "Cron") + [ -z "$cron" ] || [ -z "$ZAF_CROND" ] && return ! [ -d "$ZAF_CROND" ] && { zaf_wrn "$ZAF_CROND nonexistent! Skipping cron install!"; return 1; } zaf_dbg "Installing cron entry $ZAF_CROND/zaf_$plugin" - cron=$(zaf_ctrl_get_global_option $2 "Cron") [ -z "$cron" ] && return # Nothing to install zaf_ctrl_get_global_option $2 "Cron" | zaf_far '{PLUGINDIR}' "${plugindir}" >$ZAF_CROND/zaf_$plugin || zaf_err "Error during zaf_ctrl_cron" } diff --git a/lib/plugin.lib.sh b/lib/plugin.lib.sh index c9e48de..8a31fe6 100644 --- a/lib/plugin.lib.sh +++ b/lib/plugin.lib.sh @@ -101,15 +101,19 @@ zaf_install_plugin() { local url local plugin local plugindir + local tmpplugindir local control local version - if zaf_prepare_plugin "$1" "${ZAF_TMP_DIR}/plugin"; then + plugin=$(basename "$1") + tmpplugindir="${ZAF_TMP_DIR}/zaf-installer/$plugin" + mkdir -p $tmpplugindir + if zaf_prepare_plugin "$1" "$tmpplugindir"; then url=$(zaf_get_plugin_url "$1") - control="${ZAF_TMP_DIR}/plugin/control.zaf" + control="$tmpplugindir/control.zaf" plugin=$(zaf_ctrl_get_global_option $control Plugin) version=$(zaf_ctrl_get_global_option $control Version) - plugindir="${ZAF_PLUGINS_DIR}"/$plugin + plugindir="${ZAF_PLUGINS_DIR}/$plugin" if [ -n "$plugin" ] && zaf_prepare_plugin "$1" $plugindir; then zaf_wrn "Installing plugin $plugin version $version" zaf_dbg "Source url: $url, Destination dir: $plugindir" @@ -129,6 +133,7 @@ zaf_install_plugin() { else zaf_err "Cannot prepare plugin $1" fi + rm -rf $tmpplugindir } # List installed plugins diff --git a/lib/zaf.lib.sh b/lib/zaf.lib.sh index 58d4749..18dcb43 100644 --- a/lib/zaf.lib.sh +++ b/lib/zaf.lib.sh @@ -12,12 +12,15 @@ zaf_msg() { } zaf_trc() { [ "$ZAF_DEBUG" -ge "3" ] && logger -p user.info ${ZAF_LOG_STDERR} -t zaf-trace -- $@ + return 0 } zaf_dbg() { [ "$ZAF_DEBUG" -ge "2" ] && logger -p user.debug ${ZAF_LOG_STDERR} -t zaf-debug -- $@ + return 0 } zaf_wrn() { [ "$ZAF_DEBUG" -ge "1" ] && logger -p user.warn ${ZAF_LOG_STDERR} -t zaf-warning -- $@ + return 0 } zaf_err() { logger ${ZAF_LOG_STDERR} -p user.err -t zaf-error -- $@ diff --git a/lib/zafcache b/lib/zafcache index 7cca7fb..9a2b8c5 100644 --- a/lib/zafcache +++ b/lib/zafcache @@ -19,16 +19,8 @@ else fi if ! zaf_fromcache "$key"; then - zaf_fromcache "$key" >/dev/null 2>/dev/null - [ $? -eq 3 ] && { zaf_err "Operation $key already in progress."; } - if zaf_which at >/dev/null 2>/dev/null; then - at -M now >/dev/null 2>/dev/null <"\$tmpf" | zaf_tocache_stdin "$key" "$seconds" ; [ -s \$tmpf ] && zaf_wrn <\$tmpf -EOF - else - tmpf=$(zaf_tmpfile cache) - $@ 2>"$tmpf" | zaf_tocache_stdin "$key" "$seconds" - [ -s $tmpf ] && zaf_wrn <$tmpf - fi + tmpf=$(zaf_tmpfile cache) + $@ 2>"$tmpf" | zaf_tocache_stdin "$key" "$seconds" + [ -s $tmpf ] && zaf_wrn <$tmpf fi diff --git a/zaf b/zaf index f4a3f59..86a9f2a 100755 --- a/zaf +++ b/zaf @@ -59,6 +59,18 @@ cache-clean) cache-list) zaf_cache_list ;; +cache-test1) + for i in test1 test2 test3; do + echo "To cache:" + echo "$i" | zaf_tocache_stdin "test" 10 + done + ;; +cache-test2) + for i in test1 test2 test3; do + echo "From cache:" + zaf_fromcache "test" + done + ;; version) echo "$ZAF_VERSION"