zabbix/limosek-zaf
1
0
Fork 0
mirror of https://github.com/limosek/zaf.git synced 2024-11-21 09:59:07 +01:00
Code Issues Releases Wiki Activity

Repaired and reworked caching

Browse Source
This commit is contained in:
Lukas Macura 2016-12-01 16:57:26 +01:00
parent bc1cbaf144
commit 407fb2f77f
6 changed files with 125 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
lib/cache.lib.sh
View File

@ -1,17 +1,23 @@
# Zaf cache related functions
zaf_cache_init(){
if [ -z "$ZAF_CACHE_DIR" ] || [ "$ZAF_CACHE_DIR" = "/tmp/zafc" ]; then
local files
local file
if [ -z "$ZAF_CACHE_DIR" ] || [ "$ZAF_CACHE_DIR" = "/tmp/zafc/" ]; then
ZAF_CACHE_DIR=${ZAF_TMP_DIR}/zafc
mkdir -p $ZAF_CACHE_DIR
chown $ZAF_FILES_UID $ZAF_CACHE_DIR >/dev/null 2>/dev/null
fi
if [ -w $ZAF_CACHE_DIR ]; then
zaf_trc "Cache: Removing stale entries"
files=$(find $ZAF_CACHE_DIR/ -type f -name '*.lock' -mmin +1)
[ -n "$files" ] && rm -f $files
(cd $ZAF_CACHE_DIR && find ./ -type f -name '*.info' -mmin +1 2>/dev/null | \
while read line ; do
rm -f $line $(basename $line .info)
done
file=$(basename $line .info)
[ "$line" -nt "$file" ] && { rm ${file}*; zaf_trc "rm ${file}*"; }
done
)
else
zaf_dbg "Cache dir $ZAF_CACHE_DIR is not accessible! Disabling cache."
@ -19,13 +25,12 @@ zaf_cache_init(){
}
zaf_cache_clean(){
local files
if [ -n "$ZAF_CACHE_DIR" ]; then
zaf_wrn "Removing cache entries"
(cd $ZAF_CACHE_DIR && find ./ -type f -name '*.info' 2>/dev/null | \
while read line ; do
rm -f $line $(basename $line .info)
done
)
files=$(find $ZAF_CACHE_DIR/ -type f)
[ -n "$files" ] && rm -f $files
else
zaf_dbg "Cache dir not set. Disabling cache."
fi
@ -37,6 +42,26 @@ zaf_cache_key(){
echo "$1" | (md5sum - ||md5) 2>/dev/null | cut -d ' ' -f 1
}
# Wait for lock
# $1 - key
zaf_cache_lock(){
local lockfile
lockfile="${ZAF_CACHE_DIR}/${key}.lock"
[ -f "$lockfile" ] && sleep 1
[ -f "$lockfile" ] && return 1
return 0
}
# Unlock cache key
# $1 - key
zaf_cache_unlock(){
local lockfile
lockfile="${ZAF_CACHE_DIR}/${key}.lock"
rm -f $lockfile
}
# Put object into cache
# $1 key
# $2 value
@ -46,24 +71,22 @@ zaf_tocache(){
local key
local value
local expiry
local lockfile
local infofile
local datafile
key=$(zaf_cache_key "$1")
datafile=$ZAF_CACHE_DIR/$key
lockfile=$ZAF_CACHE_DIR/${key}.lock
infofile=$ZAF_CACHE_DIR/${key}.info
datafile=${ZAF_CACHE_DIR}/$key
infofile=${ZAF_CACHE_DIR}/${key}.info
zaf_cache_lock "$key" || { zaf_wrn "Cache: Entry $1[$key] locked!"; return 1; }
[ -f $lockfile ] && return 1 # Key is locked, return
touch $lockfile
rm -f $datafile $infofile
echo "$2" >$datafile
echo "$1" >$infofile
expiry=$(zaf_date_add "$3")
zaf_trc "Cache: Saving entry $1[$key,expiry=$expiry]"
touch -m -d "$expiry" $infofile
rm -f $lockfile
zaf_cache_unlock $key
}
# Put object into cache from stdin and copy to stdout
@ -73,30 +96,24 @@ zaf_tocache_stdin(){
! [ -w $ZAF_CACHE_DIR ] && { cat; return 1; }
local key
local expiry
local lockfile
local infofile
local datafile
key=$(zaf_cache_key "$1")
datafile=$ZAF_CACHE_DIR/$key
lockfile=$ZAF_CACHE_DIR/${key}.lock
infofile=$ZAF_CACHE_DIR/${key}.info
datafile=${ZAF_CACHE_DIR}/$key
lockfile=${ZAF_CACHE_DIR}/${key}.lock
infofile=${ZAF_CACHE_DIR}/${key}.info
[ -f $lockfile ] && return 1 # Key is locked, return
touch $lockfile
zaf_cache_lock "$key" || { zaf_wrn "Cache: Entry $1[$key] locked!"; return 1; }
rm -f $datafile $infofile
cat >$datafile
if [ -s $datafile ]; then
expiry="$(zaf_date_add $2)"
echo "$1 [key=$key,expiry=$expiry]" >$infofile
zaf_trc "Cache: Saving entry $1[key=$key,expiry=$expiry]"
touch -m -d "$expiry" $infofile
cat $datafile
rm -f $lockfile
else
rm -f "$datafile"
fi
expiry="$(zaf_date_add $2)"
echo "$1 [key=$key,expiry=$expiry]" >$infofile
zaf_trc "Cache: Saving entry $1[key=$key,expiry=$expiry]"
touch -m -d "$expiry" $infofile
zaf_cache_unlock "$key"
cat $datafile
}
# Remove entry from cache
@ -106,15 +123,15 @@ zaf_cache_delentry(){
local key
key=$(zaf_cache_key "$1")
zaf_trc "Cache: removing $1($key) from cache"
rm -f "$ZAF_CACHE_DIR/$key*"
rm -f "${ZAF_CACHE_DIR}/$key*"
}
# List entries in cache
zaf_cache_list(){
local i
ls $ZAF_CACHE_DIR/*info >/dev/null 2>/dev/null || return 1
ls ${ZAF_CACHE_DIR}/*info >/dev/null 2>/dev/null || return 1
local key
for i in $ZAF_CACHE_DIR/*info; do
for i in ${ZAF_CACHE_DIR}/*info; do
cat $i
done
}
@ -129,17 +146,17 @@ zaf_fromcache(){
local datafile
key=$(zaf_cache_key "$1")
datafile=$ZAF_CACHE_DIR/$key
infofile=$ZAF_CACHE_DIR/${key}.info
datafile="${ZAF_CACHE_DIR}/${key}"
infofile="${ZAF_CACHE_DIR}/${key}.info"
if [ -f $datafile ]; then
! [ -f "$infofile" ] && { return 3; }
#zaf_cache_lock $key || return 3
#zaf_cache_unlock $key
if [ "$infofile" -nt "$datafile" ]; then
zaf_trc "Cache: serving $1($key) from cache"
cat $datafile
cat "$datafile" || { ls -la "$datafile" >&2; zaf_err "auuu: $1"; }
else
zaf_trc "Cache: removing old entry $1"
rm -f "$ZAF_CACHE_DIR/$key*"
#zaf_cache_delentry $key
return 2
fi
else

70
lib/ctrl.lib.sh
View File

@ -38,7 +38,6 @@ zaf_ctrl_get_item_block() {
}'
}
# Get global plugin block body from stdin
# $1 itemname
zaf_ctrl_get_global_block() {
@ -77,15 +76,19 @@ zaf_block_get_option() {
zaf_ctrl_get_global_option() {
local ctrlvar
local ctrlopt
local key
ctrlopt="ZAF_CTRL_$(zaf_stripctrl $2)"
eval ctrlvar=\$$ctrlopt
if [ -n "$ctrlvar" ]; then
zaf_dbg "Overriding control field $2 from env $ctrlopt($ctrlvar)"
echo $ctrlopt
else
zaf_ctrl_get_global_block <$1 | zaf_block_get_moption "$2" \
|| zaf_ctrl_get_global_block <$1 | zaf_block_get_option "$2"
key="$1-$2"
if ! zaf_fromcache $key; then
ctrlopt="ZAF_CTRL_$(zaf_stripctrl $2)"
eval ctrlvar=\$$ctrlopt
if [ -n "$ctrlvar" ]; then
zaf_dbg "Overriding control field $2 from env $ctrlopt($ctrlvar)"
echo $ctrlopt
else
zaf_ctrl_get_global_block <$1 | zaf_block_get_moption "$2" \
|| zaf_ctrl_get_global_block <$1 | zaf_block_get_option "$2"
fi | zaf_tocache_stdin "$key" "600"
fi
}
@ -96,16 +99,20 @@ zaf_ctrl_get_global_option() {
zaf_ctrl_get_item_option() {
local ctrlvar
local ctrlopt
local key
ctrlopt="ZAF_CTRLI_$(zaf_stripctrl $2)_$(zaf_stripctrl $3)"
eval ctrlvar=\$$ctrlopt
if [ -n "$ctrlvar" ]; then
zaf_dbg "Overriding item control field $2/$3 from env $ctrlopt($ctrlvar)"
echo $ctrlopt
else
zaf_ctrl_get_item_block <$1 "$2" | zaf_block_get_moption "$3" \
|| zaf_ctrl_get_item_block <$1 "$2" | zaf_block_get_option "$3"
fi
key="i$1-$2-$3"
if ! zaf_fromcache "$key"; then
ctrlopt="ZAF_CTRLI_$(zaf_stripctrl $2)_$(zaf_stripctrl $3)"
eval ctrlvar=\$$ctrlopt
if [ -n "$ctrlvar" ]; then
zaf_dbg "Overriding item control field $2/$3 from env $ctrlopt($ctrlvar)"
echo $ctrlopt
else
zaf_ctrl_get_item_block <$1 "$2" | zaf_block_get_moption "$3" \
|| zaf_ctrl_get_item_block <$1 "$2" | zaf_block_get_option "$3"
fi | zaf_tocache_stdin "$key" "600"
fi
}
# Get external item specific option (single or multiline)
@ -115,15 +122,19 @@ zaf_ctrl_get_item_option() {
zaf_ctrl_get_extitem_option() {
local ctrlvar
local ctrlopt
local key
ctrlopt="ZAF_CTRLI_$(zaf_stripctrl $2)_$(zaf_stripctrl $3)"
eval ctrlvar=\$$ctrlopt
if [ -n "$ctrlvar" ]; then
zaf_dbg "Overriding item control field $2/$3 from env $ctrlopt($ctrlvar)"
echo $ctrlopt
else
zaf_ctrl_get_extitem_block <$1 "$2" | zaf_block_get_moption "$3" \
|| zaf_ctrl_get_extitem_block <$1 "$2" | zaf_block_get_option "$3"
key="e$1-$2-$3"
if ! zaf_fromcache "$key"; then
ctrlopt="ZAF_CTRLI_$(zaf_stripctrl $2)_$(zaf_stripctrl $3)"
eval ctrlvar=\$$ctrlopt
if [ -n "$ctrlvar" ]; then
zaf_dbg "Overriding item control field $2/$3 from env $ctrlopt($ctrlvar)"
echo $ctrlopt
else
zaf_ctrl_get_extitem_block <$1 "$2" | zaf_block_get_moption "$3" \
|| zaf_ctrl_get_extitem_block <$1 "$2" | zaf_block_get_option "$3"
fi | zaf_tocache_stdin "$key" "600"
fi
}
@ -158,9 +169,11 @@ zaf_ctrl_sudo() {
pdir="$3"
plugin=$1
sudo=$(zaf_ctrl_get_global_option $2 "Sudo" | zaf_far '{PLUGINDIR}' "${plugindir}")
[ -z "$sudo" ] || [ -z "$ZAF_SUDOERSD" ] && return
! [ -d "$ZAF_SUDOERSD" ] && { zaf_wrn "$ZAF_SUDOERSD nonexistent! Skipping sudo install!"; return 1; }
zaf_dbg "Installing sudoers entry $ZAF_SUDOERSD/zaf_$plugin"
sudo=$(zaf_ctrl_get_global_option $2 "Sudo" | zaf_far '{PLUGINDIR}' "${plugindir}")
[ -z "$sudo" ] && return # Nothing to install
if ! zaf_which sudo >/dev/null; then
zaf_wrn "Sudo needed bud not installed?"
@ -186,9 +199,10 @@ zaf_ctrl_cron() {
pdir="$3"
plugin=$1
cron=$(zaf_ctrl_get_global_option $2 "Cron")
[ -z "$cron" ] || [ -z "$ZAF_CROND" ] && return
! [ -d "$ZAF_CROND" ] && { zaf_wrn "$ZAF_CROND nonexistent! Skipping cron install!"; return 1; }
zaf_dbg "Installing cron entry $ZAF_CROND/zaf_$plugin"
cron=$(zaf_ctrl_get_global_option $2 "Cron")
[ -z "$cron" ] && return # Nothing to install
zaf_ctrl_get_global_option $2 "Cron" | zaf_far '{PLUGINDIR}' "${plugindir}" >$ZAF_CROND/zaf_$plugin || zaf_err "Error during zaf_ctrl_cron"
}

11
lib/plugin.lib.sh
View File

@ -101,15 +101,19 @@ zaf_install_plugin() {
local url
local plugin
local plugindir
local tmpplugindir
local control
local version
if zaf_prepare_plugin "$1" "${ZAF_TMP_DIR}/plugin"; then
plugin=$(basename "$1")
tmpplugindir="${ZAF_TMP_DIR}/zaf-installer/$plugin"
mkdir -p $tmpplugindir
if zaf_prepare_plugin "$1" "$tmpplugindir"; then
url=$(zaf_get_plugin_url "$1")
control="${ZAF_TMP_DIR}/plugin/control.zaf"
control="$tmpplugindir/control.zaf"
plugin=$(zaf_ctrl_get_global_option $control Plugin)
version=$(zaf_ctrl_get_global_option $control Version)
plugindir="${ZAF_PLUGINS_DIR}"/$plugin
plugindir="${ZAF_PLUGINS_DIR}/$plugin"
if [ -n "$plugin" ] && zaf_prepare_plugin "$1" $plugindir; then
zaf_wrn "Installing plugin $plugin version $version"
zaf_dbg "Source url: $url, Destination dir: $plugindir"
@ -129,6 +133,7 @@ zaf_install_plugin() {
else
zaf_err "Cannot prepare plugin $1"
fi
rm -rf $tmpplugindir
}
# List installed plugins

3
lib/zaf.lib.sh
View File

@ -12,12 +12,15 @@ zaf_msg() {
}
zaf_trc() {
[ "$ZAF_DEBUG" -ge "3" ] && logger -p user.info ${ZAF_LOG_STDERR} -t zaf-trace -- $@
return 0
}
zaf_dbg() {
[ "$ZAF_DEBUG" -ge "2" ] && logger -p user.debug ${ZAF_LOG_STDERR} -t zaf-debug -- $@
return 0
}
zaf_wrn() {
[ "$ZAF_DEBUG" -ge "1" ] && logger -p user.warn ${ZAF_LOG_STDERR} -t zaf-warning -- $@
return 0
}
zaf_err() {
logger ${ZAF_LOG_STDERR} -p user.err -t zaf-error -- $@

14
lib/zafcache
View File

@ -19,16 +19,8 @@ else
fi
if ! zaf_fromcache "$key"; then
zaf_fromcache "$key" >/dev/null 2>/dev/null
[ $? -eq 3 ] && { zaf_err "Operation $key already in progress."; }
if zaf_which at >/dev/null 2>/dev/null; then
at -M now >/dev/null 2>/dev/null <<EOF
. $(dirname $0)/preload.sh; tmpf=\$(zaf_tmpfile cache); $@ 2>"\$tmpf" | zaf_tocache_stdin "$key" "$seconds" ; [ -s \$tmpf ] && zaf_wrn <\$tmpf
EOF
else
tmpf=$(zaf_tmpfile cache)
$@ 2>"$tmpf" | zaf_tocache_stdin "$key" "$seconds"
[ -s $tmpf ] && zaf_wrn <$tmpf
fi
tmpf=$(zaf_tmpfile cache)
$@ 2>"$tmpf" | zaf_tocache_stdin "$key" "$seconds"
[ -s $tmpf ] && zaf_wrn <$tmpf
fi

12
zaf
View File

@ -59,6 +59,18 @@ cache-clean)
cache-list)
zaf_cache_list
;;
cache-test1)
for i in test1 test2 test3; do
echo "To cache:"
echo "$i" | zaf_tocache_stdin "test" 10
done
;;
cache-test2)
for i in test1 test2 test3; do
echo "From cache:"
zaf_fromcache "test"
done
;;
version)
echo "$ZAF_VERSION<git $ZAF_GITBRANCH>"