Repaired and reworked caching

2025-10-31 17:47:37 +01:00 · 2016-12-01 16:57:26 +01:00
parent bc1cbaf144
commit 407fb2f77f
6 changed files with 125 additions and 82 deletions
--- a/lib/cache.lib.sh
+++ b/lib/cache.lib.sh
@@ -1,17 +1,23 @@
 # Zaf cache related functions

 zaf_cache_init(){
-	if [ -z "$ZAF_CACHE_DIR" ] || [ "$ZAF_CACHE_DIR" = "/tmp/zafc" ]; then
+	local files
+	local file
+
+	if [ -z "$ZAF_CACHE_DIR" ] || [ "$ZAF_CACHE_DIR" = "/tmp/zafc/" ]; then
 		ZAF_CACHE_DIR=${ZAF_TMP_DIR}/zafc
 		mkdir -p $ZAF_CACHE_DIR
 		chown $ZAF_FILES_UID $ZAF_CACHE_DIR >/dev/null 2>/dev/null
 	fi
 	if [ -w $ZAF_CACHE_DIR ]; then
 		zaf_trc "Cache: Removing stale entries"
+		files=$(find $ZAF_CACHE_DIR/ -type f -name '*.lock' -mmin +1)
+		[ -n "$files" ] && rm -f $files
 		(cd $ZAF_CACHE_DIR && find ./ -type f -name '*.info' -mmin +1 2>/dev/null | \
 		while read line ; do
-			rm -f $line $(basename $line .info)
-		done 
+			 file=$(basename $line .info)
+			 [ "$line" -nt "$file" ] && { rm ${file}*; zaf_trc "rm ${file}*"; }
+		done
 		)
 	else
 		zaf_dbg "Cache dir $ZAF_CACHE_DIR is not accessible! Disabling cache."
@@ -19,13 +25,12 @@ zaf_cache_init(){
 }

 zaf_cache_clean(){
+	local files
+
 	if [ -n "$ZAF_CACHE_DIR" ]; then
 		zaf_wrn "Removing cache entries"
-		(cd $ZAF_CACHE_DIR && find ./ -type f -name '*.info' 2>/dev/null | \
-		while read line ; do
-			rm -f $line $(basename $line .info)
-		done 
-		)
+		files=$(find $ZAF_CACHE_DIR/ -type f)
+		[ -n "$files" ] && rm -f $files
 	else
 		zaf_dbg "Cache dir not set. Disabling cache."
 	fi
@@ -37,6 +42,26 @@ zaf_cache_key(){
 	echo "$1" | (md5sum - ||md5) 2>/dev/null | cut -d ' ' -f 1
 }

+# Wait for lock 
+# $1 - key
+zaf_cache_lock(){
+	local lockfile
+	lockfile="${ZAF_CACHE_DIR}/${key}.lock"
+	
+	[ -f "$lockfile" ] && sleep 1
+	[ -f "$lockfile" ] && return 1
+	return 0
+}
+
+# Unlock cache key
+# $1 - key
+zaf_cache_unlock(){
+	local lockfile
+	lockfile="${ZAF_CACHE_DIR}/${key}.lock"
+	
+	rm -f $lockfile
+}
+
 # Put object into cache
 # $1 key
 # $2 value
@@ -46,24 +71,22 @@ zaf_tocache(){
 	local key
 	local value
 	local expiry
-	local lockfile
 	local infofile
 	local datafile

 	key=$(zaf_cache_key "$1")
-	datafile=$ZAF_CACHE_DIR/$key
-	lockfile=$ZAF_CACHE_DIR/${key}.lock
-	infofile=$ZAF_CACHE_DIR/${key}.info
+	datafile=${ZAF_CACHE_DIR}/$key
+	infofile=${ZAF_CACHE_DIR}/${key}.info
+
+	zaf_cache_lock "$key" || { zaf_wrn "Cache: Entry $1[$key] locked!"; return 1; }

-	[ -f $lockfile ] && return 1	# Key is locked, return
-	touch $lockfile
 	rm -f $datafile $infofile
 	echo "$2" >$datafile
 	echo "$1" >$infofile
 	expiry=$(zaf_date_add "$3")
 	zaf_trc "Cache: Saving entry $1[$key,expiry=$expiry]"
 	touch -m -d "$expiry" $infofile
-	rm -f $lockfile
+	zaf_cache_unlock $key
 }

 # Put object into cache from stdin and copy to stdout
@@ -73,30 +96,24 @@ zaf_tocache_stdin(){
 	! [ -w $ZAF_CACHE_DIR ] && { cat; return 1; }
 	local key
 	local expiry
-	local lockfile
 	local infofile
 	local datafile

 	key=$(zaf_cache_key "$1")
-	datafile=$ZAF_CACHE_DIR/$key
-	lockfile=$ZAF_CACHE_DIR/${key}.lock
-	infofile=$ZAF_CACHE_DIR/${key}.info
+	datafile=${ZAF_CACHE_DIR}/$key
+	lockfile=${ZAF_CACHE_DIR}/${key}.lock
+	infofile=${ZAF_CACHE_DIR}/${key}.info

-	[ -f $lockfile ] && return 1	# Key is locked, return
-	touch $lockfile
+	zaf_cache_lock "$key" || { zaf_wrn "Cache: Entry $1[$key] locked!"; return 1; }

 	rm -f $datafile $infofile
 	cat >$datafile
-	if [ -s $datafile ]; then
-		expiry="$(zaf_date_add $2)"
-		echo "$1 [key=$key,expiry=$expiry]" >$infofile
-		zaf_trc "Cache: Saving entry $1[key=$key,expiry=$expiry]"
-		touch -m -d "$expiry" $infofile
-		cat $datafile
-		rm -f $lockfile
-	else
-		rm -f "$datafile"
-	fi
+	expiry="$(zaf_date_add $2)"
+	echo "$1 [key=$key,expiry=$expiry]" >$infofile
+	zaf_trc "Cache: Saving entry $1[key=$key,expiry=$expiry]"
+	touch -m -d "$expiry" $infofile
+	zaf_cache_unlock "$key"
+	cat $datafile
 }

 # Remove entry from cache
@@ -106,15 +123,15 @@ zaf_cache_delentry(){
 	local key
 	key=$(zaf_cache_key "$1")
 	zaf_trc "Cache: removing $1($key) from cache"
-	rm -f "$ZAF_CACHE_DIR/$key*"
+	rm -f "${ZAF_CACHE_DIR}/$key*"
 }

 # List entries in cache
 zaf_cache_list(){
 	local i
-	ls $ZAF_CACHE_DIR/*info >/dev/null 2>/dev/null || return 1
+	ls ${ZAF_CACHE_DIR}/*info >/dev/null 2>/dev/null || return 1
 	local key
-	for i in $ZAF_CACHE_DIR/*info; do
+	for i in ${ZAF_CACHE_DIR}/*info; do
 		cat $i
 	done
 }
@@ -129,17 +146,17 @@ zaf_fromcache(){
 	local datafile

 	key=$(zaf_cache_key "$1")
-	datafile=$ZAF_CACHE_DIR/$key
-	infofile=$ZAF_CACHE_DIR/${key}.info
+	datafile="${ZAF_CACHE_DIR}/${key}"
+	infofile="${ZAF_CACHE_DIR}/${key}.info"

 	if [ -f $datafile ]; then
-		! [ -f "$infofile" ] && { return 3; }
+		#zaf_cache_lock $key || return 3
+		#zaf_cache_unlock $key
 		if [ "$infofile" -nt "$datafile" ]; then
 			zaf_trc "Cache: serving $1($key) from cache"
-			cat $datafile
+			cat "$datafile" || { ls -la "$datafile" >&2; zaf_err "auuu: $1";  }
 		else
-			zaf_trc "Cache: removing old entry $1"
-			rm -f "$ZAF_CACHE_DIR/$key*"
+			#zaf_cache_delentry $key
 			return 2
 		fi
 	else
--- a/lib/ctrl.lib.sh
+++ b/lib/ctrl.lib.sh
@@ -38,7 +38,6 @@ zaf_ctrl_get_item_block() {
 	}'
 }

-
 # Get global plugin block body from stdin
 # $1 itemname
 zaf_ctrl_get_global_block() {
@@ -77,15 +76,19 @@ zaf_block_get_option() {
 zaf_ctrl_get_global_option() {
 	local ctrlvar
 	local ctrlopt
+	local key

-	ctrlopt="ZAF_CTRL_$(zaf_stripctrl $2)"
-	eval ctrlvar=\$$ctrlopt
-	if [ -n "$ctrlvar" ]; then
-		zaf_dbg "Overriding control field $2 from env $ctrlopt($ctrlvar)"
-		echo $ctrlopt
-	else
-		zaf_ctrl_get_global_block <$1 | zaf_block_get_moption "$2" \
-		|| zaf_ctrl_get_global_block <$1 | zaf_block_get_option "$2"
+	key="$1-$2"
+	if ! zaf_fromcache $key; then 
+		ctrlopt="ZAF_CTRL_$(zaf_stripctrl $2)"
+		eval ctrlvar=\$$ctrlopt
+		if [ -n "$ctrlvar" ]; then
+			zaf_dbg "Overriding control field $2 from env $ctrlopt($ctrlvar)"
+			echo $ctrlopt
+		else
+			zaf_ctrl_get_global_block <$1 | zaf_block_get_moption "$2" \
+			|| zaf_ctrl_get_global_block <$1 | zaf_block_get_option "$2"
+		fi | zaf_tocache_stdin "$key" "600" 
 	fi
 }

@@ -96,16 +99,20 @@ zaf_ctrl_get_global_option() {
 zaf_ctrl_get_item_option() {
 	local ctrlvar
 	local ctrlopt
+	local key

-	ctrlopt="ZAF_CTRLI_$(zaf_stripctrl $2)_$(zaf_stripctrl $3)"
-	eval ctrlvar=\$$ctrlopt
-	if [ -n "$ctrlvar" ]; then
-		zaf_dbg "Overriding item control field $2/$3 from env $ctrlopt($ctrlvar)"
-		echo $ctrlopt
-	else
-		zaf_ctrl_get_item_block <$1 "$2" | zaf_block_get_moption "$3" \
-		|| zaf_ctrl_get_item_block <$1 "$2" | zaf_block_get_option "$3"
-	fi
+	key="i$1-$2-$3"
+	if ! zaf_fromcache "$key"; then
+		ctrlopt="ZAF_CTRLI_$(zaf_stripctrl $2)_$(zaf_stripctrl $3)"
+		eval ctrlvar=\$$ctrlopt
+		if [ -n "$ctrlvar" ]; then
+			zaf_dbg "Overriding item control field $2/$3 from env $ctrlopt($ctrlvar)"
+			echo $ctrlopt
+		else
+			zaf_ctrl_get_item_block <$1 "$2" | zaf_block_get_moption "$3" \
+			|| zaf_ctrl_get_item_block <$1 "$2" | zaf_block_get_option "$3"
+		fi | zaf_tocache_stdin "$key" "600" 
+	fi 
 }

 # Get external item specific option (single or multiline)
@@ -115,15 +122,19 @@ zaf_ctrl_get_item_option() {
 zaf_ctrl_get_extitem_option() {
 	local ctrlvar
 	local ctrlopt
+	local key

-	ctrlopt="ZAF_CTRLI_$(zaf_stripctrl $2)_$(zaf_stripctrl $3)"
-	eval ctrlvar=\$$ctrlopt
-	if [ -n "$ctrlvar" ]; then
-		zaf_dbg "Overriding item control field $2/$3 from env $ctrlopt($ctrlvar)"
-		echo $ctrlopt
-	else
-		zaf_ctrl_get_extitem_block <$1 "$2" | zaf_block_get_moption "$3" \
-		|| zaf_ctrl_get_extitem_block <$1 "$2" | zaf_block_get_option "$3"
+	key="e$1-$2-$3"
+	if ! zaf_fromcache "$key"; then
+		ctrlopt="ZAF_CTRLI_$(zaf_stripctrl $2)_$(zaf_stripctrl $3)"
+		eval ctrlvar=\$$ctrlopt
+		if [ -n "$ctrlvar" ]; then
+			zaf_dbg "Overriding item control field $2/$3 from env $ctrlopt($ctrlvar)"
+			echo $ctrlopt
+		else
+			zaf_ctrl_get_extitem_block <$1 "$2" | zaf_block_get_moption "$3" \
+			|| zaf_ctrl_get_extitem_block <$1 "$2" | zaf_block_get_option "$3"
+		fi | zaf_tocache_stdin "$key" "600" 
 	fi
 }

@@ -158,9 +169,11 @@ zaf_ctrl_sudo() {

 	pdir="$3"
 	plugin=$1
+	sudo=$(zaf_ctrl_get_global_option $2 "Sudo" | zaf_far '{PLUGINDIR}' "${plugindir}")
+	[ -z "$sudo" ] || [ -z "$ZAF_SUDOERSD" ] && return
 	! [ -d "$ZAF_SUDOERSD" ] && { zaf_wrn "$ZAF_SUDOERSD nonexistent! Skipping sudo install!"; return 1; }
 	zaf_dbg "Installing sudoers entry $ZAF_SUDOERSD/zaf_$plugin"
-	sudo=$(zaf_ctrl_get_global_option $2 "Sudo" | zaf_far '{PLUGINDIR}' "${plugindir}")
+	
 	[ -z "$sudo" ] && return  # Nothing to install
 	if ! zaf_which sudo >/dev/null; then
 		zaf_wrn "Sudo needed bud not installed?"
@@ -186,9 +199,10 @@ zaf_ctrl_cron() {

 	pdir="$3"
 	plugin=$1
+	cron=$(zaf_ctrl_get_global_option $2 "Cron")
+	[ -z "$cron" ]  || [ -z "$ZAF_CROND" ] && return
 	! [ -d "$ZAF_CROND" ] && { zaf_wrn "$ZAF_CROND nonexistent! Skipping cron install!"; return 1; }
 	zaf_dbg "Installing cron entry $ZAF_CROND/zaf_$plugin"
-	cron=$(zaf_ctrl_get_global_option $2 "Cron")
 	[ -z "$cron" ] && return # Nothing to install
 	zaf_ctrl_get_global_option $2 "Cron" | zaf_far '{PLUGINDIR}' "${plugindir}" >$ZAF_CROND/zaf_$plugin || zaf_err "Error during zaf_ctrl_cron"
 }
--- a/lib/plugin.lib.sh
+++ b/lib/plugin.lib.sh
@@ -101,15 +101,19 @@ zaf_install_plugin() {
 	local url
 	local plugin
 	local plugindir
+	local tmpplugindir
 	local control
 	local version

-	if zaf_prepare_plugin "$1" "${ZAF_TMP_DIR}/plugin"; then
+	plugin=$(basename "$1")
+	tmpplugindir="${ZAF_TMP_DIR}/zaf-installer/$plugin"
+	mkdir -p $tmpplugindir
+	if zaf_prepare_plugin "$1" "$tmpplugindir"; then
 		url=$(zaf_get_plugin_url "$1")
-		control="${ZAF_TMP_DIR}/plugin/control.zaf"
+		control="$tmpplugindir/control.zaf"
                plugin=$(zaf_ctrl_get_global_option $control Plugin)
 		version=$(zaf_ctrl_get_global_option $control Version)
-		plugindir="${ZAF_PLUGINS_DIR}"/$plugin
+		plugindir="${ZAF_PLUGINS_DIR}/$plugin"
 		if [ -n "$plugin" ] && zaf_prepare_plugin "$1" $plugindir; then
 			zaf_wrn "Installing plugin $plugin version $version"
 			zaf_dbg "Source url: $url, Destination dir: $plugindir"
@@ -129,6 +133,7 @@ zaf_install_plugin() {
        else
            	zaf_err "Cannot prepare plugin $1"
 	fi
+	rm -rf $tmpplugindir
 }

 # List installed plugins
--- a/lib/zaf.lib.sh
+++ b/lib/zaf.lib.sh
@@ -12,12 +12,15 @@ zaf_msg() {
 }
 zaf_trc() {
 	[ "$ZAF_DEBUG" -ge "3" ] && logger -p user.info ${ZAF_LOG_STDERR} -t zaf-trace -- $@
+	return 0
 }
 zaf_dbg() {
 	[ "$ZAF_DEBUG" -ge "2" ] && logger -p user.debug ${ZAF_LOG_STDERR} -t zaf-debug -- $@
+	return 0
 }
 zaf_wrn() {
 	[ "$ZAF_DEBUG" -ge "1" ] && logger -p user.warn ${ZAF_LOG_STDERR} -t zaf-warning -- $@
+	return 0
 }
 zaf_err() {
 	logger ${ZAF_LOG_STDERR} -p user.err -t zaf-error -- $@
--- a/lib/zafcache
+++ b/lib/zafcache
@@ -19,16 +19,8 @@ else
 fi

 if ! zaf_fromcache "$key"; then
-	zaf_fromcache "$key" >/dev/null 2>/dev/null
-	[ $? -eq 3 ] && { zaf_err "Operation $key already in progress."; }
-	if zaf_which at >/dev/null 2>/dev/null; then
-		at -M now >/dev/null 2>/dev/null <<EOF
-		. $(dirname $0)/preload.sh; tmpf=\$(zaf_tmpfile cache); $@ 2>"\$tmpf" | zaf_tocache_stdin "$key" "$seconds" ; [ -s \$tmpf ] && zaf_wrn <\$tmpf
-EOF
-	else
-		tmpf=$(zaf_tmpfile cache)
-		$@ 2>"$tmpf" | zaf_tocache_stdin "$key" "$seconds" 
-		[ -s $tmpf ] && zaf_wrn <$tmpf
-	fi
+	tmpf=$(zaf_tmpfile cache)
+	$@ 2>"$tmpf" | zaf_tocache_stdin "$key" "$seconds" 
+	[ -s $tmpf ] && zaf_wrn <$tmpf
 fi

--- a/12
+++ b/12
@@ -59,6 +59,18 @@ cache-clean)
 cache-list)
 	zaf_cache_list
 	;;
+cache-test1)
+	for i in test1 test2 test3; do
+		echo "To cache:"
+		echo "$i" | zaf_tocache_stdin "test" 10
+	done
+	;;
+cache-test2)
+	for i in test1 test2 test3; do
+		echo "From cache:"
+		zaf_fromcache "test"
+	done
+	;;

 version)
 	echo "$ZAF_VERSION<git $ZAF_GITBRANCH>"