You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
521 lines
17 KiB
521 lines
17 KiB
#!/bin/sh
|
|
|
|
# Auto-Record-Cleaner (aurecl) - automatically remove old recorded events
|
|
|
|
# Copyright (C) 2014 LtCmdrLuke
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; either version 2 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software
|
|
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110, USA
|
|
|
|
|
|
# This is a test-script for automatically handling the oldest records in a
|
|
# specified directory after the directory has reached a certain size. Files
|
|
# which will be deleted next are first moved to a "last-chance" directory
|
|
# before they are finally deleted. In each run, only as much files will be
|
|
# really deleted as needed to limit the maximum allowed size of the directory.
|
|
|
|
# The main intention of this script is its usage in conjunction with the
|
|
# "channel-crawling"-feature of the Auto-Timer on a neutrino-based set top box.
|
|
# Basic idea:
|
|
# Record every event of certain channels and delete always the
|
|
# oldest recordings in the record directory, but give the user a chance
|
|
# to see what is about to be deleted next. By running this script regularly
|
|
# you can keep always the latest x days of all events from a certain channel
|
|
# available for watching anytime, i.e. fully independent of broadcasting
|
|
# times. (It basically resembles 'Sky Anytime')
|
|
|
|
# Install & usage:
|
|
# - Copy the script somewhere on your box and make it executable
|
|
# - Edit the following variables to correspond to your environment, e.g.
|
|
# which directory should be watched and how much space you want to give it
|
|
# - call it regularly, either manually, or in some automated way. When calling
|
|
# it, provide the command line option "--yes" to confirm, you really want
|
|
# files to be deleted. Without this command line option, it will perform a
|
|
# dry run, i.e. just tell you what it would do now.
|
|
|
|
# Related files:
|
|
# General Configuration: /var/tuxbox/config/auto-record-cleaner.conf
|
|
# Controlled directories: /var/tuxbox/config/auto-record-cleaner.rules
|
|
|
|
VERSION=0.2
|
|
|
|
# Changelog:
|
|
#
|
|
# 0.2
|
|
# - Added new command line options:
|
|
# -y|--yes twice to deactivate the 5s safety
|
|
# -q|--quiet deactivate all console output (and also deactivate the 5s safety)
|
|
# -h|--help shows the command line options
|
|
# - Logs now to a logfile (besides the console, if not deactivated with -q)
|
|
# - Now uses own configuration file under /var/tuxbox/config/auto-record-cleaner.conf
|
|
# * Here the default path for the log-file and the rule-file can be changed
|
|
# * Provided a template file (auto-record-cleaner.conf.template); default values should be fine.
|
|
# - Now uses own rules file under /var/tuxbox/config/auto-record-cleaner.rules
|
|
# * An arbitrary number of different directories can now be cleaned
|
|
# * Provided a template (auto-record-cleaner.rules.template) with examples
|
|
# * Basic Syntax is CLEANING_PATH,MAX_SIZE_CP;LAST_CHANCE_PATH,MAX_SIZE_LCP
|
|
#
|
|
# 0.1
|
|
# - Initial release
|
|
# - Restrictions:
|
|
# * only one controlled directory
|
|
# * Configuration directly in the script
|
|
# * Shell-only, no information in neutrino
|
|
# * No log-file, logs to stdout
|
|
|
|
# Beyond this point, no user-configurable settings
|
|
###############################################################################
|
|
|
|
NAME="Auto-Record-Cleaner"
|
|
ME=${0##*/}
|
|
|
|
CONFIG_FILE=/var/tuxbox/config/$ME.conf
|
|
PID_FILE=/var/run/$ME.pid
|
|
|
|
EXIT_NORMAL=0
|
|
EXIT_SIGNAL=1
|
|
EXIT_NO_RULE_FILE=2
|
|
EXIT_ALREADY_RUNNING=3
|
|
EXIT_UNKNOWN_COMMAND_OPTION=4
|
|
|
|
#######################################################################################
|
|
#BEGIN SECTION "Helper functions"
|
|
|
|
signal_handler() {
|
|
#Handle INT, TERM signals and clean up.
|
|
log "Caught signal. Cleaning up."
|
|
cleanup
|
|
set +f
|
|
log "done."
|
|
log "$ME V${VERSION} exiting now."
|
|
exit $EXIT_SIGNAL
|
|
}
|
|
|
|
cleanup() {
|
|
# Remove the pid-file
|
|
rm -rf $PID_FILE 2>/dev/null
|
|
}
|
|
|
|
log() {
|
|
#Log message to log file
|
|
#$*: Log message
|
|
if [ "$LOG_FILE" != "" ]; then
|
|
echo -e $(date +'%F %H:%M:%S') [$$]: "$*" >> $LOG_FILE
|
|
fi
|
|
if [ $quiet == 0 ]; then
|
|
echo -e "$*"
|
|
fi
|
|
}
|
|
|
|
begin_ifs_block() {
|
|
#Backup IFS (input field separator) to restore it after parsing arguments
|
|
IFS_SAVE=$IFS
|
|
set -f
|
|
}
|
|
|
|
end_ifs_block() {
|
|
#Restore (input field separator) IFS after parsing arguments
|
|
IFS=$IFS_SAVE
|
|
set +f
|
|
}
|
|
|
|
#END SECTION "Helper functions"
|
|
#######################################################################################
|
|
|
|
#######################################################################################
|
|
#BEGIN SECTION "Initialization"
|
|
|
|
init_config() {
|
|
#Parse config file (default: /var/tuxbox/config/auto-record-cleaner.conf)
|
|
if [ -e $CONFIG_FILE ]; then
|
|
source $CONFIG_FILE 2>/dev/null
|
|
fi
|
|
|
|
# Initialize the logfile first, so we can write to it...
|
|
if [ ! -d "${LOG_FILE%/*}" ]; then
|
|
case $LOG_FILE in
|
|
[oO][fF][fF])
|
|
LOG_FILE=''
|
|
;;
|
|
*)
|
|
LOG_FILE=/tmp/${ME}_$(date +'%F').log
|
|
;;
|
|
esac
|
|
fi
|
|
echo -e "\n\n========================== $NAME started new log at $(date) ======================================" >> $LOG_FILE
|
|
|
|
# Check other settings and reset them to default if unset or invalid
|
|
if [ ! -e "$RULE_FILE" ]; then
|
|
RULE_FILE=/var/tuxbox/config/$ME.rules
|
|
if [ ! -e "$RULE_FILE" ]; then
|
|
log "ERROR: Rules file '$RULE_FILE' does not exist! Exiting."
|
|
exit $EXIT_NO_RULE_FILE
|
|
fi
|
|
fi
|
|
}
|
|
|
|
#END SECTION "Initialization"
|
|
#######################################################################################
|
|
|
|
#######################################################################################
|
|
#BEGIN SECTION "Command line options handling"
|
|
|
|
parse_options() {
|
|
#Parse auto-record-cleaner command line arguments
|
|
local option
|
|
|
|
while [ $# -gt 0 ]
|
|
do
|
|
option=$1
|
|
shift
|
|
|
|
case "$option" in
|
|
-y|--yes)
|
|
dry_run=$((dry_run-1))
|
|
;;
|
|
-q|--quiet)
|
|
quiet=1
|
|
;;
|
|
-h|--help)
|
|
usage
|
|
exit $EXIT_NORMAL
|
|
;;
|
|
*)
|
|
echo "Unknown command line option '$option'. What did you want to do? Exiting!"
|
|
usage
|
|
exit $EXIT_UNKNOWN_COMMAND_OPTION
|
|
;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
usage() {
|
|
# Print short usage message on console
|
|
echo -e "Usage: $ME [options]"
|
|
echo -e "Valid options are:"
|
|
echo -e "\t-y|--yes\t\tSwitches safety off and really deletes/moves files. Use twice to deactivate 5s safety interval."
|
|
echo -e "\t-q|--quiet\t\tDeactivates all output on console. Also deactivates 5s safety interval."
|
|
echo -e "\t-h|--help\t\tPrint this help and exit."
|
|
}
|
|
|
|
#END SECTION "Command line options handling"
|
|
#######################################################################################
|
|
|
|
#######################################################################################
|
|
#BEGIN SECTION "Work functions"
|
|
|
|
limit_directory() {
|
|
# Moves or deletes oldest files in a given directory
|
|
# Parameters:
|
|
# $1: The directory from which the files should be moved/deleted
|
|
# $2: the maximum size of the directory in kB
|
|
# $3: optional: the directory into which files should be moved. If this
|
|
# parameter is missing, files will be deleted
|
|
# Precondition: All provided directories must exist.
|
|
|
|
local dir max_dir_size dir_size dest_dir over_limit ts_files ts
|
|
|
|
dir="$1"
|
|
max_dir_size="$2"
|
|
dir_size=$(du -s -k "$dir" | cut -f1)
|
|
dest_dir="$3"
|
|
|
|
# in case there is a destination folder given, we need to get its size
|
|
# and subtract this from the current dir size
|
|
if [ -n "$dest_dir" ]; then
|
|
dest_dir_size=$(du -s -k "$dest_dir" | cut -f1)
|
|
dir_size=$((dir_size-dest_dir_size))
|
|
log "\t\t\tLimiting '$dir' to max size $((max_dir_size/(1024*1024)))GB by moving the oldest files to '$dest_dir' ... "
|
|
else
|
|
log "\t\t\tLimiting '$dir' to max size $((max_dir_size/(1024*1024)))GB by deleting the oldest files ..."
|
|
fi
|
|
|
|
if [ $dir_size -gt $max_dir_size ]; then
|
|
over_limit=$((dir_size-max_dir_size))
|
|
if [ -z "$dest_dir" ]; then
|
|
log "\t\t\t\tWe need to delete $((over_limit/(1024*1024)))GB (${over_limit}kB) from '$dir' ..."
|
|
else
|
|
log "\t\t\t\tWe need to move $((over_limit/(1024*1024)))GB (${over_limit}kB) from '$dir' to '$dest_dir' ..."
|
|
fi
|
|
else
|
|
log "\t\t\t\tNothing to do in directory '$dir'. Current size has not reached the limit."
|
|
return 0
|
|
fi
|
|
|
|
# we don't want to use the ls command since its output is not POSIX-standard
|
|
# therefore, we first find all files and then we use the date command to
|
|
# determine the modification time
|
|
|
|
# first collect all *.ts files
|
|
ts_files=$(find "$dir" -name "*.ts")
|
|
|
|
# now gather additional information, like modification time and size
|
|
# (we could have got this info from ls in ONE call..)
|
|
begin_ifs_block
|
|
IFS=$'\n'
|
|
for ts in $ts_files; do
|
|
ts_date=$(date -r "$ts" +%s)
|
|
ts_size=$(du -k "$ts" | cut -f1)
|
|
ts_files_ext="${ts_files_ext}${ts}|$ts_date|$ts_size\n"
|
|
#echo "$ts|$ts_date|$ts_size"
|
|
done
|
|
end_ifs_block
|
|
|
|
# sort the final list with respect to the modification time
|
|
sorted_ts_files_ext=$(echo -ne "$ts_files_ext" | sort -t '|' -k2)
|
|
|
|
count=0
|
|
# now (re)move until limit is reached
|
|
begin_ifs_block
|
|
IFS=$'\n'
|
|
for ts_file in $sorted_ts_files_ext; do
|
|
IFS='|'
|
|
set -- $ts_file
|
|
filename=$1
|
|
filetime=$2
|
|
filesize=$3
|
|
|
|
# skip the file, if it is already in dest_dir
|
|
if [ "${filename%/*}" == "$dest_dir" ]; then
|
|
#echo -e "\tSkipping $filename .."
|
|
continue
|
|
fi
|
|
|
|
xml_file=${filename%".ts"}".xml"
|
|
|
|
if [ $dry_run == 0 ]; then
|
|
if [ -z "$dest_dir" ]; then
|
|
log "\t\t\t\tDeleting now '$filename' (${filesize}kB).."
|
|
rm "$filename" 2>/dev/null
|
|
if [ -f "$xml_file" ];then
|
|
log "\t\t\t\tDeleting now also the corresponding '$xml_file' ... "
|
|
rm "$xml_file" 2>/dev/null
|
|
fi
|
|
else
|
|
log "\t\t\t\tMoving '$filename' (${filesize}kB) to '$dest_dir' ..."
|
|
mv "$filename" "$dest_dir" 2>/dev/null
|
|
if [ -f "$xml_file" ];then
|
|
log "\t\t\t\tMoving now also the corresponding '$xml_file' to '$dest_dir' ... "
|
|
mv "$xml_file" "$dest_dir" 2>/dev/null
|
|
fi
|
|
fi
|
|
else
|
|
if [ -z "$dest_dir" ]; then
|
|
log "\t\t\t\tDRY-RUN: Would remove now '$filename' (${filesize}kB).."
|
|
if [ -f "$xml_file" ];then
|
|
log "\t\t\t\tDRY-RUN: Would delete now also the corresponding '$xml_file' ... "
|
|
fi
|
|
else
|
|
log "\t\t\t\tDRY-RUN: Would move now '$filename' (${filesize}kB) to '$dest_dir' ..."
|
|
if [ -f "$xml_file" ];then
|
|
log "\t\t\t\tDRY-RUN: Would move now also the corresponding '$xml_file' to '$dest_dir' ... "
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
over_limit=$((over_limit-filesize))
|
|
count=$((count+1))
|
|
|
|
if [ $over_limit -le 0 ]; then
|
|
removed=$((dir_size-max_dir_size-over_limit))
|
|
|
|
if [ $dry_run == 0 ]; then
|
|
if [ -z "$dest_dir" ]; then
|
|
log "\t\t\t\tDone. Removed $((removed/(1024*1024)))GB (${removed}kB) by deleting $count recorded events."
|
|
else
|
|
log "\t\t\t\tDone. Moved $((removed/(1024*1024)))GB (${removed}kB) in $count recorded events to directory '$dest_dir'."
|
|
fi
|
|
else
|
|
if [ -z "$dest_dir" ]; then
|
|
log "\t\t\t\tDRY_RUN: Done. Would have removed $((removed/(1024*1024)))GB (${removed}kB) by deleting $count recorded events."
|
|
else
|
|
log "\t\t\t\tDRY_RUN: Done. Would have moved $((removed/(1024*1024)))GB (${removed}kB) in $count recorded events to directory '$dest_dir'."
|
|
fi
|
|
fi
|
|
# we are done, so break from the loop
|
|
break
|
|
fi
|
|
done
|
|
end_ifs_block
|
|
|
|
return 0
|
|
}
|
|
|
|
#END SECTION "Work functions"
|
|
#######################################################################################
|
|
|
|
###############################################################################
|
|
#BEGIN Section "Main"
|
|
|
|
# initialize some more variables
|
|
dry_run=1
|
|
quiet=0
|
|
|
|
# set the signal handler
|
|
trap signal_handler INT TERM
|
|
|
|
# First initialize the values from the config, otherwise we cannot log anywhere
|
|
init_config
|
|
|
|
# Now get command line option, as these might override some values from the config or default variables
|
|
parse_options $@
|
|
|
|
if [ -e $PID_FILE ]; then
|
|
log "$ME ist already running. Exiting..."
|
|
exit $EXIT_ALREADY_RUNNING
|
|
else
|
|
echo $$ > $PID_FILE
|
|
fi
|
|
|
|
# We happily started..
|
|
log ""
|
|
log "$ME V$VERSION initialized and starting main operations."
|
|
|
|
if [ $dry_run == 1 ]; then
|
|
log "\tThis is a dry run, i.e. no files will be harmed. Use '-y' or '--yes' to deactivate the safety."
|
|
elif [ $dry_run == 0 ] && [ $quiet == 0 ]; then
|
|
log "\t!!! WARNING!!! This is now the real thing - files WILL BE DELETED. You can still abort within the next 5 seconds. !!! WARNING !!!"
|
|
echo "Waiting for 5 more seconds.."
|
|
sleep 1
|
|
echo "Waiting for 4 more seconds.."
|
|
sleep 1
|
|
echo "Waiting for 3 more seconds.."
|
|
sleep 1
|
|
echo "Waiting for 2 more seconds.."
|
|
sleep 1
|
|
echo "Waiting for 1 more seconds.."
|
|
sleep 1
|
|
echo "You have been warned. Proceeding..."
|
|
else
|
|
log "\t!!! WARNING!!! $ME is armed and targeting your files. !!! WARNING !!!"
|
|
fi
|
|
|
|
# now process each directory given in the rule-file
|
|
|
|
rule_line=0
|
|
cat $RULE_FILE | while read line; do
|
|
rule_line=$((${rule_line}+1))
|
|
if echo $line | egrep -q '^[[:space:]]*([^#;]+),([0-9]+);?([^;]+)?(,[0-9]+)?$'; then
|
|
|
|
log ""
|
|
log "\tProcessing rule: '$line'"
|
|
|
|
# split rule line
|
|
begin_ifs_block
|
|
IFS=';'
|
|
set -- $line
|
|
record_part=$1
|
|
last_chance_part=$2
|
|
end_ifs_block
|
|
|
|
# split record_part
|
|
begin_ifs_block
|
|
IFS=','
|
|
set -- $record_part
|
|
record_dir=$1
|
|
record_dir_size=$2
|
|
end_ifs_block
|
|
|
|
if [ -n "$last_chance_part" ]; then
|
|
# split last_chance_part
|
|
begin_ifs_block
|
|
IFS=','
|
|
set -- $last_chance_part
|
|
last_chance_dir=$1
|
|
last_chance_dir_size=$2
|
|
end_ifs_block
|
|
if [ -z $last_chance_dir_size ]; then
|
|
last_chance_dir_size=$((record_dir_size / 10))
|
|
fi
|
|
else
|
|
last_chance_dir="last_chance"
|
|
last_chance_dir_size=$((record_dir_size / 10))
|
|
fi
|
|
|
|
# convert GB into kB
|
|
record_dir_size_k=$((record_dir_size * 1024 * 1024))
|
|
last_chance_dir_size_k=$((last_chance_dir_size * 1024 * 1024))
|
|
|
|
# print the collected information to the log:
|
|
log "\t\tCleaning path: '$record_dir', Maximum size: ${record_dir_size}GB (${record_dir_size_k}kB)"
|
|
log "\t\tLast chance subdirectory: '$last_chance_dir', Maximum size: ${last_chance_dir_size}GB (${last_chance_dir_size_k}kB)"
|
|
|
|
# now check if directories exist
|
|
# if the cleaning directory does not exist, print an error and continue with next rule
|
|
if [ ! -d "$record_dir" ]; then
|
|
log "\t\tThe given directory '$record_dir' does not exist. Create it or correct this rule. Skipping this rule."
|
|
continue
|
|
fi
|
|
|
|
#convert the last_chance relative path to an absolut one
|
|
last_chance_dir="${record_dir%/}/$last_chance_dir"
|
|
|
|
# if the last chance directory does not exist yet, create it.
|
|
if [ ! -d "$last_chance_dir" ]; then
|
|
|
|
if [ $dry_run == 0 ]; then
|
|
log "\t\tCreating directory '$last_chance_dir' for last chance files, as it does not exist yet."
|
|
mkdir "$last_chance_dir"
|
|
else
|
|
log "\t\tWould now create directory '$last_chance_dir' for last chance files, as it does not exist yet. (dry-run, i.e. NOT performing any action)"
|
|
fi
|
|
fi
|
|
|
|
# get the current size of the directories (in kB)...
|
|
current_record_dir_usage_k=$(du -s -k "$record_dir" | cut -f1)
|
|
current_last_chance_dir_size_k=$(du -s -k "$last_chance_dir" | cut -f1)
|
|
|
|
# ... and print them into the log
|
|
log "\t\tCurrent full size of '$record_dir' (recursively) is $((current_record_dir_usage_k/(1024*1024)))GB (${current_record_dir_usage_k}kB)."
|
|
log "\t\tCurrent size of '$last_chance_dir' is $((current_last_chance_dir_size_k/(1024*1024)))GB (${current_last_chance_dir_size_k}kB)."
|
|
|
|
# perform some initial checks, if we actually need to do something
|
|
if [ $((current_record_dir_usage_k-current_last_chance_dir_size_k)) -le $((record_dir_size_k-last_chance_dir_size_k)) ] && [ $current_last_chance_dir_size_k -le $last_chance_dir_size_k ] ;then
|
|
log "\t\tNothing to do for this rule - disk usage is within the given specification."
|
|
continue
|
|
fi
|
|
|
|
over_limit=0
|
|
if [ $current_record_dir_usage_k -gt $record_dir_size_k ];then
|
|
over_limit=$((current_record_dir_usage_k-record_dir_size_k))
|
|
log "\t\tWe need to remove $((over_limit/(1024*1024)))GB (${over_limit}kB) from directory '$record_dir'."
|
|
fi
|
|
|
|
if [ $((current_record_dir_usage_k-current_last_chance_dir_size_k-over_limit)) -gt $((record_dir_size_k-last_chance_dir_size_k)) ];then
|
|
move_size=$((current_record_dir_usage_k-current_last_chance_dir_size_k-over_limit-(record_dir_size_k-last_chance_dir_size_k)))
|
|
log "\t\tWe need to move $((move_size/(1024*1024)))GB (${move_size}kB) from directory '$record_dir' to '$last_chance_dir'."
|
|
fi
|
|
|
|
# first we delete the oldest files from the main directory (including last chance)
|
|
# if your last_chance is too small, some files will never appear in there
|
|
limit_directory "$record_dir" "$record_dir_size_k"
|
|
|
|
# now fill the last chance directory
|
|
limit_directory "$record_dir" "$((record_dir_size_k-last_chance_dir_size_k))" "$last_chance_dir"
|
|
|
|
# final status
|
|
new_record_dir_usage_k=$(du -s -k "$record_dir" | cut -f1)
|
|
new_last_chance_dir_size_k=$(du -s -k "$last_chance_dir" | cut -f1)
|
|
|
|
log "\t\tNew full size of '$record_dir' (recursively) is $((new_record_dir_usage_k/(1024*1024)))GB (${new_record_dir_usage_k}kB)."
|
|
log "\t\tNew size of '$last_chance_dir' is $((new_last_chance_dir_size_k/(1024*1024)))GB (${new_last_chance_dir_size_k}kB)."
|
|
fi
|
|
done
|
|
|
|
log ""
|
|
log "$ME V${VERSION} finished rule processing."
|
|
|
|
cleanup
|
|
log "========================== $NAME finished at $(date) ======================================"
|
|
exit $EXIT_NORMAL;
|
|
|