#!/bin/bash # Default locations. CONFIG="./scripts/config.yml" YAML="./scripts/yq" function usage() { cat <v4.35.2) -v, --verbose -> -b, --base-dir: overrides base_dir -> -d, --dry-run: dry run EOF exit 1 } function error() { echo $0: "$1" exit } function debug() { if [[ -v DEBUG ]]; then echo debug: "$1" fi } function execute() { debug "Executing script: \"$@\"" if [[ ! -v DRY_RUN ]]; then $@ fi } parse_yaml() { echo $($YAML eval ".$1" "${CONFIG}") } get_repo_config() { local LOCAL_CFG=$($YAML eval ".repos.$1.$2" "${CONFIG}") if [[ "$LOCAL_CFG" == "null" || "$LOCAL_CFG" == "" ]]; then echo $($YAML eval ".defaults.$2" "${CONFIG}"); return 1 fi echo $LOCAL_CFG return 0 # 1 if global 0 if local } sync_repo() { local repo=$1 local name=$(get_repo_config ${repo} name) local type=$(get_repo_config ${repo} type) local url=$(get_repo_config ${repo} url) local path=$(get_repo_config ${repo} path) local log; log=$(get_repo_config ${repo} log) || log=${log/\${path\}/$path}; if [[ "$type" == "null" || "$url" == "null" || "$path" == "null" || "$log" == "null" ]]; then echo "Error config wrong." return fi full_path="${BASE_DIR}/$(parse_yaml global.data_dir)/$path/" log="${BASE_DIR}/$log" echo -e "--------\nRepo: $name\nType: $type\nUpstream: $url\nPath: $full_path\nLog: $log\n--------" if [[ ! -v DRY_RUN ]]; then rotate_log $log case $type in "rsync") local rsync_options=$(get_repo_config $repo 'rsync.options') local exclude_list=($(get_repo_config $repo 'rsync.exclude[]')) local exclude="" for ex in "${exclude_list[@]}"; do exclude="${exclude} --exclude='${ex}'" done echo rsync ${rsync_options} ${exclude} $url $full_path >> $log rsync ${rsync_options} ${exclude} $url $full_path >> $log 2>> ${log}-error ;; "ftpsync") cd ${BASE_DIR}/scripts export DEB_ADMIN="Morgan " export DEB_COUNTRY="Korea" export DEB_LOCATION="Korea/Seoul" export DEB_PATH="$full_path" export DEB_HOST="$(parse_yaml global.hostname)" export DEB_UPSTREAM="$url" export DEB_LOG="${BASE_DIR}/$(parse_yaml global.log_dir)" ./ftpsync >> $log 2>> ${log}-error cd ${BASE_DIR} ;; "http") echo ${repo} Fetch >> $log 2>> ${log}-error python3 -u $BASE_DIR/scripts/getFetch.py "${url}" $full_path $BASE_DIR/scripts/${path}.fetch >> $log 2>> ${log}-error echo ${repo} Download >> $log 2>> ${log}-error python3 -u $BASE_DIR/scripts/getFile.py $BASE_DIR/scripts/${path}.fetch >> $log 2>> ${log}-error ;; *) echo "Unknown type $type for $repo." | tee ${log}-error ;; esac clean_log $log fi } rotate_log() { local log_file=$1 if [[ -f $log_file ]]; then PREV_LOG=$(cat "$log_file" | head -n 1) old_log_file="$(dirname $log_file)/old/$(basename $log_file)-$PREV_LOG" mkdir -p "$(dirname $old_log_file)" mv "$log_file" "$old_log_file" fi local error_file=$1-error if [[ -f $error_file ]]; then PREV_LOG=$(cat "$error_file" | head -n 1) old_error_file="$(dirname $error_file)/old/$(basename $error_file)-$PREV_LOG" mkdir -p "$(dirname $old_error_file)" mv "$error_file" "$old_error_file" fi echo $TIMESTAMP >> $log_file echo $TIMESTAMP >> $error_file } clean_log() { local error_file=$1-error nl=$(cat "$error_file" | wc -l) if [ $nl -eq 1 ]; then rm "$error_file" fi } function global_log() { GLOBAL_LOG_FILE="$(parse_yaml 'global.log')" echo "$1 $TIMESTAMP SUCCESS" >> $GLOBAL_LOG_FILE } function global_log_error() { GLOBAL_LOG_FILE="$(parse_yaml 'global.log')" echo "$1 $TIMESTAMP ERROR" >> $GLOBAL_LOG_FILE } ########## # Main # ########## while [ "$1" != "" ]; do case $1 in -c | --config) shift if [[ -z $1 ]]; then error "option '-c' requires argument 'config_path'" fi CONFIG=$1 ;; -y | --yq) shift if [[ -z $1 ]]; then error "option '-y' requires argument 'yq_path'" fi YAML=$1 ;; -b | --base-dir) shift if [[ -z $1 ]]; then error "option '-b' requires argument 'base_dir'" fi BASE_DIR_OVERRIDE=$1 ;; -h | --help) usage ;; -v | --verbose) DEBUG=1 ;; -d | --dry-run) DRY_RUN=1 ;; *) usage exit 1 ;; esac shift done if [[ -v DEBUG ]]; then debug "DEBUG=1" fi if [[ -v DRY_RUN ]]; then debug "DRY_RUN=1" fi debug CONFIG="\"${CONFIG}\"" debug YQ="\"${YAML}\"" if [[ ! -f ${CONFIG} ]]; then error "config not found." fi if [[ ! -f ${YAML} ]]; then error "yq not found." fi BASE_DIR=$($YAML eval ".BASE_DIR" "${CONFIG}") if [[ -v DEBUG && -v BASE_DIR_OVERRIDE ]]; then debug "Overriding $BASE_DIR to $BASE_DIR_OVERRIDE" BASE_DIR="$BASE_DIR_OVERRIDE" fi TIMESTAMP=$(date '+%Y-%m-%dT%H:%MZ') debug BASE_DIR="\"${BASE_DIR}\"" debug TIMESTAMP="\"${TIMESTAMP}\"" echo Started job $TIMESTAMP.. cd $BASE_DIR # PRE global_pre_scripts=($(parse_yaml 'global.scripts.pre[]')) for script in "${global_pre_scripts[@]}"; do execute $BASE_DIR/$script done # repos=($(parse_yaml 'global.sync[]')) if [[ "${repos[0]}" == "ALL" ]]; then repos=($($YAML eval '.repos | keys | .[]' "${CONFIG}")) fi for repo in "${repos[@]}"; do cd $BASE_DIR debug "Checking $repo..." delay=$(get_repo_config ${repo} "delay") last_sync_timestamp=$(date -d "$(get_repo_config ${repo} "last_sync")" +%s) next_sync_timestamp=$(( last_sync_timestamp + delay * 3600 )) if [[ -v DEBUG ]]; then next_sync_timestamp=1 # read -p "Continue? " choice # case "$choice" in # y) next_sync_timestamp=1;; # *) continue;; # esac fi if [ $next_sync_timestamp -le $(date +%s) ]; then debug "Lastsync was $last_sync_timestamp." echo "Syncing $repo..." repo_pre_scripts=($(get_repo_config ${repo} "scripts.pre[]")) for script in "${repo_pre_scripts[@]}"; do execute $BASE_DIR/$script $repo done sync_repo $repo if [ $? -ne 0 ]; then repo_fail_scripts=($(get_repo_config ${repo} "scripts.fail[]")) for script in "${repo_fail_scripts[@]}"; do execute $BASE_DIR/$script $repo done global_log_error $repo echo "Error during syncing $repo." else global_log $repo $YAML eval ".repos.${repo}.last_sync = \"$TIMESTAMP\"" -i "${CONFIG}" echo "Successfully synced $repo." fi repo_post_scripts=($(get_repo_config ${repo} "scripts.post[]")) for script in "${repo_post_scripts[@]}"; do script=${script/\${config\}/$CONFIG} execute $BASE_DIR/$script $repo done fi done # POST global_post_scripts=($(parse_yaml 'global.scripts.post[]')) for script in "${global_post_scripts[@]}"; do execute $BASE_DIR/$script done # echo Ended job $TIMESTAMP..