Added interactive config

index.html

@@ -40,13 +40,13 @@
         <div class="bg-white dark:bg-gray-800 col-span-1 md:col-span-2 shadow-lg rounded-lg p-5">
   <h3 class="dark:text-white text-lg font-semibold mb-1"><a href="/archlinux"
       class="text-blue-500 hover:text-blue-700">ArchLinux (x86_64)</a></h3>
-  <p class="text-gray-600 dark:text-gray-300">Last Updated: Not Synced</p>
+  <p class="text-gray-600 dark:text-gray-300">Last Updated: 2023-10-12 11:24</p>
   <p class="text-gray-600 dark:text-gray-300">Upstream: <span class="url">rsync://mirrors.xtom.de/archlinux/</span></p>
 </div>
 <div class="bg-white dark:bg-gray-800 col-span-1 shadow-lg rounded-lg p-5">
   <h3 class="dark:text-white text-lg font-semibold mb-1"><a href="/ubuntu"
       class="text-blue-500 hover:text-blue-700">Ubuntu</a></h3>
-  <p class="text-gray-600 dark:text-gray-300">Last Updated: Not Synced</p>
+  <p class="text-gray-600 dark:text-gray-300">Last Updated: 2023-10-12 11:24</p>
   <p class="text-gray-600 dark:text-gray-300">Upstream: <span class="url">rsync://rsync.archive.ubuntu.com/ubuntu/</span></p>
 </div>
 <div class="bg-white dark:bg-gray-800 col-span-1 shadow-lg rounded-lg p-5">

scripts/config.yml

@@ -1,19 +1,25 @@
 BASE_DIR: /srv/mirror
-ENV: ./scripts/.env
+
+# Any path under this must be in relative of BASE_DIR.
+# Script doesnt do any kind of outbound check, so make sure to check all paths.
+
 global:
+  scripts:
+    pre:
+    post:
+  log: logs/sync.log
+  sync:
+    - ALL
+
+defaults:
   type: rsync
   scripts:
     pre:
-    # - ./scripts/cleanup.sh
     fail:
-      - ./scripts/alert.sh
+      - scripts/alert.sh
     post:
-      - ./script/index.py
+      - scripts/index.py
-    # - ./script/rotate_log.py
+  log: logs/${path}.log
-  log: ./logs/sync.log
-  log_dir: ./logs/
-  sync:
-    - ALL
   rsync:
     options: "-rtlHpv --chmod=D0755,F0644 --partial --hard-links --safe-links --stats --delete --delete-after --delay-updates --max-delete=70000"
     exclude:
@@ -25,35 +31,32 @@ global:
       - "*.links.tar.gz*"
       - "/other"
       - "/sources"
+
 repos:
   ARCHLINUX:
     url: rsync://mirrors.xtom.de/archlinux/
     duration: 4
     name: "ArchLinux (x86_64)"
     path: archlinux
-    log: ./logs/archlinux
+
-    last_sync: "2023-10-09T23:48Z"
   UBUNTU:
     url: rsync://rsync.archive.ubuntu.com/ubuntu/
     duration: 4
     name: "Ubuntu"
     path: ubuntu
-    log: ./logs/ubuntu
+
-    last_sync: "2023-10-09T23:48Z"
   UBUNTU_CD:
     url: rsync://releases.ubuntu.com/releases/
-    duration: 24
+    duration: 12
     name: "Ubuntu Releases"
     path: ubuntu_cd
-    log: ./logs/ubuntu_cd
+
-    last_sync: "2023-10-09T23:48Z"
   UBUNTU_CD_OLD:
     url: rsync://old-releases.ubuntu.com/releases/
-    duration: 24
+    duration: 12
     name: "Ubuntu Releases (Old)"
     path: ubuntu_cd_old
-    log: ./logs/ubuntu_cd_old
+
-    last_sync: "2023-10-09T23:48Z"
   DEBIAN:
     url: rsync://mirrors.xtom.jp/debian/
     type: ftpsync
@@ -61,50 +64,47 @@ repos:
     name: Debian
     path: debian
     scripts:
-    # pre: ./scripts/pre_debian.sh
+    # pre: scripts/pre_debian.sh
-    last_sync: "2023-10-09T23:48Z"
+    last_sync: "2023-10-12T10:58Z"
+  
   DEBIAN_CD:
     url: rsync://ftp.lanet.kr/debian-cd/
-    duration: 24
+    duration: 12
     name: "Debian CD"
     path: debian_cd
-    log: ./logs/debian_cd
+  
-    last_sync: "2023-10-09T23:48Z"
   FEDORA:
     url: rsync://dl.fedoraproject.org/fedora-enchilada/linux/
     duration: 4
     name: "Fedora"
     path: fedora
-    log: ./logs/fedora
+  
-    last_sync: "2023-10-09T23:30Z"
   RASPBIAN:
     url: rsync://archive.raspbian.org/archive/
     duration: 6
     name: "Raspbian"
     path: raspbian
-    log: ./logs/raspbian
+  
-    last_sync: "2023-10-09T23:30Z"
   MANJARO:
     url: rsync://ftp.riken.jp/manjaro/
     duration: 4
     name: Manjaro
     path: manjaro
-    log: ./logs/manjaro
+  
-    last_sync: "2023-10-09T23:30Z"
   ARCHLINUXARM:
     name: "ArchLinux (ARM)"
     path: archlinuxarm
     url: http://jp.mirror.archlinuxarm.org/
     type: http
     duration: 12
-    last_sync: "2023-10-09T23:30Z"
+  
   ASAHILINUX:
     name: "AsahiLinux"
     path: asahilinux
     url: https://cdn.asahilinux.org/
     type: http
     duration: 12
-    last_sync: "2023-10-09T23:30Z"
+
 index:
   - ARCHLINUX
   - UBUNTU UBUNTU_CD
@@ -115,4 +115,4 @@ index:
   - DIVIDER
   - ARCHLINUXARM
   - ASAHILINUX
-  - UBUNTU_CD_OLD
+  - UBUNTU_CD_OLD

scripts/index.py

@@ -6,22 +6,21 @@ import jinja2
 import datetime
 
 BASE_DIR = "."
-
 CONFIG_PATH = os.path.join(BASE_DIR, "scripts/config.yml")
 TEMPLATES_DIR = os.path.join(BASE_DIR, "scripts/templates")
 LOG_DIR = os.path.join(BASE_DIR, "logs")
+SYNC_LOG = os.path.join(LOG_DIR, "sync.log")
 OUTPUT_PATH = os.path.join(BASE_DIR, "index.html")
 
-# def get_last_sync(repo_name):
+def get_last_sync(repo_name):
-#     log_path = os.path.join(LOG_DIR, f"all.log")
+    if os.path.exists(SYNC_LOG):
-#     if os.path.exists(log_path):
+        with open(SYNC_LOG, 'r') as log_file:
-#         with open(log_path, 'r') as log_file:
+            log_all = reversed(log_file.readlines())
-#             log_all = reversed(log_file.readlines())
+            for logline in log_all:
-#             for logline in log_all:
+                dist, time, stat = logline.split()
-#                 time, stat, dist = logline.split()
+                if repo_name == dist and stat =="SUCCESS":
-#                 if repo_name == dist and stat =="DONE":
+                    return datetime.datetime.strptime(time, '%Y-%m-%dT%H:%MZ').strftime("%Y-%m-%d %H:%M")
-#                     return datetime.datetime.strptime(time, '%Y%m%d_%H%M').strftime("%Y-%m-%d %H:%M")
+    return "Not Synced"
-#     return "Not Synced"
 
 # main()
 
@@ -51,7 +50,8 @@ if __name__=="__main__":
             context = {
                 'path': repo_data['path'],
                 'name': repo_data['name'],
-                'lastsync': repo_data.get('lastsync', "Not Synced"),
+                'lastsync': get_last_sync(repo_name),
+                # 'lastsync': repo_data.get('lastsync', "Not Synced"),
                 'upstream': repo_data['url']
             }
             print(context)

scripts/sync.sh

@@ -1,66 +1,109 @@
 #!/bin/bash
 
-CONFIG_PATH="./scripts/config.yml"
+# Default locations.
+CONFIG="./scripts/config.yml"
 YAML="./scripts/yq"
 
-BASE_DIR=$($YAML eval ".BASE_DIR" "${CONFIG_PATH}")
+function usage() {
-TIMESTAMP=$(date '+%Y-%m-%dT%H:%MZ')
+    cat <<EOF
+Usage: $0 [-c config_path] [-y yq_path]
+
+Flags:
+    -c, --config_path:      config file path
+    -y, --yq:               yq binary path (>v4.35.2)
+    -v, --verbose
+      -> -b, --base-dir:         overrides base_dir
+      -> -d, --dry-run:          dry run
+EOF
+    exit 1
+}
+
+function error() {
+    echo $0: "$1"
+    exit
+}
+
+function debug() {
+    if [[ -v DEBUG ]]; then
+        echo debug: "$1"
+    fi
+}
+
+function execute() {
+    debug "Executing script: \"$@\""
+    if [[ ! -v DRY_RUN ]]; then
+        $@
+    fi
+}
 
 parse_yaml() {
-    echo $($YAML eval ".$1" "${CONFIG_PATH}")
+    echo $($YAML eval ".$1" "${CONFIG}")
+}
+
+get_repo_config() {
+    local LOCAL_CFG=$($YAML eval ".repos.$1.$2" "${CONFIG}")
+    if [[ "$LOCAL_CFG" == "null" || "$LOCAL_CFG" == "" ]]; then
+        echo $($YAML eval ".defaults.$2" "${CONFIG}");
+        return 1
+    fi
+    echo $LOCAL_CFG
+    return 0
+
+    # 1 if global 0 if local
 }
 
 sync_repo() {
-
     local repo=$1
-    local type=$(parse_yaml "repos.${repo}.type")
+    local name=$(get_repo_config ${repo} name)
-    local url=$(parse_yaml "repos.${repo}.url")
+    local type=$(get_repo_config ${repo} type)
-    local path=$(parse_yaml "repos.${repo}.path")
+    local url=$(get_repo_config ${repo} url)
-    local log=$(parse_yaml "repos.${repo}.log")
+    local path=$(get_repo_config ${repo} path)
+    local log; log=$(get_repo_config ${repo} log) || log=${log/\${path\}/$path};
 
-    if [[ "$type" == "null" ]]; then
+    if [[ "$type" == "null" || "$url" == "null" || "$path" == "null" || "$log" == "null" ]]; then
-        type=$(parse_yaml "global.type")
+        echo "Error config wrong."
+        return
     fi
 
-    if [[ "$log" == "null" ]]; then
-        log="$(parse_yaml "global.log_dir")$path"
-    fi
-
-    log=$(realpath $log)
     path="${BASE_DIR}/$path/"
-    echo -e "----\n|  Repo: $repo\n|  Type: $type\n|  Upstream: $url\n|  Path: $path\n|  Log: $log\n----"
+    log="${BASE_DIR}/$log"
-    
-    rotate_log $log
 
-    case $type in
+    echo -e "--------\nRepo:      $name\nType:      $type\nUpstream:  $url\nPath:      $path\nLog:       $log\n--------"
-        "rsync")
-            local rsync_options=$(parse_yaml 'global.rsync.options')
-            local exclude_list=($(parse_yaml 'global.rsync.exclude[]'))
-            local exclude=""
-            for ex in "${exclude_list[@]}"; do
-                exclude="${exclude} --exclude='${ex}'"
-            done
-            echo rsync ${rsync_options} ${exclude} $url $path >> $log
-            rsync ${rsync_options} ${exclude} $url $path >> $log 2>> ${log}-error
-            ;;
-        "ftpsync")
-            cd ${BASE_DIR}/scripts
-            export BASE_DIR=${BASE_DIR}
-            ./ftpsync >> $log 2>> ${log}-error
-            cd ${BASE_DIR}
-            ;;
-        "http")
-            echo ${repo} Fetch >> $log 2>> ${log}-error
-            python3 -u $BASE_DIR/scripts/getFetch.py "${url}" $path $BASE_DIR/scripts/${path}.fetch >> $log 2>> ${log}-error
-            echo ${repo} Download >> $log 2>> ${log}-error
-            python3 -u $BASE_DIR/scripts/getFile.py $BASE_DIR/scripts/${path}.fetch >> $log 2>> ${log}-error
-            ;;
-        *)
-            echo "Unknown type $type for $repo." | tee ${log}-error
-            ;;
-    esac
 
-    clean_log $log
+    if [[ ! -v DRY_RUN ]]; then
+
+        rotate_log $log
+
+        case $type in
+            "rsync")
+                local rsync_options=$(get_repo_config $repo 'rsync.options')
+                local exclude_list=($(get_repo_config $repo 'rsync.exclude[]'))
+                local exclude=""
+                for ex in "${exclude_list[@]}"; do
+                    exclude="${exclude} --exclude='${ex}'"
+                done
+                echo rsync ${rsync_options} ${exclude} $url $path >> $log
+                rsync ${rsync_options} ${exclude} $url $path >> $log 2>> ${log}-error
+                ;;
+            "ftpsync")
+                cd ${BASE_DIR}/scripts
+                export BASE_DIR=${BASE_DIR}
+                ./ftpsync >> $log 2>> ${log}-error
+                cd ${BASE_DIR}
+                ;;
+            "http")
+                echo ${repo} Fetch >> $log 2>> ${log}-error
+                python3 -u $BASE_DIR/scripts/getFetch.py "${url}" $path $BASE_DIR/scripts/${path}.fetch >> $log 2>> ${log}-error
+                echo ${repo} Download >> $log 2>> ${log}-error
+                python3 -u $BASE_DIR/scripts/getFile.py $BASE_DIR/scripts/${path}.fetch >> $log 2>> ${log}-error
+                ;;
+            *)
+                echo "Unknown type $type for $repo." | tee ${log}-error
+                ;;
+        esac
+
+        clean_log $log
+    fi
 }
 
 rotate_log() {
@@ -92,50 +135,158 @@ clean_log() {
     fi
 }
 
+function global_log() {
+    GLOBAL_LOG_FILE="$(parse_yaml 'global.log')"
+    echo "$1 $TIMESTAMP SUCCESS" >> $GLOBAL_LOG_FILE
+}
+
+function global_log_error() {
+    GLOBAL_LOG_FILE="$(parse_yaml 'global.log')"
+    echo "$1 $TIMESTAMP ERROR" >> $GLOBAL_LOG_FILE
+}
+
 ##########
 #  Main  # 
 ##########
 
-echo Started job $TIMESTAMP..
+while [ "$1" != "" ]; do
-
+    case $1 in
-global_pre_scripts=($(parse_yaml 'global.scripts.pre[]'))
+    -c | --config)
-for script in "${global_pre_scripts[@]}"; do
+        shift
-    $script
+        if [[ -z $1 ]]; then
+            error "option '-c' requires argument 'config_path'"
+        fi
+        CONFIG=$1
+        ;;
+    -y | --yq)
+        shift
+        if [[ -z $1 ]]; then
+            error "option '-y' requires argument 'yq_path'"
+        fi
+        YAML=$1
+        ;;
+    -b | --base-dir)
+        shift
+        if [[ -z $1 ]]; then
+            error "option '-b' requires argument 'base_dir'"
+        fi
+        BASE_DIR_OVERRIDE=$1
+        ;;
+    -h | --help)
+        usage
+        ;;
+    -v | --verbose)
+        DEBUG=1
+        ;;
+    -d | --dry-run)
+        DRY_RUN=1
+        ;;
+    *)
+        usage
+        exit 1
+        ;;
+    esac
+    shift
 done
 
+if [[ -v DEBUG ]]; then
+    debug "DEBUG=1"
+fi
+if [[ -v DRY_RUN ]]; then
+    debug "DRY_RUN=1"
+fi
+debug CONFIG="\"${CONFIG}\""
+debug YQ="\"${YAML}\""
+
+if [[ ! -f ${CONFIG} ]]; then
+    error "config not found."
+fi
+
+if [[ ! -f ${YAML} ]]; then
+    error "yq not found."
+fi
+
+BASE_DIR=$($YAML eval ".BASE_DIR" "${CONFIG}")
+
+if [[ -v DEBUG && -v BASE_DIR_OVERRIDE ]]; then
+    debug "Overriding $BASE_DIR to $BASE_DIR_OVERRIDE"
+    BASE_DIR="$BASE_DIR_OVERRIDE"
+fi
+
+TIMESTAMP=$(date '+%Y-%m-%dT%H:%MZ')
+
+debug BASE_DIR="\"${BASE_DIR}\""
+debug TIMESTAMP="\"${TIMESTAMP}\""
+
+echo Started job $TIMESTAMP..
+
+cd $BASE_DIR
+# PRE
+global_pre_scripts=($(parse_yaml 'global.scripts.pre[]'))
+for script in "${global_pre_scripts[@]}"; do
+    execute $BASE_DIR/$script
+done
+#
 repos=($(parse_yaml 'global.sync[]'))
 
 if [[ "${repos[0]}" == "ALL" ]]; then
-    repos=($($YAML eval '.repos | keys| .[]' "${CONFIG_PATH}"))
+    repos=($($YAML eval '.repos | keys | .[]' "${CONFIG}"))
 fi
 for repo in "${repos[@]}"; do
-    echo Checking $repo...
+    cd $BASE_DIR
-    duration=$(parse_yaml "repos.${repo}.duration")
+    debug "Checking $repo..."
-    last_sync_timestamp=$(date -d "$(parse_yaml "repos.${repo}.last_sync")" +%s)
+    
+    duration=$(get_repo_config ${repo} "duration")
+    last_sync_timestamp=$(date -d "$(get_repo_config ${repo} "last_sync")" +%s)
     next_sync_timestamp=$(( last_sync_timestamp + duration * 3600 ))
-    next_sync_timestamp=1
+    
+    if [[ -v DEBUG ]]; then
+        next_sync_timestamp=1
+        # read -p "Continue? " choice
+        # case "$choice" in
+        #     y) next_sync_timestamp=1;;
+        #     *) continue;;
+        # esac
+    fi
+
     if [ $next_sync_timestamp -le $(date +%s) ]; then
-        echo "Lastsync $last_sync_timestamp"
+        debug "Lastsync was $last_sync_timestamp."
         echo "Syncing $repo..."
 
+        repo_pre_scripts=($(get_repo_config ${repo} "scripts.pre[]"))
+        for script in "${repo_pre_scripts[@]}"; do
+            execute $BASE_DIR/$script $repo
+        done
+
         sync_repo $repo
 
         if [ $? -ne 0 ]; then
-            global_fail_scripts=($(parse_yaml 'global.scripts.fail[]'))
+            repo_fail_scripts=($(get_repo_config ${repo} "scripts.fail[]"))
-            for script in "${global_fail_scripts[@]}"; do
+            for script in "${repo_fail_scripts[@]}"; do
-                $script
+                execute $BASE_DIR/$script $repo
             done
-            echo "Error syncing $repo"
+
+            global_log_error $repo
+            echo "Error during syncing $repo."
         else
-            $YAML eval ".repos.${repo}.last_sync = \"$TIMESTAMP\"" -i "${CONFIG_PATH}"
+
+            global_log $repo
+            $YAML eval ".repos.${repo}.last_sync = \"$TIMESTAMP\"" -i "${CONFIG}"
             echo "Successfully synced $repo."
         fi
+
+        repo_post_scripts=($(get_repo_config ${repo} "scripts.post[]"))
+        for script in "${repo_post_scripts[@]}"; do
+            execute $BASE_DIR/$script $repo
+        done
     fi
 done
 
+# POST
 global_post_scripts=($(parse_yaml 'global.scripts.post[]'))
 for script in "${global_post_scripts[@]}"; do
-    $script
+    execute $BASE_DIR/$script
 done
+#
 
-echo Ended job $TIMESTAMP..
+echo Ended job $TIMESTAMP..