Added interactive config

2023-10-12 12:54:00 +09:00 · 2023-10-12 12:54:00 +09:00 · bdc2eb09bb
parent d2eeadba89
commit bdc2eb09bb
5 changed files with 263 additions and 112 deletions
--- a/index.html
+++ b/index.html
@ -40,13 +40,13 @@
        <div class="bg-white dark:bg-gray-800 col-span-1 md:col-span-2 shadow-lg rounded-lg p-5">
  <h3 class="dark:text-white text-lg font-semibold mb-1"><a href="/archlinux"
      class="text-blue-500 hover:text-blue-700">ArchLinux (x86_64)</a></h3>
-  <p class="text-gray-600 dark:text-gray-300">Last Updated: Not Synced</p>
+  <p class="text-gray-600 dark:text-gray-300">Last Updated: 2023-10-12 11:24</p>
  <p class="text-gray-600 dark:text-gray-300">Upstream: <span class="url">rsync://mirrors.xtom.de/archlinux/</span></p>
 </div>
 <div class="bg-white dark:bg-gray-800 col-span-1 shadow-lg rounded-lg p-5">
  <h3 class="dark:text-white text-lg font-semibold mb-1"><a href="/ubuntu"
      class="text-blue-500 hover:text-blue-700">Ubuntu</a></h3>
-  <p class="text-gray-600 dark:text-gray-300">Last Updated: Not Synced</p>
+  <p class="text-gray-600 dark:text-gray-300">Last Updated: 2023-10-12 11:24</p>
  <p class="text-gray-600 dark:text-gray-300">Upstream: <span class="url">rsync://rsync.archive.ubuntu.com/ubuntu/</span></p>
 </div>
 <div class="bg-white dark:bg-gray-800 col-span-1 shadow-lg rounded-lg p-5">
--- a/logs/old/.~tmp~
+++ b/logs/old/.~tmp~
--- a/scripts/config.yml
+++ b/scripts/config.yml
@ -1,19 +1,25 @@
 BASE_DIR: /srv/mirror
-ENV: ./scripts/.env
+
+# Any path under this must be in relative of BASE_DIR.
+# Script doesnt do any kind of outbound check, so make sure to check all paths.
+
 global:
+  scripts:
+    pre:
+    post:
+  log: logs/sync.log
+  sync:
+    - ALL
+
+defaults:
  type: rsync
  scripts:
    pre:
-    # - ./scripts/cleanup.sh
    fail:
-      - ./scripts/alert.sh
+      - scripts/alert.sh
    post:
-      - ./script/index.py
-    # - ./script/rotate_log.py
-  log: ./logs/sync.log
-  log_dir: ./logs/
-  sync:
-    - ALL
+      - scripts/index.py
+  log: logs/${path}.log
  rsync:
    options: "-rtlHpv --chmod=D0755,F0644 --partial --hard-links --safe-links --stats --delete --delete-after --delay-updates --max-delete=70000"
    exclude:
@ -25,35 +31,32 @@ global:
      - "*.links.tar.gz*"
      - "/other"
      - "/sources"
+
 repos:
  ARCHLINUX:
    url: rsync://mirrors.xtom.de/archlinux/
    duration: 4
    name: "ArchLinux (x86_64)"
    path: archlinux
-    log: ./logs/archlinux
-    last_sync: "2023-10-09T23:48Z"
+
  UBUNTU:
    url: rsync://rsync.archive.ubuntu.com/ubuntu/
    duration: 4
    name: "Ubuntu"
    path: ubuntu
-    log: ./logs/ubuntu
-    last_sync: "2023-10-09T23:48Z"
+
  UBUNTU_CD:
    url: rsync://releases.ubuntu.com/releases/
-    duration: 24
+    duration: 12
    name: "Ubuntu Releases"
    path: ubuntu_cd
-    log: ./logs/ubuntu_cd
-    last_sync: "2023-10-09T23:48Z"
+
  UBUNTU_CD_OLD:
    url: rsync://old-releases.ubuntu.com/releases/
-    duration: 24
+    duration: 12
    name: "Ubuntu Releases (Old)"
    path: ubuntu_cd_old
-    log: ./logs/ubuntu_cd_old
-    last_sync: "2023-10-09T23:48Z"
+
  DEBIAN:
    url: rsync://mirrors.xtom.jp/debian/
    type: ftpsync
@ -61,50 +64,47 @@ repos:
    name: Debian
    path: debian
    scripts:
-    # pre: ./scripts/pre_debian.sh
-    last_sync: "2023-10-09T23:48Z"
+    # pre: scripts/pre_debian.sh
+    last_sync: "2023-10-12T10:58Z"
+  
  DEBIAN_CD:
    url: rsync://ftp.lanet.kr/debian-cd/
-    duration: 24
+    duration: 12
    name: "Debian CD"
    path: debian_cd
-    log: ./logs/debian_cd
-    last_sync: "2023-10-09T23:48Z"
+  
  FEDORA:
    url: rsync://dl.fedoraproject.org/fedora-enchilada/linux/
    duration: 4
    name: "Fedora"
    path: fedora
-    log: ./logs/fedora
-    last_sync: "2023-10-09T23:30Z"
+  
  RASPBIAN:
    url: rsync://archive.raspbian.org/archive/
    duration: 6
    name: "Raspbian"
    path: raspbian
-    log: ./logs/raspbian
-    last_sync: "2023-10-09T23:30Z"
+  
  MANJARO:
    url: rsync://ftp.riken.jp/manjaro/
    duration: 4
    name: Manjaro
    path: manjaro
-    log: ./logs/manjaro
-    last_sync: "2023-10-09T23:30Z"
+  
  ARCHLINUXARM:
    name: "ArchLinux (ARM)"
    path: archlinuxarm
    url: http://jp.mirror.archlinuxarm.org/
    type: http
    duration: 12
-    last_sync: "2023-10-09T23:30Z"
+  
  ASAHILINUX:
    name: "AsahiLinux"
    path: asahilinux
    url: https://cdn.asahilinux.org/
    type: http
    duration: 12
-    last_sync: "2023-10-09T23:30Z"
+
 index:
  - ARCHLINUX
  - UBUNTU UBUNTU_CD
@ -115,4 +115,4 @@ index:
  - DIVIDER
  - ARCHLINUXARM
  - ASAHILINUX
-  - UBUNTU_CD_OLD
+  - UBUNTU_CD_OLD
--- a/scripts/index.py
+++ b/scripts/index.py
@ -6,22 +6,21 @@ import jinja2
 import datetime

 BASE_DIR = "."
-
 CONFIG_PATH = os.path.join(BASE_DIR, "scripts/config.yml")
 TEMPLATES_DIR = os.path.join(BASE_DIR, "scripts/templates")
 LOG_DIR = os.path.join(BASE_DIR, "logs")
+SYNC_LOG = os.path.join(LOG_DIR, "sync.log")
 OUTPUT_PATH = os.path.join(BASE_DIR, "index.html")

-# def get_last_sync(repo_name):
-#     log_path = os.path.join(LOG_DIR, f"all.log")
-#     if os.path.exists(log_path):
-#         with open(log_path, 'r') as log_file:
-#             log_all = reversed(log_file.readlines())
-#             for logline in log_all:
-#                 time, stat, dist = logline.split()
-#                 if repo_name == dist and stat =="DONE":
-#                     return datetime.datetime.strptime(time, '%Y%m%d_%H%M').strftime("%Y-%m-%d %H:%M")
-#     return "Not Synced"
+def get_last_sync(repo_name):
+    if os.path.exists(SYNC_LOG):
+        with open(SYNC_LOG, 'r') as log_file:
+            log_all = reversed(log_file.readlines())
+            for logline in log_all:
+                dist, time, stat = logline.split()
+                if repo_name == dist and stat =="SUCCESS":
+                    return datetime.datetime.strptime(time, '%Y-%m-%dT%H:%MZ').strftime("%Y-%m-%d %H:%M")
+    return "Not Synced"

 # main()

@ -51,7 +50,8 @@ if __name__=="__main__":
            context = {
                'path': repo_data['path'],
                'name': repo_data['name'],
-                'lastsync': repo_data.get('lastsync', "Not Synced"),
+                'lastsync': get_last_sync(repo_name),
+                # 'lastsync': repo_data.get('lastsync', "Not Synced"),
                'upstream': repo_data['url']
            }
            print(context)
--- a/scripts/sync.sh
+++ b/scripts/sync.sh
@ -1,66 +1,109 @@
 #!/bin/bash

-CONFIG_PATH="./scripts/config.yml"
+# Default locations.
+CONFIG="./scripts/config.yml"
 YAML="./scripts/yq"

-BASE_DIR=$($YAML eval ".BASE_DIR" "${CONFIG_PATH}")
-TIMESTAMP=$(date '+%Y-%m-%dT%H:%MZ')
+function usage() {
+    cat <<EOF
+Usage: $0 [-c config_path] [-y yq_path]
+
+Flags:
+    -c, --config_path:      config file path
+    -y, --yq:               yq binary path (>v4.35.2)
+    -v, --verbose
+      -> -b, --base-dir:         overrides base_dir
+      -> -d, --dry-run:          dry run
+EOF
+    exit 1
+}
+
+function error() {
+    echo $0: "$1"
+    exit
+}
+
+function debug() {
+    if [[ -v DEBUG ]]; then
+        echo debug: "$1"
+    fi
+}
+
+function execute() {
+    debug "Executing script: \"$@\""
+    if [[ ! -v DRY_RUN ]]; then
+        $@
+    fi
+}

 parse_yaml() {
-    echo $($YAML eval ".$1" "${CONFIG_PATH}")
+    echo $($YAML eval ".$1" "${CONFIG}")
+}
+
+get_repo_config() {
+    local LOCAL_CFG=$($YAML eval ".repos.$1.$2" "${CONFIG}")
+    if [[ "$LOCAL_CFG" == "null" || "$LOCAL_CFG" == "" ]]; then
+        echo $($YAML eval ".defaults.$2" "${CONFIG}");
+        return 1
+    fi
+    echo $LOCAL_CFG
+    return 0
+
+    # 1 if global 0 if local
 }

 sync_repo() {
-
    local repo=$1
-    local type=$(parse_yaml "repos.${repo}.type")
-    local url=$(parse_yaml "repos.${repo}.url")
-    local path=$(parse_yaml "repos.${repo}.path")
-    local log=$(parse_yaml "repos.${repo}.log")
+    local name=$(get_repo_config ${repo} name)
+    local type=$(get_repo_config ${repo} type)
+    local url=$(get_repo_config ${repo} url)
+    local path=$(get_repo_config ${repo} path)
+    local log; log=$(get_repo_config ${repo} log) || log=${log/\${path\}/$path};

-    if [[ "$type" == "null" ]]; then
-        type=$(parse_yaml "global.type")
+    if [[ "$type" == "null" || "$url" == "null" || "$path" == "null" || "$log" == "null" ]]; then
+        echo "Error config wrong."
+        return
    fi

-    if [[ "$log" == "null" ]]; then
-        log="$(parse_yaml "global.log_dir")$path"
-    fi
-
-    log=$(realpath $log)
    path="${BASE_DIR}/$path/"
-    echo -e "----\n|  Repo: $repo\n|  Type: $type\n|  Upstream: $url\n|  Path: $path\n|  Log: $log\n----"
-    
-    rotate_log $log
+    log="${BASE_DIR}/$log"

-    case $type in
-        "rsync")
-            local rsync_options=$(parse_yaml 'global.rsync.options')
-            local exclude_list=($(parse_yaml 'global.rsync.exclude[]'))
-            local exclude=""
-            for ex in "${exclude_list[@]}"; do
-                exclude="${exclude} --exclude='${ex}'"
-            done
-            echo rsync ${rsync_options} ${exclude} $url $path >> $log
-            rsync ${rsync_options} ${exclude} $url $path >> $log 2>> ${log}-error
-            ;;
-        "ftpsync")
-            cd ${BASE_DIR}/scripts
-            export BASE_DIR=${BASE_DIR}
-            ./ftpsync >> $log 2>> ${log}-error
-            cd ${BASE_DIR}
-            ;;
-        "http")
-            echo ${repo} Fetch >> $log 2>> ${log}-error
-            python3 -u $BASE_DIR/scripts/getFetch.py "${url}" $path $BASE_DIR/scripts/${path}.fetch >> $log 2>> ${log}-error
-            echo ${repo} Download >> $log 2>> ${log}-error
-            python3 -u $BASE_DIR/scripts/getFile.py $BASE_DIR/scripts/${path}.fetch >> $log 2>> ${log}-error
-            ;;
-        *)
-            echo "Unknown type $type for $repo." | tee ${log}-error
-            ;;
-    esac
+    echo -e "--------\nRepo:      $name\nType:      $type\nUpstream:  $url\nPath:      $path\nLog:       $log\n--------"

-    clean_log $log
+    if [[ ! -v DRY_RUN ]]; then
+
+        rotate_log $log
+
+        case $type in
+            "rsync")
+                local rsync_options=$(get_repo_config $repo 'rsync.options')
+                local exclude_list=($(get_repo_config $repo 'rsync.exclude[]'))
+                local exclude=""
+                for ex in "${exclude_list[@]}"; do
+                    exclude="${exclude} --exclude='${ex}'"
+                done
+                echo rsync ${rsync_options} ${exclude} $url $path >> $log
+                rsync ${rsync_options} ${exclude} $url $path >> $log 2>> ${log}-error
+                ;;
+            "ftpsync")
+                cd ${BASE_DIR}/scripts
+                export BASE_DIR=${BASE_DIR}
+                ./ftpsync >> $log 2>> ${log}-error
+                cd ${BASE_DIR}
+                ;;
+            "http")
+                echo ${repo} Fetch >> $log 2>> ${log}-error
+                python3 -u $BASE_DIR/scripts/getFetch.py "${url}" $path $BASE_DIR/scripts/${path}.fetch >> $log 2>> ${log}-error
+                echo ${repo} Download >> $log 2>> ${log}-error
+                python3 -u $BASE_DIR/scripts/getFile.py $BASE_DIR/scripts/${path}.fetch >> $log 2>> ${log}-error
+                ;;
+            *)
+                echo "Unknown type $type for $repo." | tee ${log}-error
+                ;;
+        esac
+
+        clean_log $log
+    fi
 }

 rotate_log() {
@ -92,50 +135,158 @@ clean_log() {
    fi
 }

+function global_log() {
+    GLOBAL_LOG_FILE="$(parse_yaml 'global.log')"
+    echo "$1 $TIMESTAMP SUCCESS" >> $GLOBAL_LOG_FILE
+}
+
+function global_log_error() {
+    GLOBAL_LOG_FILE="$(parse_yaml 'global.log')"
+    echo "$1 $TIMESTAMP ERROR" >> $GLOBAL_LOG_FILE
+}
+
 ##########
 #  Main  # 
 ##########

-echo Started job $TIMESTAMP..
-
-global_pre_scripts=($(parse_yaml 'global.scripts.pre[]'))
-for script in "${global_pre_scripts[@]}"; do
-    $script
+while [ "$1" != "" ]; do
+    case $1 in
+    -c | --config)
+        shift
+        if [[ -z $1 ]]; then
+            error "option '-c' requires argument 'config_path'"
+        fi
+        CONFIG=$1
+        ;;
+    -y | --yq)
+        shift
+        if [[ -z $1 ]]; then
+            error "option '-y' requires argument 'yq_path'"
+        fi
+        YAML=$1
+        ;;
+    -b | --base-dir)
+        shift
+        if [[ -z $1 ]]; then
+            error "option '-b' requires argument 'base_dir'"
+        fi
+        BASE_DIR_OVERRIDE=$1
+        ;;
+    -h | --help)
+        usage
+        ;;
+    -v | --verbose)
+        DEBUG=1
+        ;;
+    -d | --dry-run)
+        DRY_RUN=1
+        ;;
+    *)
+        usage
+        exit 1
+        ;;
+    esac
+    shift
 done

+if [[ -v DEBUG ]]; then
+    debug "DEBUG=1"
+fi
+if [[ -v DRY_RUN ]]; then
+    debug "DRY_RUN=1"
+fi
+debug CONFIG="\"${CONFIG}\""
+debug YQ="\"${YAML}\""
+
+if [[ ! -f ${CONFIG} ]]; then
+    error "config not found."
+fi
+
+if [[ ! -f ${YAML} ]]; then
+    error "yq not found."
+fi
+
+BASE_DIR=$($YAML eval ".BASE_DIR" "${CONFIG}")
+
+if [[ -v DEBUG && -v BASE_DIR_OVERRIDE ]]; then
+    debug "Overriding $BASE_DIR to $BASE_DIR_OVERRIDE"
+    BASE_DIR="$BASE_DIR_OVERRIDE"
+fi
+
+TIMESTAMP=$(date '+%Y-%m-%dT%H:%MZ')
+
+debug BASE_DIR="\"${BASE_DIR}\""
+debug TIMESTAMP="\"${TIMESTAMP}\""
+
+echo Started job $TIMESTAMP..
+
+cd $BASE_DIR
+# PRE
+global_pre_scripts=($(parse_yaml 'global.scripts.pre[]'))
+for script in "${global_pre_scripts[@]}"; do
+    execute $BASE_DIR/$script
+done
+#
 repos=($(parse_yaml 'global.sync[]'))

 if [[ "${repos[0]}" == "ALL" ]]; then
-    repos=($($YAML eval '.repos | keys| .[]' "${CONFIG_PATH}"))
+    repos=($($YAML eval '.repos | keys | .[]' "${CONFIG}"))
 fi
 for repo in "${repos[@]}"; do
-    echo Checking $repo...
-    duration=$(parse_yaml "repos.${repo}.duration")
-    last_sync_timestamp=$(date -d "$(parse_yaml "repos.${repo}.last_sync")" +%s)
+    cd $BASE_DIR
+    debug "Checking $repo..."
+    
+    duration=$(get_repo_config ${repo} "duration")
+    last_sync_timestamp=$(date -d "$(get_repo_config ${repo} "last_sync")" +%s)
    next_sync_timestamp=$(( last_sync_timestamp + duration * 3600 ))
-    next_sync_timestamp=1
+    
+    if [[ -v DEBUG ]]; then
+        next_sync_timestamp=1
+        # read -p "Continue? " choice
+        # case "$choice" in
+        #     y) next_sync_timestamp=1;;
+        #     *) continue;;
+        # esac
+    fi
+
    if [ $next_sync_timestamp -le $(date +%s) ]; then
-        echo "Lastsync $last_sync_timestamp"
+        debug "Lastsync was $last_sync_timestamp."
        echo "Syncing $repo..."

+        repo_pre_scripts=($(get_repo_config ${repo} "scripts.pre[]"))
+        for script in "${repo_pre_scripts[@]}"; do
+            execute $BASE_DIR/$script $repo
+        done
+
        sync_repo $repo

        if [ $? -ne 0 ]; then
-            global_fail_scripts=($(parse_yaml 'global.scripts.fail[]'))
-            for script in "${global_fail_scripts[@]}"; do
-                $script
+            repo_fail_scripts=($(get_repo_config ${repo} "scripts.fail[]"))
+            for script in "${repo_fail_scripts[@]}"; do
+                execute $BASE_DIR/$script $repo
            done
-            echo "Error syncing $repo"
+
+            global_log_error $repo
+            echo "Error during syncing $repo."
        else
-            $YAML eval ".repos.${repo}.last_sync = \"$TIMESTAMP\"" -i "${CONFIG_PATH}"
+
+            global_log $repo
+            $YAML eval ".repos.${repo}.last_sync = \"$TIMESTAMP\"" -i "${CONFIG}"
            echo "Successfully synced $repo."
        fi
+
+        repo_post_scripts=($(get_repo_config ${repo} "scripts.post[]"))
+        for script in "${repo_post_scripts[@]}"; do
+            execute $BASE_DIR/$script $repo
+        done
    fi
 done

+# POST
 global_post_scripts=($(parse_yaml 'global.scripts.post[]'))
 for script in "${global_post_scripts[@]}"; do
-    $script
+    execute $BASE_DIR/$script
 done
+#

-echo Ended job $TIMESTAMP..
+echo Ended job $TIMESTAMP..