Added interactive config
This commit is contained in:
parent
d2eeadba89
commit
bdc2eb09bb
|
|
@ -40,13 +40,13 @@
|
|||
<div class="bg-white dark:bg-gray-800 col-span-1 md:col-span-2 shadow-lg rounded-lg p-5">
|
||||
<h3 class="dark:text-white text-lg font-semibold mb-1"><a href="/archlinux"
|
||||
class="text-blue-500 hover:text-blue-700">ArchLinux (x86_64)</a></h3>
|
||||
<p class="text-gray-600 dark:text-gray-300">Last Updated: Not Synced</p>
|
||||
<p class="text-gray-600 dark:text-gray-300">Last Updated: 2023-10-12 11:24</p>
|
||||
<p class="text-gray-600 dark:text-gray-300">Upstream: <span class="url">rsync://mirrors.xtom.de/archlinux/</span></p>
|
||||
</div>
|
||||
<div class="bg-white dark:bg-gray-800 col-span-1 shadow-lg rounded-lg p-5">
|
||||
<h3 class="dark:text-white text-lg font-semibold mb-1"><a href="/ubuntu"
|
||||
class="text-blue-500 hover:text-blue-700">Ubuntu</a></h3>
|
||||
<p class="text-gray-600 dark:text-gray-300">Last Updated: Not Synced</p>
|
||||
<p class="text-gray-600 dark:text-gray-300">Last Updated: 2023-10-12 11:24</p>
|
||||
<p class="text-gray-600 dark:text-gray-300">Upstream: <span class="url">rsync://rsync.archive.ubuntu.com/ubuntu/</span></p>
|
||||
</div>
|
||||
<div class="bg-white dark:bg-gray-800 col-span-1 shadow-lg rounded-lg p-5">
|
||||
|
|
|
|||
|
|
@ -1,19 +1,25 @@
|
|||
BASE_DIR: /srv/mirror
|
||||
ENV: ./scripts/.env
|
||||
|
||||
# Any path under this must be in relative of BASE_DIR.
|
||||
# Script doesnt do any kind of outbound check, so make sure to check all paths.
|
||||
|
||||
global:
|
||||
scripts:
|
||||
pre:
|
||||
post:
|
||||
log: logs/sync.log
|
||||
sync:
|
||||
- ALL
|
||||
|
||||
defaults:
|
||||
type: rsync
|
||||
scripts:
|
||||
pre:
|
||||
# - ./scripts/cleanup.sh
|
||||
fail:
|
||||
- ./scripts/alert.sh
|
||||
- scripts/alert.sh
|
||||
post:
|
||||
- ./script/index.py
|
||||
# - ./script/rotate_log.py
|
||||
log: ./logs/sync.log
|
||||
log_dir: ./logs/
|
||||
sync:
|
||||
- ALL
|
||||
- scripts/index.py
|
||||
log: logs/${path}.log
|
||||
rsync:
|
||||
options: "-rtlHpv --chmod=D0755,F0644 --partial --hard-links --safe-links --stats --delete --delete-after --delay-updates --max-delete=70000"
|
||||
exclude:
|
||||
|
|
@ -25,35 +31,32 @@ global:
|
|||
- "*.links.tar.gz*"
|
||||
- "/other"
|
||||
- "/sources"
|
||||
|
||||
repos:
|
||||
ARCHLINUX:
|
||||
url: rsync://mirrors.xtom.de/archlinux/
|
||||
duration: 4
|
||||
name: "ArchLinux (x86_64)"
|
||||
path: archlinux
|
||||
log: ./logs/archlinux
|
||||
last_sync: "2023-10-09T23:48Z"
|
||||
|
||||
UBUNTU:
|
||||
url: rsync://rsync.archive.ubuntu.com/ubuntu/
|
||||
duration: 4
|
||||
name: "Ubuntu"
|
||||
path: ubuntu
|
||||
log: ./logs/ubuntu
|
||||
last_sync: "2023-10-09T23:48Z"
|
||||
|
||||
UBUNTU_CD:
|
||||
url: rsync://releases.ubuntu.com/releases/
|
||||
duration: 24
|
||||
duration: 12
|
||||
name: "Ubuntu Releases"
|
||||
path: ubuntu_cd
|
||||
log: ./logs/ubuntu_cd
|
||||
last_sync: "2023-10-09T23:48Z"
|
||||
|
||||
UBUNTU_CD_OLD:
|
||||
url: rsync://old-releases.ubuntu.com/releases/
|
||||
duration: 24
|
||||
duration: 12
|
||||
name: "Ubuntu Releases (Old)"
|
||||
path: ubuntu_cd_old
|
||||
log: ./logs/ubuntu_cd_old
|
||||
last_sync: "2023-10-09T23:48Z"
|
||||
|
||||
DEBIAN:
|
||||
url: rsync://mirrors.xtom.jp/debian/
|
||||
type: ftpsync
|
||||
|
|
@ -61,50 +64,47 @@ repos:
|
|||
name: Debian
|
||||
path: debian
|
||||
scripts:
|
||||
# pre: ./scripts/pre_debian.sh
|
||||
last_sync: "2023-10-09T23:48Z"
|
||||
# pre: scripts/pre_debian.sh
|
||||
last_sync: "2023-10-12T10:58Z"
|
||||
|
||||
DEBIAN_CD:
|
||||
url: rsync://ftp.lanet.kr/debian-cd/
|
||||
duration: 24
|
||||
duration: 12
|
||||
name: "Debian CD"
|
||||
path: debian_cd
|
||||
log: ./logs/debian_cd
|
||||
last_sync: "2023-10-09T23:48Z"
|
||||
|
||||
FEDORA:
|
||||
url: rsync://dl.fedoraproject.org/fedora-enchilada/linux/
|
||||
duration: 4
|
||||
name: "Fedora"
|
||||
path: fedora
|
||||
log: ./logs/fedora
|
||||
last_sync: "2023-10-09T23:30Z"
|
||||
|
||||
RASPBIAN:
|
||||
url: rsync://archive.raspbian.org/archive/
|
||||
duration: 6
|
||||
name: "Raspbian"
|
||||
path: raspbian
|
||||
log: ./logs/raspbian
|
||||
last_sync: "2023-10-09T23:30Z"
|
||||
|
||||
MANJARO:
|
||||
url: rsync://ftp.riken.jp/manjaro/
|
||||
duration: 4
|
||||
name: Manjaro
|
||||
path: manjaro
|
||||
log: ./logs/manjaro
|
||||
last_sync: "2023-10-09T23:30Z"
|
||||
|
||||
ARCHLINUXARM:
|
||||
name: "ArchLinux (ARM)"
|
||||
path: archlinuxarm
|
||||
url: http://jp.mirror.archlinuxarm.org/
|
||||
type: http
|
||||
duration: 12
|
||||
last_sync: "2023-10-09T23:30Z"
|
||||
|
||||
ASAHILINUX:
|
||||
name: "AsahiLinux"
|
||||
path: asahilinux
|
||||
url: https://cdn.asahilinux.org/
|
||||
type: http
|
||||
duration: 12
|
||||
last_sync: "2023-10-09T23:30Z"
|
||||
|
||||
index:
|
||||
- ARCHLINUX
|
||||
- UBUNTU UBUNTU_CD
|
||||
|
|
|
|||
|
|
@ -6,22 +6,21 @@ import jinja2
|
|||
import datetime
|
||||
|
||||
BASE_DIR = "."
|
||||
|
||||
CONFIG_PATH = os.path.join(BASE_DIR, "scripts/config.yml")
|
||||
TEMPLATES_DIR = os.path.join(BASE_DIR, "scripts/templates")
|
||||
LOG_DIR = os.path.join(BASE_DIR, "logs")
|
||||
SYNC_LOG = os.path.join(LOG_DIR, "sync.log")
|
||||
OUTPUT_PATH = os.path.join(BASE_DIR, "index.html")
|
||||
|
||||
# def get_last_sync(repo_name):
|
||||
# log_path = os.path.join(LOG_DIR, f"all.log")
|
||||
# if os.path.exists(log_path):
|
||||
# with open(log_path, 'r') as log_file:
|
||||
# log_all = reversed(log_file.readlines())
|
||||
# for logline in log_all:
|
||||
# time, stat, dist = logline.split()
|
||||
# if repo_name == dist and stat =="DONE":
|
||||
# return datetime.datetime.strptime(time, '%Y%m%d_%H%M').strftime("%Y-%m-%d %H:%M")
|
||||
# return "Not Synced"
|
||||
def get_last_sync(repo_name):
|
||||
if os.path.exists(SYNC_LOG):
|
||||
with open(SYNC_LOG, 'r') as log_file:
|
||||
log_all = reversed(log_file.readlines())
|
||||
for logline in log_all:
|
||||
dist, time, stat = logline.split()
|
||||
if repo_name == dist and stat =="SUCCESS":
|
||||
return datetime.datetime.strptime(time, '%Y-%m-%dT%H:%MZ').strftime("%Y-%m-%d %H:%M")
|
||||
return "Not Synced"
|
||||
|
||||
# main()
|
||||
|
||||
|
|
@ -51,7 +50,8 @@ if __name__=="__main__":
|
|||
context = {
|
||||
'path': repo_data['path'],
|
||||
'name': repo_data['name'],
|
||||
'lastsync': repo_data.get('lastsync', "Not Synced"),
|
||||
'lastsync': get_last_sync(repo_name),
|
||||
# 'lastsync': repo_data.get('lastsync', "Not Synced"),
|
||||
'upstream': repo_data['url']
|
||||
}
|
||||
print(context)
|
||||
|
|
|
|||
221
scripts/sync.sh
221
scripts/sync.sh
|
|
@ -1,41 +1,83 @@
|
|||
#!/bin/bash
|
||||
|
||||
CONFIG_PATH="./scripts/config.yml"
|
||||
# Default locations.
|
||||
CONFIG="./scripts/config.yml"
|
||||
YAML="./scripts/yq"
|
||||
|
||||
BASE_DIR=$($YAML eval ".BASE_DIR" "${CONFIG_PATH}")
|
||||
TIMESTAMP=$(date '+%Y-%m-%dT%H:%MZ')
|
||||
function usage() {
|
||||
cat <<EOF
|
||||
Usage: $0 [-c config_path] [-y yq_path]
|
||||
|
||||
Flags:
|
||||
-c, --config_path: config file path
|
||||
-y, --yq: yq binary path (>v4.35.2)
|
||||
-v, --verbose
|
||||
-> -b, --base-dir: overrides base_dir
|
||||
-> -d, --dry-run: dry run
|
||||
EOF
|
||||
exit 1
|
||||
}
|
||||
|
||||
function error() {
|
||||
echo $0: "$1"
|
||||
exit
|
||||
}
|
||||
|
||||
function debug() {
|
||||
if [[ -v DEBUG ]]; then
|
||||
echo debug: "$1"
|
||||
fi
|
||||
}
|
||||
|
||||
function execute() {
|
||||
debug "Executing script: \"$@\""
|
||||
if [[ ! -v DRY_RUN ]]; then
|
||||
$@
|
||||
fi
|
||||
}
|
||||
|
||||
parse_yaml() {
|
||||
echo $($YAML eval ".$1" "${CONFIG_PATH}")
|
||||
echo $($YAML eval ".$1" "${CONFIG}")
|
||||
}
|
||||
|
||||
get_repo_config() {
|
||||
local LOCAL_CFG=$($YAML eval ".repos.$1.$2" "${CONFIG}")
|
||||
if [[ "$LOCAL_CFG" == "null" || "$LOCAL_CFG" == "" ]]; then
|
||||
echo $($YAML eval ".defaults.$2" "${CONFIG}");
|
||||
return 1
|
||||
fi
|
||||
echo $LOCAL_CFG
|
||||
return 0
|
||||
|
||||
# 1 if global 0 if local
|
||||
}
|
||||
|
||||
sync_repo() {
|
||||
|
||||
local repo=$1
|
||||
local type=$(parse_yaml "repos.${repo}.type")
|
||||
local url=$(parse_yaml "repos.${repo}.url")
|
||||
local path=$(parse_yaml "repos.${repo}.path")
|
||||
local log=$(parse_yaml "repos.${repo}.log")
|
||||
local name=$(get_repo_config ${repo} name)
|
||||
local type=$(get_repo_config ${repo} type)
|
||||
local url=$(get_repo_config ${repo} url)
|
||||
local path=$(get_repo_config ${repo} path)
|
||||
local log; log=$(get_repo_config ${repo} log) || log=${log/\${path\}/$path};
|
||||
|
||||
if [[ "$type" == "null" ]]; then
|
||||
type=$(parse_yaml "global.type")
|
||||
if [[ "$type" == "null" || "$url" == "null" || "$path" == "null" || "$log" == "null" ]]; then
|
||||
echo "Error config wrong."
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ "$log" == "null" ]]; then
|
||||
log="$(parse_yaml "global.log_dir")$path"
|
||||
fi
|
||||
|
||||
log=$(realpath $log)
|
||||
path="${BASE_DIR}/$path/"
|
||||
echo -e "----\n| Repo: $repo\n| Type: $type\n| Upstream: $url\n| Path: $path\n| Log: $log\n----"
|
||||
log="${BASE_DIR}/$log"
|
||||
|
||||
echo -e "--------\nRepo: $name\nType: $type\nUpstream: $url\nPath: $path\nLog: $log\n--------"
|
||||
|
||||
if [[ ! -v DRY_RUN ]]; then
|
||||
|
||||
rotate_log $log
|
||||
|
||||
case $type in
|
||||
"rsync")
|
||||
local rsync_options=$(parse_yaml 'global.rsync.options')
|
||||
local exclude_list=($(parse_yaml 'global.rsync.exclude[]'))
|
||||
local rsync_options=$(get_repo_config $repo 'rsync.options')
|
||||
local exclude_list=($(get_repo_config $repo 'rsync.exclude[]'))
|
||||
local exclude=""
|
||||
for ex in "${exclude_list[@]}"; do
|
||||
exclude="${exclude} --exclude='${ex}'"
|
||||
|
|
@ -61,6 +103,7 @@ sync_repo() {
|
|||
esac
|
||||
|
||||
clean_log $log
|
||||
fi
|
||||
}
|
||||
|
||||
rotate_log() {
|
||||
|
|
@ -92,50 +135,158 @@ clean_log() {
|
|||
fi
|
||||
}
|
||||
|
||||
function global_log() {
|
||||
GLOBAL_LOG_FILE="$(parse_yaml 'global.log')"
|
||||
echo "$1 $TIMESTAMP SUCCESS" >> $GLOBAL_LOG_FILE
|
||||
}
|
||||
|
||||
function global_log_error() {
|
||||
GLOBAL_LOG_FILE="$(parse_yaml 'global.log')"
|
||||
echo "$1 $TIMESTAMP ERROR" >> $GLOBAL_LOG_FILE
|
||||
}
|
||||
|
||||
##########
|
||||
# Main #
|
||||
##########
|
||||
|
||||
echo Started job $TIMESTAMP..
|
||||
|
||||
global_pre_scripts=($(parse_yaml 'global.scripts.pre[]'))
|
||||
for script in "${global_pre_scripts[@]}"; do
|
||||
$script
|
||||
while [ "$1" != "" ]; do
|
||||
case $1 in
|
||||
-c | --config)
|
||||
shift
|
||||
if [[ -z $1 ]]; then
|
||||
error "option '-c' requires argument 'config_path'"
|
||||
fi
|
||||
CONFIG=$1
|
||||
;;
|
||||
-y | --yq)
|
||||
shift
|
||||
if [[ -z $1 ]]; then
|
||||
error "option '-y' requires argument 'yq_path'"
|
||||
fi
|
||||
YAML=$1
|
||||
;;
|
||||
-b | --base-dir)
|
||||
shift
|
||||
if [[ -z $1 ]]; then
|
||||
error "option '-b' requires argument 'base_dir'"
|
||||
fi
|
||||
BASE_DIR_OVERRIDE=$1
|
||||
;;
|
||||
-h | --help)
|
||||
usage
|
||||
;;
|
||||
-v | --verbose)
|
||||
DEBUG=1
|
||||
;;
|
||||
-d | --dry-run)
|
||||
DRY_RUN=1
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
if [[ -v DEBUG ]]; then
|
||||
debug "DEBUG=1"
|
||||
fi
|
||||
if [[ -v DRY_RUN ]]; then
|
||||
debug "DRY_RUN=1"
|
||||
fi
|
||||
debug CONFIG="\"${CONFIG}\""
|
||||
debug YQ="\"${YAML}\""
|
||||
|
||||
if [[ ! -f ${CONFIG} ]]; then
|
||||
error "config not found."
|
||||
fi
|
||||
|
||||
if [[ ! -f ${YAML} ]]; then
|
||||
error "yq not found."
|
||||
fi
|
||||
|
||||
BASE_DIR=$($YAML eval ".BASE_DIR" "${CONFIG}")
|
||||
|
||||
if [[ -v DEBUG && -v BASE_DIR_OVERRIDE ]]; then
|
||||
debug "Overriding $BASE_DIR to $BASE_DIR_OVERRIDE"
|
||||
BASE_DIR="$BASE_DIR_OVERRIDE"
|
||||
fi
|
||||
|
||||
TIMESTAMP=$(date '+%Y-%m-%dT%H:%MZ')
|
||||
|
||||
debug BASE_DIR="\"${BASE_DIR}\""
|
||||
debug TIMESTAMP="\"${TIMESTAMP}\""
|
||||
|
||||
echo Started job $TIMESTAMP..
|
||||
|
||||
cd $BASE_DIR
|
||||
# PRE
|
||||
global_pre_scripts=($(parse_yaml 'global.scripts.pre[]'))
|
||||
for script in "${global_pre_scripts[@]}"; do
|
||||
execute $BASE_DIR/$script
|
||||
done
|
||||
#
|
||||
repos=($(parse_yaml 'global.sync[]'))
|
||||
|
||||
if [[ "${repos[0]}" == "ALL" ]]; then
|
||||
repos=($($YAML eval '.repos | keys| .[]' "${CONFIG_PATH}"))
|
||||
repos=($($YAML eval '.repos | keys | .[]' "${CONFIG}"))
|
||||
fi
|
||||
for repo in "${repos[@]}"; do
|
||||
echo Checking $repo...
|
||||
duration=$(parse_yaml "repos.${repo}.duration")
|
||||
last_sync_timestamp=$(date -d "$(parse_yaml "repos.${repo}.last_sync")" +%s)
|
||||
cd $BASE_DIR
|
||||
debug "Checking $repo..."
|
||||
|
||||
duration=$(get_repo_config ${repo} "duration")
|
||||
last_sync_timestamp=$(date -d "$(get_repo_config ${repo} "last_sync")" +%s)
|
||||
next_sync_timestamp=$(( last_sync_timestamp + duration * 3600 ))
|
||||
|
||||
if [[ -v DEBUG ]]; then
|
||||
next_sync_timestamp=1
|
||||
# read -p "Continue? " choice
|
||||
# case "$choice" in
|
||||
# y) next_sync_timestamp=1;;
|
||||
# *) continue;;
|
||||
# esac
|
||||
fi
|
||||
|
||||
if [ $next_sync_timestamp -le $(date +%s) ]; then
|
||||
echo "Lastsync $last_sync_timestamp"
|
||||
debug "Lastsync was $last_sync_timestamp."
|
||||
echo "Syncing $repo..."
|
||||
|
||||
repo_pre_scripts=($(get_repo_config ${repo} "scripts.pre[]"))
|
||||
for script in "${repo_pre_scripts[@]}"; do
|
||||
execute $BASE_DIR/$script $repo
|
||||
done
|
||||
|
||||
sync_repo $repo
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
global_fail_scripts=($(parse_yaml 'global.scripts.fail[]'))
|
||||
for script in "${global_fail_scripts[@]}"; do
|
||||
$script
|
||||
repo_fail_scripts=($(get_repo_config ${repo} "scripts.fail[]"))
|
||||
for script in "${repo_fail_scripts[@]}"; do
|
||||
execute $BASE_DIR/$script $repo
|
||||
done
|
||||
echo "Error syncing $repo"
|
||||
|
||||
global_log_error $repo
|
||||
echo "Error during syncing $repo."
|
||||
else
|
||||
$YAML eval ".repos.${repo}.last_sync = \"$TIMESTAMP\"" -i "${CONFIG_PATH}"
|
||||
|
||||
global_log $repo
|
||||
$YAML eval ".repos.${repo}.last_sync = \"$TIMESTAMP\"" -i "${CONFIG}"
|
||||
echo "Successfully synced $repo."
|
||||
fi
|
||||
|
||||
repo_post_scripts=($(get_repo_config ${repo} "scripts.post[]"))
|
||||
for script in "${repo_post_scripts[@]}"; do
|
||||
execute $BASE_DIR/$script $repo
|
||||
done
|
||||
fi
|
||||
done
|
||||
|
||||
# POST
|
||||
global_post_scripts=($(parse_yaml 'global.scripts.post[]'))
|
||||
for script in "${global_post_scripts[@]}"; do
|
||||
$script
|
||||
execute $BASE_DIR/$script
|
||||
done
|
||||
#
|
||||
|
||||
echo Ended job $TIMESTAMP..
|
||||
Loading…
Reference in New Issue