Modified ftpsync
This commit is contained in:
parent
f93bca26ed
commit
9e5c24e082
|
|
@ -8,6 +8,8 @@ global:
|
|||
pre:
|
||||
post:
|
||||
log: logs/sync.log
|
||||
log_dir: logs/
|
||||
data_dir: pub/
|
||||
sync:
|
||||
- ALL
|
||||
|
||||
|
|
@ -18,8 +20,9 @@ defaults:
|
|||
fail:
|
||||
- scripts/alert.sh
|
||||
post:
|
||||
- scripts/index.py
|
||||
- scripts/index.py ${config}
|
||||
log: logs/${path}.log
|
||||
delay: 4
|
||||
rsync:
|
||||
options: "-rtlHpv --chmod=D0755,F0644 --partial --hard-links --safe-links --stats --delete --delete-after --delay-updates --max-delete=70000"
|
||||
exclude:
|
||||
|
|
@ -35,47 +38,44 @@ defaults:
|
|||
repos:
|
||||
ARCHLINUX:
|
||||
url: rsync://mirrors.xtom.de/archlinux/
|
||||
delay: 4
|
||||
name: "ArchLinux (x86_64)"
|
||||
path: archlinux
|
||||
|
||||
UBUNTU:
|
||||
url: rsync://rsync.archive.ubuntu.com/ubuntu/
|
||||
delay: 4
|
||||
name: "Ubuntu"
|
||||
path: ubuntu
|
||||
|
||||
UBUNTU_CD:
|
||||
url: rsync://releases.ubuntu.com/releases/
|
||||
delay: 12
|
||||
name: "Ubuntu Releases"
|
||||
delay: 6
|
||||
path: ubuntu_cd
|
||||
|
||||
UBUNTU_CD_OLD:
|
||||
url: rsync://old-releases.ubuntu.com/releases/
|
||||
delay: 12
|
||||
name: "Ubuntu Releases (Old)"
|
||||
delay: 12
|
||||
path: ubuntu_cd_old
|
||||
|
||||
DEBIAN:
|
||||
url: rsync://mirrors.xtom.jp/debian/
|
||||
url: rsync://mirrors.xtom.jp/
|
||||
type: ftpsync
|
||||
delay: 4
|
||||
name: Debian
|
||||
path: debian
|
||||
scripts:
|
||||
|
||||
# pre: scripts/pre_debian.sh
|
||||
last_sync: "2023-10-12T10:58Z"
|
||||
|
||||
DEBIAN_CD:
|
||||
url: rsync://ftp.lanet.kr/debian-cd/
|
||||
delay: 12
|
||||
delay: 6
|
||||
name: "Debian CD"
|
||||
path: debian_cd
|
||||
|
||||
FEDORA:
|
||||
url: rsync://dl.fedoraproject.org/fedora-enchilada/linux/
|
||||
delay: 4
|
||||
name: "Fedora"
|
||||
path: fedora
|
||||
|
||||
|
|
@ -87,7 +87,6 @@ repos:
|
|||
|
||||
MANJARO:
|
||||
url: rsync://ftp.riken.jp/manjaro/
|
||||
delay: 4
|
||||
name: Manjaro
|
||||
path: manjaro
|
||||
|
||||
|
|
@ -105,6 +104,18 @@ repos:
|
|||
type: http
|
||||
delay: 12
|
||||
|
||||
GNU:
|
||||
name: "GNU"
|
||||
path: gnu
|
||||
url: rsync://ftp.gnu.org/gnu/
|
||||
delay: 12
|
||||
|
||||
LINUX:
|
||||
name: "Linux Kernel"
|
||||
path: linux
|
||||
url: rsync://rsync.kernel.org/pub/
|
||||
delay: 12
|
||||
|
||||
index:
|
||||
- ARCHLINUX
|
||||
- UBUNTU UBUNTU_CD
|
||||
|
|
@ -115,4 +126,6 @@ index:
|
|||
- DIVIDER
|
||||
- ARCHLINUXARM
|
||||
- ASAHILINUX
|
||||
- GNU
|
||||
- LINUX
|
||||
- UBUNTU_CD_OLD
|
||||
|
|
@ -1,12 +1,12 @@
|
|||
MIRRORNAME="pawe.me"
|
||||
TO="${BASE_DIR}/debian"
|
||||
MIRRORNAME="${DEB_HOST}"
|
||||
TO="${DEB_PATH}"
|
||||
HUB=false
|
||||
RSYNC_HOST="mirrors.xtom.jp"
|
||||
RSYNC_HOST="${DEB_UPSTREAM}"
|
||||
RSYNC_PATH="debian"
|
||||
INFO_MAINTAINER="Morgan <admin@pawe.me>"
|
||||
INFO_COUNTRY=KR
|
||||
INFO_LOCATION="Seoul, Korea"
|
||||
INFO_THROUGHPUT=500Mb
|
||||
LOGDIR="${BASE_DIR}/logs"
|
||||
INFO_MAINTAINER="${DEB_ADMIN}"
|
||||
INFO_COUNTRY="${DEB_COUNTRY}"
|
||||
INFO_LOCATION="${DEB_LOCATION}"
|
||||
INFO_THROUGHPUT=1Gbps
|
||||
LOGDIR="${DEB_LOG}"
|
||||
UIPRETRIES=9
|
||||
TRACEHOST="PAWE.ME"
|
||||
TRACEHOST="${DEB_HOST}"
|
||||
|
|
@ -5,29 +5,35 @@ import yaml
|
|||
import jinja2
|
||||
import datetime
|
||||
|
||||
BASE_DIR = "."
|
||||
CONFIG_PATH = os.path.join(BASE_DIR, "scripts/config.yml")
|
||||
TEMPLATES_DIR = os.path.join(BASE_DIR, "scripts/templates")
|
||||
LOG_DIR = os.path.join(BASE_DIR, "logs")
|
||||
SYNC_LOG = os.path.join(LOG_DIR, "sync.log")
|
||||
OUTPUT_PATH = os.path.join(BASE_DIR, "index.html")
|
||||
|
||||
def get_last_sync(repo_name):
|
||||
if os.path.exists(SYNC_LOG):
|
||||
with open(SYNC_LOG, 'r') as log_file:
|
||||
log_all = reversed(log_file.readlines())
|
||||
for logline in log_all:
|
||||
dist, time, stat = logline.split()
|
||||
if repo_name == dist and stat =="SUCCESS":
|
||||
return datetime.datetime.strptime(time, '%Y-%m-%dT%H:%MZ').strftime("%Y-%m-%d %H:%M")
|
||||
return "Not Synced"
|
||||
# def get_last_sync(repo_name):
|
||||
# if os.path.exists(SYNC_LOG):
|
||||
# with open(SYNC_LOG, 'r') as log_file:
|
||||
# log_all = reversed(log_file.readlines())
|
||||
# for logline in log_all:
|
||||
# dist, time, stat = logline.split()
|
||||
# if repo_name == dist and stat =="SUCCESS":
|
||||
# return datetime.datetime.strptime(time, '%Y-%m-%dT%H:%MZ').strftime("%Y-%m-%d %H:%M")
|
||||
# return "Not Synced"
|
||||
|
||||
# main()
|
||||
|
||||
if __name__=="__main__":
|
||||
|
||||
if len(sys,argv) == 3:
|
||||
print(sys.argv)
|
||||
|
||||
CONFIG_PATH = sys.argv[1]
|
||||
|
||||
with open(CONFIG_PATH, 'r') as f:
|
||||
config = yaml.safe_load(f)
|
||||
|
||||
|
||||
BASE_DIR = config.get("BASE_DIR", ".")
|
||||
THEME = "new"
|
||||
TEMPLATES_DIR = os.path.join(os.path.join(BASE_DIR, "scripts/templates"), THEME)
|
||||
LOG_DIR = os.path.join(BASE_DIR, config['global'].get("log_dir", "logs"))
|
||||
OUTPUT_PATH = os.path.join(os.path.join(BASE_DIR, config['global'].get("data_dir", ".")), "index.html")
|
||||
|
||||
env = jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATES_DIR))
|
||||
base_template = env.get_template('base.html')
|
||||
full_template = env.get_template('full.html')
|
||||
|
|
@ -50,8 +56,8 @@ if __name__=="__main__":
|
|||
context = {
|
||||
'path': repo_data['path'],
|
||||
'name': repo_data['name'],
|
||||
'lastsync': get_last_sync(repo_name),
|
||||
# 'lastsync': repo_data.get('lastsync', "Not Synced"),
|
||||
# 'lastsync': get_last_sync(repo_name),
|
||||
'lastsync': repo_data.get('lastsync', "Not Synced"),
|
||||
'upstream': repo_data['url']
|
||||
}
|
||||
print(context)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,91 @@
|
|||
user user;
|
||||
worker_processes auto;
|
||||
pid /run/nginx.pid;
|
||||
include /etc/nginx/modules-enabled/*.conf;
|
||||
|
||||
events {
|
||||
worker_connections 768;
|
||||
# multi_accept on;
|
||||
}
|
||||
|
||||
http {
|
||||
|
||||
##
|
||||
# Basic Settings
|
||||
##
|
||||
|
||||
sendfile on;
|
||||
tcp_nopush on;
|
||||
types_hash_max_size 2048;
|
||||
# server_tokens off;
|
||||
|
||||
# server_names_hash_bucket_size 64;
|
||||
# server_name_in_redirect off;
|
||||
|
||||
include /etc/nginx/mime.types;
|
||||
default_type application/octet-stream;
|
||||
|
||||
##
|
||||
# SSL Settings
|
||||
##
|
||||
|
||||
ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; # Dropping SSLv3, ref: POODLE
|
||||
ssl_prefer_server_ciphers on;
|
||||
|
||||
##
|
||||
# Logging Settings
|
||||
##
|
||||
|
||||
log_format main_ext '$remote_addr - $remote_user [$time_local] "$request" '
|
||||
'$status $body_bytes_sent "$http_referer" '
|
||||
'"$http_user_agent" "$http_x_forwarded_for" '
|
||||
'"$host" sn="$server_name" ' 'rt=$request_time '
|
||||
'ua="$upstream_addr" us="$upstream_status" '
|
||||
'ut="$upstream_response_time" ul="$upstream_response_length" '
|
||||
'cs=$upstream_cache_status' ;
|
||||
|
||||
access_log /var/log/nginx/access.log;
|
||||
error_log /var/log/nginx/error.log;
|
||||
|
||||
##
|
||||
# Gzip Settings
|
||||
##
|
||||
|
||||
gzip on;
|
||||
|
||||
# gzip_vary on;
|
||||
# gzip_proxied any;
|
||||
# gzip_comp_level 6;
|
||||
# gzip_buffers 16 8k;
|
||||
# gzip_http_version 1.1;
|
||||
# gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
|
||||
|
||||
##
|
||||
# Virtual Host Configs
|
||||
##
|
||||
|
||||
include /etc/nginx/conf.d/*.conf;
|
||||
include /etc/nginx/sites-enabled/*;
|
||||
}
|
||||
|
||||
|
||||
#mail {
|
||||
# # See sample authentication script at:
|
||||
# # http://wiki.nginx.org/ImapAuthenticateWithApachePhpScript
|
||||
#
|
||||
# # auth_http localhost/auth.php;
|
||||
# # pop3_capabilities "TOP" "USER";
|
||||
# # imap_capabilities "IMAP4rev1" "UIDPLUS";
|
||||
#
|
||||
# server {
|
||||
# listen localhost:110;
|
||||
# protocol pop3;
|
||||
# proxy on;
|
||||
# }
|
||||
#
|
||||
# server {
|
||||
# listen localhost:143;
|
||||
# protocol imap;
|
||||
# proxy on;
|
||||
# }
|
||||
#}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
server {
|
||||
listen 80 ;
|
||||
listen [::]:80 ;
|
||||
server_name repo.city;
|
||||
|
||||
access_log /var/log/nginx/repo-city-access.log main_ext;
|
||||
error_log /var/log/nginx/repo-city-error.log warn;
|
||||
|
||||
root /srv/mirror;
|
||||
|
||||
location /scripts/ {
|
||||
return 404;
|
||||
}
|
||||
|
||||
location / {
|
||||
autoindex on;
|
||||
fancyindex on;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
server {
|
||||
listen 127.0.0.1:80;
|
||||
server_name 127.0.0.1;
|
||||
location /nginx_status {
|
||||
stub_status on;
|
||||
access_log off;
|
||||
allow 127.0.0.1;
|
||||
deny all;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -65,10 +65,10 @@ sync_repo() {
|
|||
return
|
||||
fi
|
||||
|
||||
path="${BASE_DIR}/$path/"
|
||||
full_path="${BASE_DIR}/$(parse_yaml global.data_dir)/$path/"
|
||||
log="${BASE_DIR}/$log"
|
||||
|
||||
echo -e "--------\nRepo: $name\nType: $type\nUpstream: $url\nPath: $path\nLog: $log\n--------"
|
||||
echo -e "--------\nRepo: $name\nType: $type\nUpstream: $url\nPath: $full_path\nLog: $log\n--------"
|
||||
|
||||
if [[ ! -v DRY_RUN ]]; then
|
||||
|
||||
|
|
@ -82,18 +82,26 @@ sync_repo() {
|
|||
for ex in "${exclude_list[@]}"; do
|
||||
exclude="${exclude} --exclude='${ex}'"
|
||||
done
|
||||
echo rsync ${rsync_options} ${exclude} $url $path >> $log
|
||||
rsync ${rsync_options} ${exclude} $url $path >> $log 2>> ${log}-error
|
||||
echo rsync ${rsync_options} ${exclude} $url $full_path >> $log
|
||||
rsync ${rsync_options} ${exclude} $url $full_path >> $log 2>> ${log}-error
|
||||
;;
|
||||
"ftpsync")
|
||||
cd ${BASE_DIR}/scripts
|
||||
export BASE_DIR=${BASE_DIR}
|
||||
|
||||
export DEB_ADMIN="Morgan <mirror@devpg.net>"
|
||||
export DEB_COUNTRY="Korea"
|
||||
export DEB_LOCATION="Korea/Seoul"
|
||||
export DEB_PATH="$full_path"
|
||||
export DEB_HOST="$(parse_yaml global.hostname)"
|
||||
export DEB_UPSTREAM="$url"
|
||||
export DEB_LOG="${BASE_DIR}/$(parse_yaml global.log_dir)"
|
||||
|
||||
./ftpsync >> $log 2>> ${log}-error
|
||||
cd ${BASE_DIR}
|
||||
;;
|
||||
"http")
|
||||
echo ${repo} Fetch >> $log 2>> ${log}-error
|
||||
python3 -u $BASE_DIR/scripts/getFetch.py "${url}" $path $BASE_DIR/scripts/${path}.fetch >> $log 2>> ${log}-error
|
||||
python3 -u $BASE_DIR/scripts/getFetch.py "${url}" $full_path $BASE_DIR/scripts/${path}.fetch >> $log 2>> ${log}-error
|
||||
echo ${repo} Download >> $log 2>> ${log}-error
|
||||
python3 -u $BASE_DIR/scripts/getFile.py $BASE_DIR/scripts/${path}.fetch >> $log 2>> ${log}-error
|
||||
;;
|
||||
|
|
@ -277,6 +285,7 @@ for repo in "${repos[@]}"; do
|
|||
|
||||
repo_post_scripts=($(get_repo_config ${repo} "scripts.post[]"))
|
||||
for script in "${repo_post_scripts[@]}"; do
|
||||
script=${script/\${config\}/$CONFIG}
|
||||
execute $BASE_DIR/$script $repo
|
||||
done
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -0,0 +1,99 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>repo.city</title>
|
||||
<link rel="icon" type="image/x-icon" href="favicon.ico">
|
||||
<link href="https://fonts.googleapis.com/css2?family=Noto+Sans:wght@300;400;500;600;700;800;900&display=swap"
|
||||
rel="stylesheet">
|
||||
<script src="https://cdn.tailwindcss.com"></script>
|
||||
<script>
|
||||
tailwind.config = {
|
||||
darkMode: 'class'
|
||||
}
|
||||
// if (localStorage.theme === 'dark' || (!('theme' in localStorage) && window.matchMedia('(prefers-color-scheme: dark)').matches)) {
|
||||
// document.documentElement.classList.add('dark')
|
||||
if (localStorage.theme === 'dark') {
|
||||
document.documentElement.classList.add('dark')
|
||||
} else {
|
||||
document.documentElement.classList.remove('dark')
|
||||
}
|
||||
</script>
|
||||
<style>
|
||||
.url {
|
||||
font-style: italic;
|
||||
color: rgb(86, 105, 151);
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
|
||||
<body class="font-sans text-gray-900 bg-gray-100 py-12 px-6 mx-auto max-w-5xl dark:bg-gray-900">
|
||||
<header class="mb-10 ml-2">
|
||||
<h1 class="text-3xl font-bold mb-2 text-black dark:text-white">Welcome to <a href="https://repo.city"
|
||||
class="text-blue-900 dark:text-blue-300 font-bold">REPO.CITY</a>!</h1>
|
||||
<h2 class="text-xl text-gray-600 dark:text-gray-300">Simple mirroring & archiving server.</h2>
|
||||
</header>
|
||||
<main>
|
||||
<section class="mb-12">
|
||||
<h2 class="dark:text-gray-200 text-2xl font-semibold mb-5 ml-2">Mirroring List</h2>
|
||||
<div id="mirrorList" class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-2 gap-4">
|
||||
|
||||
{{ repos }}
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
</section>
|
||||
<section class="mb-12">
|
||||
<h2 class="dark:text-gray-200 text-2xl font-semibold mb-5 ml-2">Additional Mirror</h2>
|
||||
<!-- <button onclick="toggleView('httpView')" class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded mb-4">View More</button> -->
|
||||
<div id="httpView" class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-2 gap-4">
|
||||
|
||||
{{ repos_more }}
|
||||
|
||||
<div class="bg-white dark:bg-gray-800 col-span-1 md:col-span-2 shadow-lg rounded-lg p-5">
|
||||
<h3 class="dark:text-white text-lg font-semibold mb-1"><a href="/cd-image"
|
||||
class="text-blue-900 hover:text-blue-600 dark:text-blue-600 dark:hover:text-blue-400">Image Files</a></h3>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
<section class="ml-2">
|
||||
<h2 class="dark:text-gray-200 text-2xl font-semibold mb-4">Server Details</h2>
|
||||
<p class="mb-2 text-gray-800 dark:text-gray-500"><strong>Location:</strong> Seoul, Korea</p>
|
||||
<p class="mb-2 text-gray-800 dark:text-gray-500"><strong>Bandwidth:</strong> ~1Gbps</p>
|
||||
<p class="mb-2 text-gray-800 dark:text-gray-500"><strong>Storage size:</strong> 32TiB</p>
|
||||
<p class="mb-2 text-gray-800 dark:text-gray-500"><strong>Sync delay:</strong> 4hrs</p>
|
||||
<p class="mb-2 text-gray-800 dark:text-gray-500"><strong>Contact:</strong> <a href="mailto:mirror@devpg.net">mirror@devpg.net</a></p>
|
||||
<p class="mb-2 text-gray-800 dark:text-gray-500"><strong>Maintainer:</strong> <a href="https://github.com/morgan9e">@morgan9e</a></p>
|
||||
<p class="text-gray-500 dark:text-gray-600 url">This mirror is based on <a href="https://github.com/morgan9e/reposync" class="text-blue-500 dark:text-blue-800">reposync</a>.</p>
|
||||
<p class="text-gray-500 dark:text-gray-600 url">For additional mirroring requests, please contact me at email above.</p>
|
||||
</section>
|
||||
<button
|
||||
class="fixed bottom-6 right-6 bg-blue-200 hover:bg-blue-300 text-blue-700 dark:bg-blue-800 dark:hover:bg-blue-600 dark:text-white w-10 h-10 rounded-full flex items-center justify-center"
|
||||
onclick="toggleDarkMode()">
|
||||
+
|
||||
</button>
|
||||
</main>
|
||||
|
||||
<footer class="mt-10 text-center text-gray-600">
|
||||
<a href="mailto:mirror@devpg.net">© REPO.CITY</a>
|
||||
</footer>
|
||||
|
||||
<script>
|
||||
function toggleView(elementId) {
|
||||
const element = document.getElementById(elementId);
|
||||
element.style.display = element.style.display === "none" ? "block" : "none";
|
||||
}
|
||||
function toggleDarkMode() {
|
||||
if (document.documentElement.classList.contains('dark')) {
|
||||
document.documentElement.classList.remove('dark');
|
||||
localStorage.theme = 'light';
|
||||
} else {
|
||||
document.documentElement.classList.add('dark');
|
||||
localStorage.theme = 'dark';
|
||||
}
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
<div class="bg-white dark:bg-gray-800 col-span-1 md:col-span-2 shadow-lg rounded-lg p-5">
|
||||
<h3 class="dark:text-white text-lg font-semibold mb-1"><a href="/{{path}}"
|
||||
class="text-blue-900 hover:text-blue-600 dark:text-blue-600 dark:hover:text-blue-400">{{name}}</a></h3>
|
||||
<p class="text-gray-600 dark:text-gray-300">Last Updated: {{lastsync}}</p>
|
||||
<p class="text-gray-600 dark:text-gray-300">Upstream: <span class="url">{{upstream}}</span></p>
|
||||
</div>
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
<div class="bg-white dark:bg-gray-800 col-span-1 shadow-lg rounded-lg p-5">
|
||||
<h3 class="dark:text-white text-lg font-semibold mb-1"><a href="/{{path}}"
|
||||
class="text-blue-900 hover:text-blue-600 dark:text-blue-600 dark:hover:text-blue-400">{{name}}</a></h3>
|
||||
<p class="text-gray-600 dark:text-gray-300">Last Updated: {{lastsync}}</p>
|
||||
<p class="text-gray-600 dark:text-gray-300">Upstream: <span class="url">{{upstream}}</span></p>
|
||||
</div>
|
||||
Loading…
Reference in New Issue