mirror of
https://github.com/morgan9e/mirror
synced 2026-04-14 00:14:15 +09:00
Init
This commit is contained in:
GitHub
105
base.html
Normal file
105
base.html
Normal file
@@ -0,0 +1,105 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Mirror Archive</title>
|
||||
<style>
|
||||
body {
|
||||
font-size: 1.1em;
|
||||
margin: 0;
|
||||
margin-top: 6em;
|
||||
padding: 0;
|
||||
}
|
||||
table, th, td {
|
||||
border: 0px;
|
||||
border-collapse: collapse;
|
||||
}
|
||||
th + th, td + td {
|
||||
padding: 6px 16px;
|
||||
}
|
||||
th, td {
|
||||
text-align: left;
|
||||
vertical-align: top;
|
||||
}
|
||||
h2, h3 {
|
||||
margin-bottom: 0.4em;
|
||||
}
|
||||
footer {
|
||||
margin-top: 1.5em;
|
||||
font-size: 0.75em;
|
||||
}
|
||||
.container {
|
||||
width: 80%;
|
||||
max-width: 800px;
|
||||
margin: 0 auto;
|
||||
padding: 0 20px;
|
||||
}
|
||||
.details {
|
||||
line-height: 1.8;
|
||||
}
|
||||
.subtitle {
|
||||
font-style: italic;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<header>
|
||||
<h2>Mirror Archive</h2>
|
||||
</header>
|
||||
<main>
|
||||
<div class="mirrorList">
|
||||
<h3>Archive List</h3>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>Last Updated (UTC)</th>
|
||||
<th>Sync Time</th>
|
||||
<th>Source</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><a href="/archlinux">/archlinux</a></td>
|
||||
<td>@@archlinux@@</td>
|
||||
<td>rsync://mirror.constant.com/archlinux</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a href="/debian">/debian</a></td>
|
||||
<td>@@debian@@</td>
|
||||
<td>rsync://mirrors.xtom.com/debian</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a href="/debian-cd">/debian-cd</a></td>
|
||||
<td>@@debian_cd@@</td>
|
||||
<td>rsync://mirrors.xtom.com/debian-cd</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a href="/fedora">/fedora</a></td>
|
||||
<td>@@fedora@@</td>
|
||||
<td>rsync://dl.fedoraproject.org/fedora-enchilada</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a href="/rpmfusion">/rpmfusion</a></td>
|
||||
<td>@@rpmfusion@@</td>
|
||||
<td>rsync://download1.rpmfusion.org/rpmfusion</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="details">
|
||||
<h3>Server Details</h3>
|
||||
<strong>Location:</strong> Seoul, Korea<br>
|
||||
<strong>Bandwidth:</strong> 1 Gbps ~ 100 Mbps<br>
|
||||
<strong>Sync delay:</strong> ~ 4 hours <br>
|
||||
<strong>Contact:</strong> <a href="">mirror[at]devpg[d0t]net</a><br>
|
||||
</div>
|
||||
<footer>
|
||||
Maintained by Av from Korea <span style="font-size: 0.65em">🏳️🌈🏳️⚧️</span>
|
||||
</footer>
|
||||
</main>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
54
dosync.sh
Normal file
54
dosync.sh
Normal file
@@ -0,0 +1,54 @@
|
||||
#!/bin/bash
|
||||
|
||||
TIMENOW=$(date '+%Y-%m-%d_%H:%M:%S');
|
||||
LOCKFILE="/srv/mirror/scripts/.lock";
|
||||
WAITLIMIT=30;
|
||||
LOGFILE="/srv/mirror/logs/current.log"
|
||||
SYNCLOG="/srv/mirror/logs/master.log"
|
||||
|
||||
cleanup() {
|
||||
echo "Cleaning up...";
|
||||
rm -f "$LOCKFILE";
|
||||
exit;
|
||||
}
|
||||
|
||||
trap cleanup INT TERM KILL EXIT
|
||||
|
||||
if [ -f $LOCKFILE ]; then
|
||||
echo "Previous session $(cat $LOCKFILE) exists." | tee -a ${SYNCLOG}
|
||||
while [ $WAITLIMIT -ne 0 ]; do
|
||||
echo "Waiting for previous session.. ${WAITLIMIT} tries left.." | tee -a ${SYNCLOG};
|
||||
sleep 60;
|
||||
WAITLIMIT=$(($WAITLIMIT-1));
|
||||
if [ ! -f $LOCKFILE ]; then
|
||||
break;
|
||||
fi
|
||||
done
|
||||
echo "Sync job at $TIMENOW blocked by previous job $(cat $LOCKFILE)." | tee -a ${SYNCLOG} >(/srv/mirror/scripts/alert.sh $(cat <&0))
|
||||
exit;
|
||||
fi
|
||||
|
||||
echo $$ > $LOCKFILE;
|
||||
|
||||
echo "Starting sync job at $TIMENOW, $$" | tee ${LOGFILE} | tee -a ${SYNCLOG}
|
||||
|
||||
{
|
||||
/srv/mirror/scripts/sync.sh archlinux &
|
||||
/srv/mirror/scripts/sync.sh debian &
|
||||
/srv/mirror/scripts/sync.sh rpmfusion &
|
||||
wait;
|
||||
|
||||
# /srv/mirror/scripts/sync.sh ubuntu_cd &
|
||||
# /srv/mirror/scripts/sync.sh ubuntu_cd_old &
|
||||
# /srv/mirror/scripts/sync.sh debian_cd &
|
||||
# wait;
|
||||
|
||||
# /srv/mirror/scripts/sync.sh linux &
|
||||
# wait;
|
||||
|
||||
} 2>&1 | tee -a ${LOGFILE} | tee -a ${SYNCLOG}
|
||||
|
||||
wait;
|
||||
echo Finished sync job $$ $TIMENOW at $(date '+%Y-%m-%d_%H:%M:%S') | tee -a ${LOGFILE} | tee -a ${SYNCLOG}
|
||||
|
||||
rm $LOCKFILE;
|
||||
52
ftpsync.conf
Normal file
52
ftpsync.conf
Normal file
@@ -0,0 +1,52 @@
|
||||
########################################################################
|
||||
########################################################################
|
||||
## This is a sample configuration file for the ftpsync mirror script. ##
|
||||
## Only options most users may need are included. For documentation ##
|
||||
## and all available options see ftpsync.conf(5). ##
|
||||
########################################################################
|
||||
########################################################################
|
||||
|
||||
BASE_DIR=${BASE_DIR-"/srv/mirror"}
|
||||
DATA_DIR=${DATA_DIR-"/srv/mirror/pub"}
|
||||
|
||||
echo $BASE_DIR $DATA_DIR
|
||||
|
||||
MIRRORNAME="mirror.morgan.kr"
|
||||
TO="${DATA_DIR}/debian"
|
||||
# MAILTO="$LOGNAME"
|
||||
HUB=false
|
||||
|
||||
########################################################################
|
||||
## Connection options
|
||||
########################################################################
|
||||
|
||||
RSYNC_HOST="mirrors.xtom.com"
|
||||
RSYNC_PATH="debian"
|
||||
# RSYNC_USER=
|
||||
# RSYNC_PASSWORD=
|
||||
|
||||
########################################################################
|
||||
## Mirror information options
|
||||
########################################################################
|
||||
|
||||
INFO_MAINTAINER="Morgan <mirror@morgan.kr>"
|
||||
# INFO_SPONSOR="Example <https://example.com>"
|
||||
INFO_COUNTRY=KR
|
||||
INFO_LOCATION="Seoul, Korea"
|
||||
INFO_THROUGHPUT=1Gb
|
||||
|
||||
########################################################################
|
||||
## Include and exclude options
|
||||
########################################################################
|
||||
|
||||
# ARCH_INCLUDE="amd64 arm64 riscv64 source"
|
||||
# ARCH_EXCLUDE=
|
||||
|
||||
########################################################################
|
||||
## Log option
|
||||
########################################################################
|
||||
|
||||
LOGDIR="${BASE_DIR}/logs"
|
||||
|
||||
UIPRETRIES=9
|
||||
TRACEHOST="mirror.morgan.kr"
|
||||
90
index.py
Normal file
90
index.py
Normal file
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
import re
|
||||
import datetime
|
||||
|
||||
base_path = sys.argv[1]
|
||||
assert os.path.exists(base_path)
|
||||
|
||||
out_path = os.path.join(base_path, "pub/")
|
||||
log_path = os.path.join(base_path, "logs/all.log")
|
||||
template = os.path.join(base_path, "scripts/base.html")
|
||||
index = os.path.join(out_path, "index.html")
|
||||
|
||||
assert os.path.exists(log_path)
|
||||
assert os.path.exists(template)
|
||||
|
||||
with open(log_path, 'r') as f:
|
||||
log = f.read().splitlines()
|
||||
log.reverse()
|
||||
|
||||
with open(template, 'r') as f:
|
||||
html_file = f.read()
|
||||
|
||||
target_dists = re.findall("@@([^@@]+)@@", html_file)
|
||||
source_dists = set(line.split(" ")[2] for line in log)
|
||||
|
||||
print(target_dists)
|
||||
|
||||
status = {k: [[0, 0], 0, 0] for k in target_dists}
|
||||
|
||||
for target in target_dists:
|
||||
for line in log:
|
||||
time, stat, dist = (spl := line.split(" "))[:3]
|
||||
time = datetime.datetime.strptime(time, '%Y%m%d_%H%M')
|
||||
|
||||
if dist == target:
|
||||
if stat == "DONE":
|
||||
if not status[target][0][0]:
|
||||
status[target][0][0] = time
|
||||
if len(spl) == 5:
|
||||
status[target][0][1] = spl[4]
|
||||
|
||||
elif stat == "ERROR":
|
||||
if not status[target][1]:
|
||||
status[target][1] = time
|
||||
|
||||
elif stat == "STARTED":
|
||||
if not status[target][2]:
|
||||
status[target][2] = time
|
||||
|
||||
if status[target][0][0]:
|
||||
break
|
||||
|
||||
print(status)
|
||||
|
||||
error_tmpl = "<br><span style='color:red;font-size:12px'>{}</span>"
|
||||
|
||||
for target in status:
|
||||
# if status[target][1] exists, recently errored
|
||||
done = status[target][0]
|
||||
error = status[target][1]
|
||||
running = status[target][2]
|
||||
|
||||
HTML = done[0].strftime("%Y-%m-%d %H:00") if done[0] else "Not Synced"
|
||||
|
||||
if error and (error > running):
|
||||
HTML += error_tmpl.format(f'ERR {error.strftime("%Y-%m-%d %H:%M")}')
|
||||
|
||||
if running:
|
||||
HTML += error_tmpl.format(f'RUN {running.strftime("%Y-%m-%d %H:%M")}')
|
||||
|
||||
HTML += "</td><td>"
|
||||
|
||||
def sec_to_hum(sec):
|
||||
if sec >= 60:
|
||||
if sec >= 3600:
|
||||
return f"{sec//3600}h {(sec%3600)//60}m"
|
||||
return f"{sec//60}m {sec%60}s"
|
||||
return f"{sec}s"
|
||||
|
||||
HTML += sec_to_hum(int(done[1])) if done[1] else ""
|
||||
|
||||
html_file = html_file.replace(f"@@{target}@@", HTML)
|
||||
|
||||
with open(index, 'w') as f:
|
||||
f.write(html_file)
|
||||
|
||||
print("Written to index.html")
|
||||
1761
quick-fedora-mirror
Normal file
1761
quick-fedora-mirror
Normal file
File diff suppressed because it is too large
Load Diff
192
quick-fedora-mirror.conf
Normal file
192
quick-fedora-mirror.conf
Normal file
@@ -0,0 +1,192 @@
|
||||
# Configuration file for quick-fedora-mirror
|
||||
# This file is sourced by the shell and must be in valid sh syntax.
|
||||
|
||||
#### Required settings
|
||||
# Required: The the directory holding your copy of all of the modules you
|
||||
# mirror. Does not include any module name
|
||||
DESTD=/srv/mirror/pub/
|
||||
|
||||
# Required: The file in which to store the last mirror time.
|
||||
# Note: this really should not be in the repository itself.
|
||||
TIMEFILE=/srv/mirror/logs/qfs_lastsync
|
||||
|
||||
#### Settings Involving How and What to Mirror
|
||||
# The remote host to rsync from, not including a module name
|
||||
REMOTE=rsync://dl.fedoraproject.org
|
||||
|
||||
# The master module, which holds the other modules
|
||||
# Note that the mirror you pull from MUST have this master module. You cannot
|
||||
# use quick-fedora-mirror to pull from a host which does not have a master
|
||||
# module containing the other modules.
|
||||
MASTERMODULE=fedora-buffet
|
||||
|
||||
# Tier 1 Fedora mirrors should uncomment the following to get the proper
|
||||
# pre-bitflip content.
|
||||
# MASTERMODULE=fedora-buffet0
|
||||
# PREBITFLIP=1
|
||||
|
||||
# Define if the entire repository (all modules under fedora-buffet) should be
|
||||
# mirrored. If set, MODULES (below) is ignored
|
||||
# MIRRORBUFFET=
|
||||
|
||||
# An array containing the modules to be mirrored
|
||||
MODULES=(fedora-enchilada)
|
||||
|
||||
#### Filtering
|
||||
# A regular expression used to filter the file lists. It must be quoted (or
|
||||
# very carefully escaped). Entries matching this expression will not be synced
|
||||
# and are expected not to be present locally. They will also be ignored by
|
||||
# quick-fedora-hardlink. Cannot contain commas. Run against the file list that
|
||||
# includes sizes (by quick-fedora-mirror) and the fullfiletimelist (by
|
||||
# quick-fedora-hardlink), so don't use expressions that would match those
|
||||
# metadata (which are digit strings and single characters). Example is a heavy
|
||||
# filter which gives you an x86_64-only mirror with source packages, debuginfo
|
||||
# packages, Alpha and Beta releases, and most image files excluded.
|
||||
#FILTEREXP='(/i386|/armhfp|/source|/SRPMS|/debug/|\.iso|\.img|\.qcow2|\.raw\.xz|\.box|/releases/test)'
|
||||
FILTEREXP='(/i386|/armhfp|\.img|\.raw\.xz|\.box|/releases/test)'
|
||||
|
||||
#### Email
|
||||
# If you wish to receive email notices of transfers you can call this script
|
||||
# via cron, or configure the following. Note that email won't be sent if
|
||||
# quick-fedora-mirror is run interactively.
|
||||
|
||||
# Address to receive status email
|
||||
# EMAILDEST=
|
||||
|
||||
# Subject to use for messages
|
||||
# EMAILSUBJECT=
|
||||
|
||||
|
||||
#### Logging
|
||||
# Verbosity levels - info sent to stdout; usually this gets mailed to root when
|
||||
# being run by cron
|
||||
# 0 - quiet
|
||||
# 1 - quiet if no changes, otherwise basic transfer info
|
||||
# 2 - no -q to rsync
|
||||
# 3 - very slightly more verbosity
|
||||
# 4 - One -v to rsync
|
||||
# 5 - Another -v to rsync
|
||||
# 6 - Output of all settings
|
||||
# 7 - Add --progress to rsync
|
||||
# 8 - Shell script trace
|
||||
VERBOSE=5
|
||||
|
||||
# Define (to anything) to enable logging to the systemd journal (via
|
||||
# systemd-cat). the identifier "quick-fedora-mirror" is used, so logs can be
|
||||
# retrieved with: journalctl -t quick-fedora-mirror
|
||||
# LOGJOURNAL=
|
||||
|
||||
# Define to a full path to enable logging to that file.
|
||||
# The provided file must already exist and be writable.
|
||||
# Is only considered if LOGJOURNAL above is not defined.
|
||||
LOGFILE=/srv/mirror/logs/fedora-qs.log
|
||||
|
||||
# Logging fields - Each character selects a piece of information to log.
|
||||
# @ - Absolutely everything.
|
||||
# a - aborted run recovery
|
||||
# A - each recovered file from an aborted run
|
||||
# c - rsync calls
|
||||
# C - rsync call completions
|
||||
# d - File/directory deletion start/end
|
||||
# D - all file/directory deletes
|
||||
# e - minor errors
|
||||
# E - serious errors
|
||||
# F - all transferred files (not impl)
|
||||
# g - file list generation start/end
|
||||
# k - lock contention
|
||||
# l - per-module local file list generation (recursive find) start/end
|
||||
# L - file list generation counts
|
||||
# m - mirrormanager checkin
|
||||
# M - mirrormanager checkin detail
|
||||
# n - lack of updates in a run
|
||||
# N - lack of updates in a module
|
||||
# o - remote file list download start/end
|
||||
# p - module processing start/end
|
||||
# p - per-module module processing start/end
|
||||
# r - run start
|
||||
# R - run end
|
||||
# s - Basic transfer statistics
|
||||
# S - Detailed transfer statistics
|
||||
# t - directory time updates (not impl)
|
||||
# LOGITEMS=aeElrRs
|
||||
|
||||
#### Tuneables
|
||||
# Rsync timeout value, in seconds
|
||||
# Will be passed to rsync via --timeout.
|
||||
RSYNCTIMEOUT=600
|
||||
|
||||
# In some situations rsync may experience a bug which causes it to emit lines like the following:
|
||||
# WARNING: foo failed verification -- update put into partial-dir (will try again).
|
||||
# These will break transfers and will not go away. Define
|
||||
# RSYNC_PARTIAL_DIR_BUG (to anything) to work around this.
|
||||
# RSYNC_PARTIAL_DIR_BUG=
|
||||
|
||||
# When q-f-m fails to run becuse it is already running, it checks the time
|
||||
# since the last successful run. If that is larger than this value, it logs a
|
||||
# serious error. Thid helps to detect a hung run or issues with slow
|
||||
# transfers.
|
||||
# WARNDELAY=$((60 * 60 * 24)) # One day
|
||||
|
||||
# When q-f-m encounters an error calling rsync, it may (depending on the error)
|
||||
# sleep and retry. It will always sleep with exponential backup. Set
|
||||
# MAXRETRIES to limit the number of times it retries.
|
||||
MAXRETRIES=10
|
||||
|
||||
#### Important file and directory locations
|
||||
# curl binary (only if MirrorManager checkins are enabled; see below).
|
||||
# CURL=/usr/bin/curl
|
||||
|
||||
# rsync binary
|
||||
# RSYNC=/usr/bin/rsync
|
||||
|
||||
# mktemp will be called after this file is sourced to make a temporary
|
||||
# directory. This directory can contain a large amount of data, and that data
|
||||
# is specified by the server. If your /tmp is small and you are concerned
|
||||
# about the server potentially sending extra-huge files and filling things up,
|
||||
# you can set TMPDIR here.
|
||||
# TMPDIR=/srv/mirror/.tmp
|
||||
|
||||
#### Settings which most (Fedora) missors should not change
|
||||
# Mapping of MODULES to directory names, as an associative array
|
||||
# Fedora mirrors shouldn't change this
|
||||
# MODULEMAPPING=(fedora-alt alt fedora-archive archive
|
||||
# fedora-enchilada fedora fedora-epel epel)
|
||||
#
|
||||
# The name of the file holding the file list on the mirror host Note: the
|
||||
# string '$mdir' will be replaced with the name of the current module directory
|
||||
# in context, and so the '$' must be escaped or the string quoted.
|
||||
# FILELIST='fullfiletimelist-$mdir'
|
||||
|
||||
# An array of extra file lists to be transferred. They won't be processed, but
|
||||
# will implicitly be included in every transfer because file lists can't be
|
||||
# included in the file lists. '$mdir' is substituted as above.
|
||||
# Note that if you change this, you will want to do a run with -a to pick up
|
||||
# those extra files even in unchanged modules.
|
||||
EXTRAFILES=(fullfilelist imagelist-\$mdir)
|
||||
|
||||
# Array of default options to pass to rsync
|
||||
# Will be modified automatically according to VERBOSITY level; no need to set
|
||||
# -q, -v or --info here.
|
||||
#
|
||||
# You can add excludes here, but the script will always detect those files as
|
||||
# missing and will add them back to the file list. This may generate
|
||||
# complaints from rsync, but should not cause any problems, though it is almost
|
||||
# certainly better to use the provided FILTEREXP to do exclusion instead.
|
||||
#
|
||||
# Note that some of these options may be required for proper operation of the
|
||||
# script. You really should be careful if you change things here, as the
|
||||
# default value is carefully crafted and rsync can react in unexpected ways to
|
||||
# seeminly compatible sets of arguments.
|
||||
# RSYNCOPTS=(-aSH -f 'R .~tmp~' --stats --preallocate --delay-updates --out-format='@ %i %n%L')
|
||||
|
||||
# By default quick-fedora-mirror will try to detect and recover from an aborted
|
||||
# rsync run by moving any already downloaded files into place before
|
||||
# processing, Define NORSYNCRECOVERY (to anything) to prevent this.
|
||||
# NORSYNCRECOVERY=
|
||||
|
||||
# Define KEEPDIRTIMES (to anything) to make a third rsync call which restores
|
||||
# the timestamps of any directories which were modified after file removal.
|
||||
# This won't ensure that timestamps are always up to date, but its good enough
|
||||
# if you don't modify your repository locally. Maintaining directory
|
||||
# timestamps isn't important for mirroring in any case.
|
||||
# KEEPDIRTIMES=
|
||||
24
stat.sh
Normal file
24
stat.sh
Normal file
@@ -0,0 +1,24 @@
|
||||
#!/bin/bash
|
||||
echo "<h3><bold>$(date +%Y%m%d)</bold></h3><h4><bold>Log</bold></h4>"> /srv/mirror/pub/stat.html
|
||||
|
||||
AFILE="/var/log/nginx/mirror/access.log"
|
||||
|
||||
echo "<pre>$(/srv/mirror/scripts/ngparse.py html $AFILE)</pre>" >> /srv/mirror/pub/stat.html
|
||||
|
||||
echo "<h4><bold>Traffic</bold></h4>" >> /srv/mirror/pub/stat.html
|
||||
|
||||
vnstati -i enp1s0 -d --output /srv/mirror/pub/stat.jpg
|
||||
|
||||
echo "<img src='/stat.jpg'></img>" >> /srv/mirror/pub/stat.html
|
||||
|
||||
for i in /var/log/nginx/mirror/access.log-*;
|
||||
do
|
||||
DATE=${i#*-}
|
||||
if [ ! -f /srv/mirror/pub/stat/$DATE.html ]; then
|
||||
echo $DATE
|
||||
echo "<date>$DATE</date>" >> /srv/mirror/pub/stat/$DATE.html;
|
||||
echo "<pre>$(/srv/mirror/scripts/ngparse.py html $i)</pre>" >> /srv/mirror/pub/stat/$DATE.html
|
||||
echo "<br/><br/>" >> /srv/mirror/pub/stat/$DATE.html
|
||||
echo "<br/><br/>" >> /srv/mirror/pub/stat/$DATE.html
|
||||
fi
|
||||
done
|
||||
130
sync.sh
Normal file
130
sync.sh
Normal file
@@ -0,0 +1,130 @@
|
||||
#!/bin/bash
|
||||
|
||||
TIMENOW=$(date -u '+%Y%m%d_%H%M')
|
||||
BASE_DIR="/srv/mirror"
|
||||
ALERT="$BASE_DIR/scripts/alert.sh"
|
||||
DATA_DIR="$BASE_DIR/pub"
|
||||
|
||||
echo $TIMENOW $BASE_DIR $DATA_DIR $USER
|
||||
|
||||
if [[ "$USER" == "root" ]]; then
|
||||
echo "Dont run as root."
|
||||
sudo -u user $0 $@
|
||||
exit
|
||||
fi
|
||||
|
||||
# option="-rtlHpvi --chmod=D0755,F0644 --partial --hard-links --safe-links --stats --delete --delete-after --delay-updates --max-delete=70000 -P"
|
||||
|
||||
option="-rlptHS -B 8192 --chmod=D0755,F0644 --partial --hard-links --safe-links --delete-after --delay-updates --itemize-changes --delete --stats --no-i-r --verbose"
|
||||
# exclude="--exclude=.*.?????? --exclude='.~tmp~/' --exclude='Packages*' --exclude='Sources*' --exclude='Release*' --exclude='*.links.tar.gz*' --exclude='/other' --exclude='/sources'"
|
||||
|
||||
ubuntu="rsync://rsync.archive.ubuntu.com/ubuntu/"
|
||||
ubuntu_cd="rsync://releases.ubuntu.com/releases/"
|
||||
ubuntu_cd_old="rsync://old-releases.ubuntu.com/releases/"
|
||||
debian="rsync://mirrors.xtom.jp/debian/"
|
||||
debian_cd="rsync://ftp.riken.jp/debian-cd/"
|
||||
fedora="rsync://dl.fedoraproject.org/fedora-enchilada/linux/"
|
||||
epel="rsync://dl.fedoraproject.org/fedora-epel/"
|
||||
rpmfusion="rsync://download1.rpmfusion.org/rpmfusion/"
|
||||
archlinux="rsync://mirror.constant.com/archlinux/"
|
||||
raspbian="rsync://archive.raspbian.org/archive/"
|
||||
manjaro="rsync://ftp.riken.jp/manjaro/"
|
||||
gnu="rsync://ftp.gnu.org/gnu/"
|
||||
kali_images="rsync://repo.jing.rocks/kali-images"
|
||||
kali="rsync://repo.jing.rocks/kali"
|
||||
linux="rsync://rsync.kernel.org/pub/"
|
||||
fail="rsync://aa"
|
||||
|
||||
if [[ ! -v $1 ]]; then
|
||||
echo Not found.
|
||||
exit
|
||||
fi
|
||||
|
||||
ubuntu_cd_name="ubuntu-cd"
|
||||
debian_cd_name="debian-cd"
|
||||
ubuntu_cd_old_name="ubuntu-old"
|
||||
kali_images_name="kali-images"
|
||||
|
||||
dist=$1
|
||||
echo Syncing $1...
|
||||
set -o pipefail
|
||||
LASTLOG=`head -1 ${BASE_DIR}/logs/${dist}.log`
|
||||
|
||||
mv ${BASE_DIR}/logs/${dist}.log ${BASE_DIR}/logs/previous/${dist}-${LASTLOG}.log
|
||||
mv ${BASE_DIR}/logs/${dist}-error.log ${BASE_DIR}/logs/previous/${dist}-error-${LASTLOG}.log
|
||||
|
||||
if [[ -v ${dist}_name ]]; then
|
||||
dist_name_var="${dist}_name"
|
||||
dist_dir=${!dist_name_var}
|
||||
else
|
||||
dist_dir=$dist
|
||||
fi
|
||||
|
||||
echo ${TIMENOW} >> ${BASE_DIR}/logs/${dist}.log
|
||||
echo ${TIMENOW} >> ${BASE_DIR}/logs/${dist}-error.log
|
||||
echo "${TIMENOW}: Mirroring ${dist} from ${!dist} to ${DATA_DIR}/${dist}"
|
||||
echo "${TIMENOW} STARTED ${dist}" >> ${BASE_DIR}/logs/all.log
|
||||
TMEASURE=$(date +%s)
|
||||
TRY=3
|
||||
while [ $TRY -ne 0 ]; do
|
||||
echo Try $TRY...
|
||||
if [ "$dist" == "debian" ];
|
||||
then
|
||||
cd ${BASE_DIR}/scripts;
|
||||
export BASE_DIR=${BASE_DIR}
|
||||
export DATA_DIR=${DATA_DIR}
|
||||
./ftpsync
|
||||
EXIT=$?
|
||||
elif [ "$dist" == "fedora" ];
|
||||
then
|
||||
unset RSYNC_CONNECT_PROG
|
||||
# echo "Running stage 1 sync." | tee -a ${BASE_DIR}/logs/${dist}.log
|
||||
# echo "rsync -rlptHi --chmod=D0755,F0644 --safe-links --delete-delay --delay-updates --stats --progress -v --no-motd --log-file=rsync_fedora.log rsync://ftp.kaist.ac.kr/fedora /srv/mirror/pub/fedora/" | tee -a ${BASE_DIR}/logs/${dist}.log
|
||||
# rsync -rlptHi --chmod=D0755,F0644 --safe-links --delete-delay --delay-updates --stats --progress -v --no-motd --log-file=/srv/mirror/logs/rsync_fedora.log rsync://ftp.kaist.ac.kr/fedora /srv/mirror/pub/fedora/ 2> >(tee -a ${BASE_DIR}/logs/${dist}-error.log) | tee -a ${BASE_DIR}/logs/${dist}.log
|
||||
echo "Running via quick-fedora-mirror (Stage 1)." | tee -a ${BASE_DIR}/logs/${dist}.log
|
||||
cd ${BASE_DIR}/scripts;
|
||||
./quick-fedora-mirror 2> >(tee -a ${BASE_DIR}/logs/${dist}-error.log) | tee -a ${BASE_DIR}/logs/${dist}.log;
|
||||
EXIT=$?
|
||||
|
||||
echo "Running via quick-fedora-mirror (Stage 2)." | tee -a ${BASE_DIR}/logs/${dist}.log
|
||||
cd ${BASE_DIR}/scripts;
|
||||
FILTEREXP='(./development/rawhide/.*\.(iso|qcow2|img|raw\.xz)$)' \
|
||||
./quick-fedora-mirror 2> >(tee -a ${BASE_DIR}/logs/${dist}-error.log) | tee -a ${BASE_DIR}/logs/${dist}.log;
|
||||
else
|
||||
unset RSYNC_CONNECT_PROG
|
||||
if [ "$dist" == "kali_images" ];
|
||||
then
|
||||
# export RSYNC_CONNECT_PROG='ssh zhr0 nc %H 873'
|
||||
echo Connecting to RSYNC PROG
|
||||
fi
|
||||
echo "rsync ${option} ${exclude} ${!dist} ${DATA_DIR}/${dist_dir}" | tee -a ${BASE_DIR}/logs/${dist}.log
|
||||
rsync ${option} ${exclude} ${!dist} ${DATA_DIR}/${dist_dir} 2> >(tee -a ${BASE_DIR}/logs/${dist}-error.log) | tee -a ${BASE_DIR}/logs/${dist}.log
|
||||
EXIT=$?
|
||||
fi
|
||||
if [[ $EXIT == 0 ]]; then break; fi
|
||||
TRY=$(($TRY-1))
|
||||
done
|
||||
|
||||
if [ $EXIT -ne 0 ];
|
||||
then
|
||||
MSG="Sync ${dist} failed at ${TIMENOW}.\nLog: $(tail -n 1 ${BASE_DIR}/logs/${dist}-error.log)"
|
||||
if [ -f "$ALERT" ];
|
||||
then
|
||||
${ALERT} "${MSG}"
|
||||
fi
|
||||
|
||||
echo Sync ${dist} Error
|
||||
echo "${TIMENOW} ERROR ${dist}" >> ${BASE_DIR}/logs/all.log
|
||||
else
|
||||
echo Sync ${dist} Success
|
||||
if [ `echo ${BASE_DIR}/logs/${dist}-error.log | wc -l` -eq 1 ];
|
||||
then
|
||||
rm ${BASE_DIR}/logs/${dist}-error.log
|
||||
fi
|
||||
echo "${TIMENOW} DONE ${dist} $(date '+%Y%m%d_%H%M') $(($(date +%s)-$TMEASURE))" >> ${BASE_DIR}/logs/all.log
|
||||
cd $BASE_DIR
|
||||
echo "Updating Index"
|
||||
python3 -u ./scripts/index.py ${BASE_DIR}
|
||||
fi
|
||||
|
||||
exit;
|
||||
39
tmux.sh
Normal file
39
tmux.sh
Normal file
@@ -0,0 +1,39 @@
|
||||
#!/bin/bash
|
||||
|
||||
SESSION_NAME=$1
|
||||
WINDOW_NAME=$2
|
||||
COMMAND=$3
|
||||
|
||||
export SHELL=/bin/bash
|
||||
|
||||
echo [$(date)] $SESSION_NAME $WINDOW_NAME \"$COMMAND\"
|
||||
|
||||
if tmux has-session -t "$SESSION_NAME" 2>/dev/null; then
|
||||
|
||||
if tmux list-windows -t "$SESSION_NAME" | grep -q "$WINDOW_NAME"; then
|
||||
PANE_ID=$(tmux list-panes -t "$SESSION_NAME:$WINDOW_NAME" -F "#{pane_id}" | head -n 1)
|
||||
|
||||
if [ -n "$(ps -o state= -o comm= -t "$(tmux list-panes -F "#{pane_tty}" -t "$SESSION_NAME:$WINDOW_NAME" | head -n 1)" | grep -v -E '(bash|zsh|fish|sh|tmux)')" ]; then
|
||||
echo "ERROR-BUSY"
|
||||
exit 1
|
||||
|
||||
else
|
||||
echo "EXECUTE"
|
||||
tmux send-keys -t "$SESSION_NAME:$WINDOW_NAME" "$COMMAND" C-m
|
||||
fi
|
||||
|
||||
else
|
||||
echo "CREATE-WINDOW-EXECUTE"
|
||||
tmux new-window -t "$SESSION_NAME" -n "$WINDOW_NAME"
|
||||
tmux send-keys -t "$SESSION_NAME:$WINDOW_NAME" "$COMMAND" C-m
|
||||
fi
|
||||
|
||||
else
|
||||
echo "CREATE-SESSION-CREATE-WINDOW-EXECUTE"
|
||||
tmux new-session -d -s "$SESSION_NAME" -n "$WINDOW_NAME"
|
||||
tmux send-keys -t "$SESSION_NAME:$WINDOW_NAME" "$COMMAND" C-m
|
||||
fi
|
||||
|
||||
# 0 */4 * * * /home/user/tmux.sh cron mirror "/srv/mirror/scripts/dosync.sh" | tee -a $HOME/cron.log
|
||||
# 0 */4 * * * /home/user/tmux.sh cron fedora "/srv/mirror/scripts/fedora-sync.sh" | tee -a $HOME/cron.log
|
||||
|
||||
Reference in New Issue
Block a user