mirror of
https://github.com/morgan9e/mirror
synced 2026-04-14 00:14:15 +09:00
Init
This commit is contained in:
GitHub
105
base.html
Normal file
105
base.html
Normal file
@@ -0,0 +1,105 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
|
<title>Mirror Archive</title>
|
||||||
|
<style>
|
||||||
|
body {
|
||||||
|
font-size: 1.1em;
|
||||||
|
margin: 0;
|
||||||
|
margin-top: 6em;
|
||||||
|
padding: 0;
|
||||||
|
}
|
||||||
|
table, th, td {
|
||||||
|
border: 0px;
|
||||||
|
border-collapse: collapse;
|
||||||
|
}
|
||||||
|
th + th, td + td {
|
||||||
|
padding: 6px 16px;
|
||||||
|
}
|
||||||
|
th, td {
|
||||||
|
text-align: left;
|
||||||
|
vertical-align: top;
|
||||||
|
}
|
||||||
|
h2, h3 {
|
||||||
|
margin-bottom: 0.4em;
|
||||||
|
}
|
||||||
|
footer {
|
||||||
|
margin-top: 1.5em;
|
||||||
|
font-size: 0.75em;
|
||||||
|
}
|
||||||
|
.container {
|
||||||
|
width: 80%;
|
||||||
|
max-width: 800px;
|
||||||
|
margin: 0 auto;
|
||||||
|
padding: 0 20px;
|
||||||
|
}
|
||||||
|
.details {
|
||||||
|
line-height: 1.8;
|
||||||
|
}
|
||||||
|
.subtitle {
|
||||||
|
font-style: italic;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="container">
|
||||||
|
<header>
|
||||||
|
<h2>Mirror Archive</h2>
|
||||||
|
</header>
|
||||||
|
<main>
|
||||||
|
<div class="mirrorList">
|
||||||
|
<h3>Archive List</h3>
|
||||||
|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Name</th>
|
||||||
|
<th>Last Updated (UTC)</th>
|
||||||
|
<th>Sync Time</th>
|
||||||
|
<th>Source</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td><a href="/archlinux">/archlinux</a></td>
|
||||||
|
<td>@@archlinux@@</td>
|
||||||
|
<td>rsync://mirror.constant.com/archlinux</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><a href="/debian">/debian</a></td>
|
||||||
|
<td>@@debian@@</td>
|
||||||
|
<td>rsync://mirrors.xtom.com/debian</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><a href="/debian-cd">/debian-cd</a></td>
|
||||||
|
<td>@@debian_cd@@</td>
|
||||||
|
<td>rsync://mirrors.xtom.com/debian-cd</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><a href="/fedora">/fedora</a></td>
|
||||||
|
<td>@@fedora@@</td>
|
||||||
|
<td>rsync://dl.fedoraproject.org/fedora-enchilada</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><a href="/rpmfusion">/rpmfusion</a></td>
|
||||||
|
<td>@@rpmfusion@@</td>
|
||||||
|
<td>rsync://download1.rpmfusion.org/rpmfusion</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
<div class="details">
|
||||||
|
<h3>Server Details</h3>
|
||||||
|
<strong>Location:</strong> Seoul, Korea<br>
|
||||||
|
<strong>Bandwidth:</strong> 1 Gbps ~ 100 Mbps<br>
|
||||||
|
<strong>Sync delay:</strong> ~ 4 hours <br>
|
||||||
|
<strong>Contact:</strong> <a href="">mirror[at]devpg[d0t]net</a><br>
|
||||||
|
</div>
|
||||||
|
<footer>
|
||||||
|
Maintained by Av from Korea <span style="font-size: 0.65em">🏳️🌈🏳️⚧️</span>
|
||||||
|
</footer>
|
||||||
|
</main>
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
54
dosync.sh
Normal file
54
dosync.sh
Normal file
@@ -0,0 +1,54 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
TIMENOW=$(date '+%Y-%m-%d_%H:%M:%S');
|
||||||
|
LOCKFILE="/srv/mirror/scripts/.lock";
|
||||||
|
WAITLIMIT=30;
|
||||||
|
LOGFILE="/srv/mirror/logs/current.log"
|
||||||
|
SYNCLOG="/srv/mirror/logs/master.log"
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
echo "Cleaning up...";
|
||||||
|
rm -f "$LOCKFILE";
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
trap cleanup INT TERM KILL EXIT
|
||||||
|
|
||||||
|
if [ -f $LOCKFILE ]; then
|
||||||
|
echo "Previous session $(cat $LOCKFILE) exists." | tee -a ${SYNCLOG}
|
||||||
|
while [ $WAITLIMIT -ne 0 ]; do
|
||||||
|
echo "Waiting for previous session.. ${WAITLIMIT} tries left.." | tee -a ${SYNCLOG};
|
||||||
|
sleep 60;
|
||||||
|
WAITLIMIT=$(($WAITLIMIT-1));
|
||||||
|
if [ ! -f $LOCKFILE ]; then
|
||||||
|
break;
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
echo "Sync job at $TIMENOW blocked by previous job $(cat $LOCKFILE)." | tee -a ${SYNCLOG} >(/srv/mirror/scripts/alert.sh $(cat <&0))
|
||||||
|
exit;
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo $$ > $LOCKFILE;
|
||||||
|
|
||||||
|
echo "Starting sync job at $TIMENOW, $$" | tee ${LOGFILE} | tee -a ${SYNCLOG}
|
||||||
|
|
||||||
|
{
|
||||||
|
/srv/mirror/scripts/sync.sh archlinux &
|
||||||
|
/srv/mirror/scripts/sync.sh debian &
|
||||||
|
/srv/mirror/scripts/sync.sh rpmfusion &
|
||||||
|
wait;
|
||||||
|
|
||||||
|
# /srv/mirror/scripts/sync.sh ubuntu_cd &
|
||||||
|
# /srv/mirror/scripts/sync.sh ubuntu_cd_old &
|
||||||
|
# /srv/mirror/scripts/sync.sh debian_cd &
|
||||||
|
# wait;
|
||||||
|
|
||||||
|
# /srv/mirror/scripts/sync.sh linux &
|
||||||
|
# wait;
|
||||||
|
|
||||||
|
} 2>&1 | tee -a ${LOGFILE} | tee -a ${SYNCLOG}
|
||||||
|
|
||||||
|
wait;
|
||||||
|
echo Finished sync job $$ $TIMENOW at $(date '+%Y-%m-%d_%H:%M:%S') | tee -a ${LOGFILE} | tee -a ${SYNCLOG}
|
||||||
|
|
||||||
|
rm $LOCKFILE;
|
||||||
52
ftpsync.conf
Normal file
52
ftpsync.conf
Normal file
@@ -0,0 +1,52 @@
|
|||||||
|
########################################################################
|
||||||
|
########################################################################
|
||||||
|
## This is a sample configuration file for the ftpsync mirror script. ##
|
||||||
|
## Only options most users may need are included. For documentation ##
|
||||||
|
## and all available options see ftpsync.conf(5). ##
|
||||||
|
########################################################################
|
||||||
|
########################################################################
|
||||||
|
|
||||||
|
BASE_DIR=${BASE_DIR-"/srv/mirror"}
|
||||||
|
DATA_DIR=${DATA_DIR-"/srv/mirror/pub"}
|
||||||
|
|
||||||
|
echo $BASE_DIR $DATA_DIR
|
||||||
|
|
||||||
|
MIRRORNAME="mirror.morgan.kr"
|
||||||
|
TO="${DATA_DIR}/debian"
|
||||||
|
# MAILTO="$LOGNAME"
|
||||||
|
HUB=false
|
||||||
|
|
||||||
|
########################################################################
|
||||||
|
## Connection options
|
||||||
|
########################################################################
|
||||||
|
|
||||||
|
RSYNC_HOST="mirrors.xtom.com"
|
||||||
|
RSYNC_PATH="debian"
|
||||||
|
# RSYNC_USER=
|
||||||
|
# RSYNC_PASSWORD=
|
||||||
|
|
||||||
|
########################################################################
|
||||||
|
## Mirror information options
|
||||||
|
########################################################################
|
||||||
|
|
||||||
|
INFO_MAINTAINER="Morgan <mirror@morgan.kr>"
|
||||||
|
# INFO_SPONSOR="Example <https://example.com>"
|
||||||
|
INFO_COUNTRY=KR
|
||||||
|
INFO_LOCATION="Seoul, Korea"
|
||||||
|
INFO_THROUGHPUT=1Gb
|
||||||
|
|
||||||
|
########################################################################
|
||||||
|
## Include and exclude options
|
||||||
|
########################################################################
|
||||||
|
|
||||||
|
# ARCH_INCLUDE="amd64 arm64 riscv64 source"
|
||||||
|
# ARCH_EXCLUDE=
|
||||||
|
|
||||||
|
########################################################################
|
||||||
|
## Log option
|
||||||
|
########################################################################
|
||||||
|
|
||||||
|
LOGDIR="${BASE_DIR}/logs"
|
||||||
|
|
||||||
|
UIPRETRIES=9
|
||||||
|
TRACEHOST="mirror.morgan.kr"
|
||||||
90
index.py
Normal file
90
index.py
Normal file
@@ -0,0 +1,90 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import re
|
||||||
|
import datetime
|
||||||
|
|
||||||
|
base_path = sys.argv[1]
|
||||||
|
assert os.path.exists(base_path)
|
||||||
|
|
||||||
|
out_path = os.path.join(base_path, "pub/")
|
||||||
|
log_path = os.path.join(base_path, "logs/all.log")
|
||||||
|
template = os.path.join(base_path, "scripts/base.html")
|
||||||
|
index = os.path.join(out_path, "index.html")
|
||||||
|
|
||||||
|
assert os.path.exists(log_path)
|
||||||
|
assert os.path.exists(template)
|
||||||
|
|
||||||
|
with open(log_path, 'r') as f:
|
||||||
|
log = f.read().splitlines()
|
||||||
|
log.reverse()
|
||||||
|
|
||||||
|
with open(template, 'r') as f:
|
||||||
|
html_file = f.read()
|
||||||
|
|
||||||
|
target_dists = re.findall("@@([^@@]+)@@", html_file)
|
||||||
|
source_dists = set(line.split(" ")[2] for line in log)
|
||||||
|
|
||||||
|
print(target_dists)
|
||||||
|
|
||||||
|
status = {k: [[0, 0], 0, 0] for k in target_dists}
|
||||||
|
|
||||||
|
for target in target_dists:
|
||||||
|
for line in log:
|
||||||
|
time, stat, dist = (spl := line.split(" "))[:3]
|
||||||
|
time = datetime.datetime.strptime(time, '%Y%m%d_%H%M')
|
||||||
|
|
||||||
|
if dist == target:
|
||||||
|
if stat == "DONE":
|
||||||
|
if not status[target][0][0]:
|
||||||
|
status[target][0][0] = time
|
||||||
|
if len(spl) == 5:
|
||||||
|
status[target][0][1] = spl[4]
|
||||||
|
|
||||||
|
elif stat == "ERROR":
|
||||||
|
if not status[target][1]:
|
||||||
|
status[target][1] = time
|
||||||
|
|
||||||
|
elif stat == "STARTED":
|
||||||
|
if not status[target][2]:
|
||||||
|
status[target][2] = time
|
||||||
|
|
||||||
|
if status[target][0][0]:
|
||||||
|
break
|
||||||
|
|
||||||
|
print(status)
|
||||||
|
|
||||||
|
error_tmpl = "<br><span style='color:red;font-size:12px'>{}</span>"
|
||||||
|
|
||||||
|
for target in status:
|
||||||
|
# if status[target][1] exists, recently errored
|
||||||
|
done = status[target][0]
|
||||||
|
error = status[target][1]
|
||||||
|
running = status[target][2]
|
||||||
|
|
||||||
|
HTML = done[0].strftime("%Y-%m-%d %H:00") if done[0] else "Not Synced"
|
||||||
|
|
||||||
|
if error and (error > running):
|
||||||
|
HTML += error_tmpl.format(f'ERR {error.strftime("%Y-%m-%d %H:%M")}')
|
||||||
|
|
||||||
|
if running:
|
||||||
|
HTML += error_tmpl.format(f'RUN {running.strftime("%Y-%m-%d %H:%M")}')
|
||||||
|
|
||||||
|
HTML += "</td><td>"
|
||||||
|
|
||||||
|
def sec_to_hum(sec):
|
||||||
|
if sec >= 60:
|
||||||
|
if sec >= 3600:
|
||||||
|
return f"{sec//3600}h {(sec%3600)//60}m"
|
||||||
|
return f"{sec//60}m {sec%60}s"
|
||||||
|
return f"{sec}s"
|
||||||
|
|
||||||
|
HTML += sec_to_hum(int(done[1])) if done[1] else ""
|
||||||
|
|
||||||
|
html_file = html_file.replace(f"@@{target}@@", HTML)
|
||||||
|
|
||||||
|
with open(index, 'w') as f:
|
||||||
|
f.write(html_file)
|
||||||
|
|
||||||
|
print("Written to index.html")
|
||||||
1761
quick-fedora-mirror
Normal file
1761
quick-fedora-mirror
Normal file
File diff suppressed because it is too large
Load Diff
192
quick-fedora-mirror.conf
Normal file
192
quick-fedora-mirror.conf
Normal file
@@ -0,0 +1,192 @@
|
|||||||
|
# Configuration file for quick-fedora-mirror
|
||||||
|
# This file is sourced by the shell and must be in valid sh syntax.
|
||||||
|
|
||||||
|
#### Required settings
|
||||||
|
# Required: The the directory holding your copy of all of the modules you
|
||||||
|
# mirror. Does not include any module name
|
||||||
|
DESTD=/srv/mirror/pub/
|
||||||
|
|
||||||
|
# Required: The file in which to store the last mirror time.
|
||||||
|
# Note: this really should not be in the repository itself.
|
||||||
|
TIMEFILE=/srv/mirror/logs/qfs_lastsync
|
||||||
|
|
||||||
|
#### Settings Involving How and What to Mirror
|
||||||
|
# The remote host to rsync from, not including a module name
|
||||||
|
REMOTE=rsync://dl.fedoraproject.org
|
||||||
|
|
||||||
|
# The master module, which holds the other modules
|
||||||
|
# Note that the mirror you pull from MUST have this master module. You cannot
|
||||||
|
# use quick-fedora-mirror to pull from a host which does not have a master
|
||||||
|
# module containing the other modules.
|
||||||
|
MASTERMODULE=fedora-buffet
|
||||||
|
|
||||||
|
# Tier 1 Fedora mirrors should uncomment the following to get the proper
|
||||||
|
# pre-bitflip content.
|
||||||
|
# MASTERMODULE=fedora-buffet0
|
||||||
|
# PREBITFLIP=1
|
||||||
|
|
||||||
|
# Define if the entire repository (all modules under fedora-buffet) should be
|
||||||
|
# mirrored. If set, MODULES (below) is ignored
|
||||||
|
# MIRRORBUFFET=
|
||||||
|
|
||||||
|
# An array containing the modules to be mirrored
|
||||||
|
MODULES=(fedora-enchilada)
|
||||||
|
|
||||||
|
#### Filtering
|
||||||
|
# A regular expression used to filter the file lists. It must be quoted (or
|
||||||
|
# very carefully escaped). Entries matching this expression will not be synced
|
||||||
|
# and are expected not to be present locally. They will also be ignored by
|
||||||
|
# quick-fedora-hardlink. Cannot contain commas. Run against the file list that
|
||||||
|
# includes sizes (by quick-fedora-mirror) and the fullfiletimelist (by
|
||||||
|
# quick-fedora-hardlink), so don't use expressions that would match those
|
||||||
|
# metadata (which are digit strings and single characters). Example is a heavy
|
||||||
|
# filter which gives you an x86_64-only mirror with source packages, debuginfo
|
||||||
|
# packages, Alpha and Beta releases, and most image files excluded.
|
||||||
|
#FILTEREXP='(/i386|/armhfp|/source|/SRPMS|/debug/|\.iso|\.img|\.qcow2|\.raw\.xz|\.box|/releases/test)'
|
||||||
|
FILTEREXP='(/i386|/armhfp|\.img|\.raw\.xz|\.box|/releases/test)'
|
||||||
|
|
||||||
|
#### Email
|
||||||
|
# If you wish to receive email notices of transfers you can call this script
|
||||||
|
# via cron, or configure the following. Note that email won't be sent if
|
||||||
|
# quick-fedora-mirror is run interactively.
|
||||||
|
|
||||||
|
# Address to receive status email
|
||||||
|
# EMAILDEST=
|
||||||
|
|
||||||
|
# Subject to use for messages
|
||||||
|
# EMAILSUBJECT=
|
||||||
|
|
||||||
|
|
||||||
|
#### Logging
|
||||||
|
# Verbosity levels - info sent to stdout; usually this gets mailed to root when
|
||||||
|
# being run by cron
|
||||||
|
# 0 - quiet
|
||||||
|
# 1 - quiet if no changes, otherwise basic transfer info
|
||||||
|
# 2 - no -q to rsync
|
||||||
|
# 3 - very slightly more verbosity
|
||||||
|
# 4 - One -v to rsync
|
||||||
|
# 5 - Another -v to rsync
|
||||||
|
# 6 - Output of all settings
|
||||||
|
# 7 - Add --progress to rsync
|
||||||
|
# 8 - Shell script trace
|
||||||
|
VERBOSE=5
|
||||||
|
|
||||||
|
# Define (to anything) to enable logging to the systemd journal (via
|
||||||
|
# systemd-cat). the identifier "quick-fedora-mirror" is used, so logs can be
|
||||||
|
# retrieved with: journalctl -t quick-fedora-mirror
|
||||||
|
# LOGJOURNAL=
|
||||||
|
|
||||||
|
# Define to a full path to enable logging to that file.
|
||||||
|
# The provided file must already exist and be writable.
|
||||||
|
# Is only considered if LOGJOURNAL above is not defined.
|
||||||
|
LOGFILE=/srv/mirror/logs/fedora-qs.log
|
||||||
|
|
||||||
|
# Logging fields - Each character selects a piece of information to log.
|
||||||
|
# @ - Absolutely everything.
|
||||||
|
# a - aborted run recovery
|
||||||
|
# A - each recovered file from an aborted run
|
||||||
|
# c - rsync calls
|
||||||
|
# C - rsync call completions
|
||||||
|
# d - File/directory deletion start/end
|
||||||
|
# D - all file/directory deletes
|
||||||
|
# e - minor errors
|
||||||
|
# E - serious errors
|
||||||
|
# F - all transferred files (not impl)
|
||||||
|
# g - file list generation start/end
|
||||||
|
# k - lock contention
|
||||||
|
# l - per-module local file list generation (recursive find) start/end
|
||||||
|
# L - file list generation counts
|
||||||
|
# m - mirrormanager checkin
|
||||||
|
# M - mirrormanager checkin detail
|
||||||
|
# n - lack of updates in a run
|
||||||
|
# N - lack of updates in a module
|
||||||
|
# o - remote file list download start/end
|
||||||
|
# p - module processing start/end
|
||||||
|
# p - per-module module processing start/end
|
||||||
|
# r - run start
|
||||||
|
# R - run end
|
||||||
|
# s - Basic transfer statistics
|
||||||
|
# S - Detailed transfer statistics
|
||||||
|
# t - directory time updates (not impl)
|
||||||
|
# LOGITEMS=aeElrRs
|
||||||
|
|
||||||
|
#### Tuneables
|
||||||
|
# Rsync timeout value, in seconds
|
||||||
|
# Will be passed to rsync via --timeout.
|
||||||
|
RSYNCTIMEOUT=600
|
||||||
|
|
||||||
|
# In some situations rsync may experience a bug which causes it to emit lines like the following:
|
||||||
|
# WARNING: foo failed verification -- update put into partial-dir (will try again).
|
||||||
|
# These will break transfers and will not go away. Define
|
||||||
|
# RSYNC_PARTIAL_DIR_BUG (to anything) to work around this.
|
||||||
|
# RSYNC_PARTIAL_DIR_BUG=
|
||||||
|
|
||||||
|
# When q-f-m fails to run becuse it is already running, it checks the time
|
||||||
|
# since the last successful run. If that is larger than this value, it logs a
|
||||||
|
# serious error. Thid helps to detect a hung run or issues with slow
|
||||||
|
# transfers.
|
||||||
|
# WARNDELAY=$((60 * 60 * 24)) # One day
|
||||||
|
|
||||||
|
# When q-f-m encounters an error calling rsync, it may (depending on the error)
|
||||||
|
# sleep and retry. It will always sleep with exponential backup. Set
|
||||||
|
# MAXRETRIES to limit the number of times it retries.
|
||||||
|
MAXRETRIES=10
|
||||||
|
|
||||||
|
#### Important file and directory locations
|
||||||
|
# curl binary (only if MirrorManager checkins are enabled; see below).
|
||||||
|
# CURL=/usr/bin/curl
|
||||||
|
|
||||||
|
# rsync binary
|
||||||
|
# RSYNC=/usr/bin/rsync
|
||||||
|
|
||||||
|
# mktemp will be called after this file is sourced to make a temporary
|
||||||
|
# directory. This directory can contain a large amount of data, and that data
|
||||||
|
# is specified by the server. If your /tmp is small and you are concerned
|
||||||
|
# about the server potentially sending extra-huge files and filling things up,
|
||||||
|
# you can set TMPDIR here.
|
||||||
|
# TMPDIR=/srv/mirror/.tmp
|
||||||
|
|
||||||
|
#### Settings which most (Fedora) missors should not change
|
||||||
|
# Mapping of MODULES to directory names, as an associative array
|
||||||
|
# Fedora mirrors shouldn't change this
|
||||||
|
# MODULEMAPPING=(fedora-alt alt fedora-archive archive
|
||||||
|
# fedora-enchilada fedora fedora-epel epel)
|
||||||
|
#
|
||||||
|
# The name of the file holding the file list on the mirror host Note: the
|
||||||
|
# string '$mdir' will be replaced with the name of the current module directory
|
||||||
|
# in context, and so the '$' must be escaped or the string quoted.
|
||||||
|
# FILELIST='fullfiletimelist-$mdir'
|
||||||
|
|
||||||
|
# An array of extra file lists to be transferred. They won't be processed, but
|
||||||
|
# will implicitly be included in every transfer because file lists can't be
|
||||||
|
# included in the file lists. '$mdir' is substituted as above.
|
||||||
|
# Note that if you change this, you will want to do a run with -a to pick up
|
||||||
|
# those extra files even in unchanged modules.
|
||||||
|
EXTRAFILES=(fullfilelist imagelist-\$mdir)
|
||||||
|
|
||||||
|
# Array of default options to pass to rsync
|
||||||
|
# Will be modified automatically according to VERBOSITY level; no need to set
|
||||||
|
# -q, -v or --info here.
|
||||||
|
#
|
||||||
|
# You can add excludes here, but the script will always detect those files as
|
||||||
|
# missing and will add them back to the file list. This may generate
|
||||||
|
# complaints from rsync, but should not cause any problems, though it is almost
|
||||||
|
# certainly better to use the provided FILTEREXP to do exclusion instead.
|
||||||
|
#
|
||||||
|
# Note that some of these options may be required for proper operation of the
|
||||||
|
# script. You really should be careful if you change things here, as the
|
||||||
|
# default value is carefully crafted and rsync can react in unexpected ways to
|
||||||
|
# seeminly compatible sets of arguments.
|
||||||
|
# RSYNCOPTS=(-aSH -f 'R .~tmp~' --stats --preallocate --delay-updates --out-format='@ %i %n%L')
|
||||||
|
|
||||||
|
# By default quick-fedora-mirror will try to detect and recover from an aborted
|
||||||
|
# rsync run by moving any already downloaded files into place before
|
||||||
|
# processing, Define NORSYNCRECOVERY (to anything) to prevent this.
|
||||||
|
# NORSYNCRECOVERY=
|
||||||
|
|
||||||
|
# Define KEEPDIRTIMES (to anything) to make a third rsync call which restores
|
||||||
|
# the timestamps of any directories which were modified after file removal.
|
||||||
|
# This won't ensure that timestamps are always up to date, but its good enough
|
||||||
|
# if you don't modify your repository locally. Maintaining directory
|
||||||
|
# timestamps isn't important for mirroring in any case.
|
||||||
|
# KEEPDIRTIMES=
|
||||||
24
stat.sh
Normal file
24
stat.sh
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
echo "<h3><bold>$(date +%Y%m%d)</bold></h3><h4><bold>Log</bold></h4>"> /srv/mirror/pub/stat.html
|
||||||
|
|
||||||
|
AFILE="/var/log/nginx/mirror/access.log"
|
||||||
|
|
||||||
|
echo "<pre>$(/srv/mirror/scripts/ngparse.py html $AFILE)</pre>" >> /srv/mirror/pub/stat.html
|
||||||
|
|
||||||
|
echo "<h4><bold>Traffic</bold></h4>" >> /srv/mirror/pub/stat.html
|
||||||
|
|
||||||
|
vnstati -i enp1s0 -d --output /srv/mirror/pub/stat.jpg
|
||||||
|
|
||||||
|
echo "<img src='/stat.jpg'></img>" >> /srv/mirror/pub/stat.html
|
||||||
|
|
||||||
|
for i in /var/log/nginx/mirror/access.log-*;
|
||||||
|
do
|
||||||
|
DATE=${i#*-}
|
||||||
|
if [ ! -f /srv/mirror/pub/stat/$DATE.html ]; then
|
||||||
|
echo $DATE
|
||||||
|
echo "<date>$DATE</date>" >> /srv/mirror/pub/stat/$DATE.html;
|
||||||
|
echo "<pre>$(/srv/mirror/scripts/ngparse.py html $i)</pre>" >> /srv/mirror/pub/stat/$DATE.html
|
||||||
|
echo "<br/><br/>" >> /srv/mirror/pub/stat/$DATE.html
|
||||||
|
echo "<br/><br/>" >> /srv/mirror/pub/stat/$DATE.html
|
||||||
|
fi
|
||||||
|
done
|
||||||
130
sync.sh
Normal file
130
sync.sh
Normal file
@@ -0,0 +1,130 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
TIMENOW=$(date -u '+%Y%m%d_%H%M')
|
||||||
|
BASE_DIR="/srv/mirror"
|
||||||
|
ALERT="$BASE_DIR/scripts/alert.sh"
|
||||||
|
DATA_DIR="$BASE_DIR/pub"
|
||||||
|
|
||||||
|
echo $TIMENOW $BASE_DIR $DATA_DIR $USER
|
||||||
|
|
||||||
|
if [[ "$USER" == "root" ]]; then
|
||||||
|
echo "Dont run as root."
|
||||||
|
sudo -u user $0 $@
|
||||||
|
exit
|
||||||
|
fi
|
||||||
|
|
||||||
|
# option="-rtlHpvi --chmod=D0755,F0644 --partial --hard-links --safe-links --stats --delete --delete-after --delay-updates --max-delete=70000 -P"
|
||||||
|
|
||||||
|
option="-rlptHS -B 8192 --chmod=D0755,F0644 --partial --hard-links --safe-links --delete-after --delay-updates --itemize-changes --delete --stats --no-i-r --verbose"
|
||||||
|
# exclude="--exclude=.*.?????? --exclude='.~tmp~/' --exclude='Packages*' --exclude='Sources*' --exclude='Release*' --exclude='*.links.tar.gz*' --exclude='/other' --exclude='/sources'"
|
||||||
|
|
||||||
|
ubuntu="rsync://rsync.archive.ubuntu.com/ubuntu/"
|
||||||
|
ubuntu_cd="rsync://releases.ubuntu.com/releases/"
|
||||||
|
ubuntu_cd_old="rsync://old-releases.ubuntu.com/releases/"
|
||||||
|
debian="rsync://mirrors.xtom.jp/debian/"
|
||||||
|
debian_cd="rsync://ftp.riken.jp/debian-cd/"
|
||||||
|
fedora="rsync://dl.fedoraproject.org/fedora-enchilada/linux/"
|
||||||
|
epel="rsync://dl.fedoraproject.org/fedora-epel/"
|
||||||
|
rpmfusion="rsync://download1.rpmfusion.org/rpmfusion/"
|
||||||
|
archlinux="rsync://mirror.constant.com/archlinux/"
|
||||||
|
raspbian="rsync://archive.raspbian.org/archive/"
|
||||||
|
manjaro="rsync://ftp.riken.jp/manjaro/"
|
||||||
|
gnu="rsync://ftp.gnu.org/gnu/"
|
||||||
|
kali_images="rsync://repo.jing.rocks/kali-images"
|
||||||
|
kali="rsync://repo.jing.rocks/kali"
|
||||||
|
linux="rsync://rsync.kernel.org/pub/"
|
||||||
|
fail="rsync://aa"
|
||||||
|
|
||||||
|
if [[ ! -v $1 ]]; then
|
||||||
|
echo Not found.
|
||||||
|
exit
|
||||||
|
fi
|
||||||
|
|
||||||
|
ubuntu_cd_name="ubuntu-cd"
|
||||||
|
debian_cd_name="debian-cd"
|
||||||
|
ubuntu_cd_old_name="ubuntu-old"
|
||||||
|
kali_images_name="kali-images"
|
||||||
|
|
||||||
|
dist=$1
|
||||||
|
echo Syncing $1...
|
||||||
|
set -o pipefail
|
||||||
|
LASTLOG=`head -1 ${BASE_DIR}/logs/${dist}.log`
|
||||||
|
|
||||||
|
mv ${BASE_DIR}/logs/${dist}.log ${BASE_DIR}/logs/previous/${dist}-${LASTLOG}.log
|
||||||
|
mv ${BASE_DIR}/logs/${dist}-error.log ${BASE_DIR}/logs/previous/${dist}-error-${LASTLOG}.log
|
||||||
|
|
||||||
|
if [[ -v ${dist}_name ]]; then
|
||||||
|
dist_name_var="${dist}_name"
|
||||||
|
dist_dir=${!dist_name_var}
|
||||||
|
else
|
||||||
|
dist_dir=$dist
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo ${TIMENOW} >> ${BASE_DIR}/logs/${dist}.log
|
||||||
|
echo ${TIMENOW} >> ${BASE_DIR}/logs/${dist}-error.log
|
||||||
|
echo "${TIMENOW}: Mirroring ${dist} from ${!dist} to ${DATA_DIR}/${dist}"
|
||||||
|
echo "${TIMENOW} STARTED ${dist}" >> ${BASE_DIR}/logs/all.log
|
||||||
|
TMEASURE=$(date +%s)
|
||||||
|
TRY=3
|
||||||
|
while [ $TRY -ne 0 ]; do
|
||||||
|
echo Try $TRY...
|
||||||
|
if [ "$dist" == "debian" ];
|
||||||
|
then
|
||||||
|
cd ${BASE_DIR}/scripts;
|
||||||
|
export BASE_DIR=${BASE_DIR}
|
||||||
|
export DATA_DIR=${DATA_DIR}
|
||||||
|
./ftpsync
|
||||||
|
EXIT=$?
|
||||||
|
elif [ "$dist" == "fedora" ];
|
||||||
|
then
|
||||||
|
unset RSYNC_CONNECT_PROG
|
||||||
|
# echo "Running stage 1 sync." | tee -a ${BASE_DIR}/logs/${dist}.log
|
||||||
|
# echo "rsync -rlptHi --chmod=D0755,F0644 --safe-links --delete-delay --delay-updates --stats --progress -v --no-motd --log-file=rsync_fedora.log rsync://ftp.kaist.ac.kr/fedora /srv/mirror/pub/fedora/" | tee -a ${BASE_DIR}/logs/${dist}.log
|
||||||
|
# rsync -rlptHi --chmod=D0755,F0644 --safe-links --delete-delay --delay-updates --stats --progress -v --no-motd --log-file=/srv/mirror/logs/rsync_fedora.log rsync://ftp.kaist.ac.kr/fedora /srv/mirror/pub/fedora/ 2> >(tee -a ${BASE_DIR}/logs/${dist}-error.log) | tee -a ${BASE_DIR}/logs/${dist}.log
|
||||||
|
echo "Running via quick-fedora-mirror (Stage 1)." | tee -a ${BASE_DIR}/logs/${dist}.log
|
||||||
|
cd ${BASE_DIR}/scripts;
|
||||||
|
./quick-fedora-mirror 2> >(tee -a ${BASE_DIR}/logs/${dist}-error.log) | tee -a ${BASE_DIR}/logs/${dist}.log;
|
||||||
|
EXIT=$?
|
||||||
|
|
||||||
|
echo "Running via quick-fedora-mirror (Stage 2)." | tee -a ${BASE_DIR}/logs/${dist}.log
|
||||||
|
cd ${BASE_DIR}/scripts;
|
||||||
|
FILTEREXP='(./development/rawhide/.*\.(iso|qcow2|img|raw\.xz)$)' \
|
||||||
|
./quick-fedora-mirror 2> >(tee -a ${BASE_DIR}/logs/${dist}-error.log) | tee -a ${BASE_DIR}/logs/${dist}.log;
|
||||||
|
else
|
||||||
|
unset RSYNC_CONNECT_PROG
|
||||||
|
if [ "$dist" == "kali_images" ];
|
||||||
|
then
|
||||||
|
# export RSYNC_CONNECT_PROG='ssh zhr0 nc %H 873'
|
||||||
|
echo Connecting to RSYNC PROG
|
||||||
|
fi
|
||||||
|
echo "rsync ${option} ${exclude} ${!dist} ${DATA_DIR}/${dist_dir}" | tee -a ${BASE_DIR}/logs/${dist}.log
|
||||||
|
rsync ${option} ${exclude} ${!dist} ${DATA_DIR}/${dist_dir} 2> >(tee -a ${BASE_DIR}/logs/${dist}-error.log) | tee -a ${BASE_DIR}/logs/${dist}.log
|
||||||
|
EXIT=$?
|
||||||
|
fi
|
||||||
|
if [[ $EXIT == 0 ]]; then break; fi
|
||||||
|
TRY=$(($TRY-1))
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ $EXIT -ne 0 ];
|
||||||
|
then
|
||||||
|
MSG="Sync ${dist} failed at ${TIMENOW}.\nLog: $(tail -n 1 ${BASE_DIR}/logs/${dist}-error.log)"
|
||||||
|
if [ -f "$ALERT" ];
|
||||||
|
then
|
||||||
|
${ALERT} "${MSG}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo Sync ${dist} Error
|
||||||
|
echo "${TIMENOW} ERROR ${dist}" >> ${BASE_DIR}/logs/all.log
|
||||||
|
else
|
||||||
|
echo Sync ${dist} Success
|
||||||
|
if [ `echo ${BASE_DIR}/logs/${dist}-error.log | wc -l` -eq 1 ];
|
||||||
|
then
|
||||||
|
rm ${BASE_DIR}/logs/${dist}-error.log
|
||||||
|
fi
|
||||||
|
echo "${TIMENOW} DONE ${dist} $(date '+%Y%m%d_%H%M') $(($(date +%s)-$TMEASURE))" >> ${BASE_DIR}/logs/all.log
|
||||||
|
cd $BASE_DIR
|
||||||
|
echo "Updating Index"
|
||||||
|
python3 -u ./scripts/index.py ${BASE_DIR}
|
||||||
|
fi
|
||||||
|
|
||||||
|
exit;
|
||||||
39
tmux.sh
Normal file
39
tmux.sh
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
SESSION_NAME=$1
|
||||||
|
WINDOW_NAME=$2
|
||||||
|
COMMAND=$3
|
||||||
|
|
||||||
|
export SHELL=/bin/bash
|
||||||
|
|
||||||
|
echo [$(date)] $SESSION_NAME $WINDOW_NAME \"$COMMAND\"
|
||||||
|
|
||||||
|
if tmux has-session -t "$SESSION_NAME" 2>/dev/null; then
|
||||||
|
|
||||||
|
if tmux list-windows -t "$SESSION_NAME" | grep -q "$WINDOW_NAME"; then
|
||||||
|
PANE_ID=$(tmux list-panes -t "$SESSION_NAME:$WINDOW_NAME" -F "#{pane_id}" | head -n 1)
|
||||||
|
|
||||||
|
if [ -n "$(ps -o state= -o comm= -t "$(tmux list-panes -F "#{pane_tty}" -t "$SESSION_NAME:$WINDOW_NAME" | head -n 1)" | grep -v -E '(bash|zsh|fish|sh|tmux)')" ]; then
|
||||||
|
echo "ERROR-BUSY"
|
||||||
|
exit 1
|
||||||
|
|
||||||
|
else
|
||||||
|
echo "EXECUTE"
|
||||||
|
tmux send-keys -t "$SESSION_NAME:$WINDOW_NAME" "$COMMAND" C-m
|
||||||
|
fi
|
||||||
|
|
||||||
|
else
|
||||||
|
echo "CREATE-WINDOW-EXECUTE"
|
||||||
|
tmux new-window -t "$SESSION_NAME" -n "$WINDOW_NAME"
|
||||||
|
tmux send-keys -t "$SESSION_NAME:$WINDOW_NAME" "$COMMAND" C-m
|
||||||
|
fi
|
||||||
|
|
||||||
|
else
|
||||||
|
echo "CREATE-SESSION-CREATE-WINDOW-EXECUTE"
|
||||||
|
tmux new-session -d -s "$SESSION_NAME" -n "$WINDOW_NAME"
|
||||||
|
tmux send-keys -t "$SESSION_NAME:$WINDOW_NAME" "$COMMAND" C-m
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 0 */4 * * * /home/user/tmux.sh cron mirror "/srv/mirror/scripts/dosync.sh" | tee -a $HOME/cron.log
|
||||||
|
# 0 */4 * * * /home/user/tmux.sh cron fedora "/srv/mirror/scripts/fedora-sync.sh" | tee -a $HOME/cron.log
|
||||||
|
|
||||||
Reference in New Issue
Block a user