Modified ftpsync

scripts/config.yml

@@ -8,6 +8,8 @@ global:
     pre:
     post:
   log: logs/sync.log
+  log_dir: logs/
+  data_dir: pub/
   sync:
     - ALL
 
@@ -18,8 +20,9 @@ defaults:
     fail:
       - scripts/alert.sh
     post:
-      - scripts/index.py
+      - scripts/index.py ${config}
   log: logs/${path}.log
+  delay: 4
   rsync:
     options: "-rtlHpv --chmod=D0755,F0644 --partial --hard-links --safe-links --stats --delete --delete-after --delay-updates --max-delete=70000"
     exclude:
@@ -35,47 +38,44 @@ defaults:
 repos:
   ARCHLINUX:
     url: rsync://mirrors.xtom.de/archlinux/
-    delay: 4
     name: "ArchLinux (x86_64)"
     path: archlinux
 
   UBUNTU:
     url: rsync://rsync.archive.ubuntu.com/ubuntu/
-    delay: 4
     name: "Ubuntu"
     path: ubuntu
 
   UBUNTU_CD:
     url: rsync://releases.ubuntu.com/releases/
-    delay: 12
     name: "Ubuntu Releases"
+    delay: 6
     path: ubuntu_cd
 
   UBUNTU_CD_OLD:
     url: rsync://old-releases.ubuntu.com/releases/
-    delay: 12
     name: "Ubuntu Releases (Old)"
+    delay: 12
     path: ubuntu_cd_old
 
   DEBIAN:
-    url: rsync://mirrors.xtom.jp/debian/
+    url: rsync://mirrors.xtom.jp/
     type: ftpsync
-    delay: 4
     name: Debian
     path: debian
     scripts:
+
     # pre: scripts/pre_debian.sh
     last_sync: "2023-10-12T10:58Z"
   
   DEBIAN_CD:
     url: rsync://ftp.lanet.kr/debian-cd/
-    delay: 12
+    delay: 6
     name: "Debian CD"
     path: debian_cd
   
   FEDORA:
     url: rsync://dl.fedoraproject.org/fedora-enchilada/linux/
-    delay: 4
     name: "Fedora"
     path: fedora
   
@@ -87,7 +87,6 @@ repos:
   
   MANJARO:
     url: rsync://ftp.riken.jp/manjaro/
-    delay: 4
     name: Manjaro
     path: manjaro
   
@@ -105,6 +104,18 @@ repos:
     type: http
     delay: 12
 
+  GNU:
+    name: "GNU"
+    path: gnu
+    url: rsync://ftp.gnu.org/gnu/
+    delay: 12
+
+  LINUX:
+    name: "Linux Kernel"
+    path: linux
+    url: rsync://rsync.kernel.org/pub/
+    delay: 12
+
 index:
   - ARCHLINUX
   - UBUNTU UBUNTU_CD
@@ -115,4 +126,6 @@ index:
   - DIVIDER
   - ARCHLINUXARM
   - ASAHILINUX
+  - GNU
+  - LINUX
   - UBUNTU_CD_OLD

scripts/ftpsync.conf

@@ -1,12 +1,12 @@
-MIRRORNAME="pawe.me"
+MIRRORNAME="${DEB_HOST}"
-TO="${BASE_DIR}/debian"
+TO="${DEB_PATH}"
 HUB=false
-RSYNC_HOST="mirrors.xtom.jp"
+RSYNC_HOST="${DEB_UPSTREAM}"
 RSYNC_PATH="debian"
-INFO_MAINTAINER="Morgan <admin@pawe.me>"
+INFO_MAINTAINER="${DEB_ADMIN}"
-INFO_COUNTRY=KR
+INFO_COUNTRY="${DEB_COUNTRY}"
-INFO_LOCATION="Seoul, Korea"
+INFO_LOCATION="${DEB_LOCATION}"
-INFO_THROUGHPUT=500Mb
+INFO_THROUGHPUT=1Gbps
-LOGDIR="${BASE_DIR}/logs"
+LOGDIR="${DEB_LOG}"
 UIPRETRIES=9
-TRACEHOST="PAWE.ME"
+TRACEHOST="${DEB_HOST}"

scripts/index.py

@@ -5,29 +5,35 @@ import yaml
 import jinja2
 import datetime
 
-BASE_DIR = "."
+# def get_last_sync(repo_name):
-CONFIG_PATH = os.path.join(BASE_DIR, "scripts/config.yml")
+#     if os.path.exists(SYNC_LOG):
-TEMPLATES_DIR = os.path.join(BASE_DIR, "scripts/templates")
+#         with open(SYNC_LOG, 'r') as log_file:
-LOG_DIR = os.path.join(BASE_DIR, "logs")
+#             log_all = reversed(log_file.readlines())
-SYNC_LOG = os.path.join(LOG_DIR, "sync.log")
+#             for logline in log_all:
-OUTPUT_PATH = os.path.join(BASE_DIR, "index.html")
+#                 dist, time, stat = logline.split()
-
+#                 if repo_name == dist and stat =="SUCCESS":
-def get_last_sync(repo_name):
+#                     return datetime.datetime.strptime(time, '%Y-%m-%dT%H:%MZ').strftime("%Y-%m-%d %H:%M")
-    if os.path.exists(SYNC_LOG):
+#     return "Not Synced"
-        with open(SYNC_LOG, 'r') as log_file:
-            log_all = reversed(log_file.readlines())
-            for logline in log_all:
-                dist, time, stat = logline.split()
-                if repo_name == dist and stat =="SUCCESS":
-                    return datetime.datetime.strptime(time, '%Y-%m-%dT%H:%MZ').strftime("%Y-%m-%d %H:%M")
-    return "Not Synced"
 
 # main()
 
 if __name__=="__main__":
+
+    if len(sys,argv) == 3:
+        print(sys.argv)
+
+    CONFIG_PATH = sys.argv[1]
+
     with open(CONFIG_PATH, 'r') as f:
         config = yaml.safe_load(f)
 
+
+    BASE_DIR = config.get("BASE_DIR", ".")
+    THEME = "new"
+    TEMPLATES_DIR = os.path.join(os.path.join(BASE_DIR, "scripts/templates"), THEME)
+    LOG_DIR = os.path.join(BASE_DIR, config['global'].get("log_dir", "logs"))
+    OUTPUT_PATH = os.path.join(os.path.join(BASE_DIR, config['global'].get("data_dir", ".")), "index.html")
+
     env = jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATES_DIR))
     base_template = env.get_template('base.html')
     full_template = env.get_template('full.html')
@@ -50,8 +56,8 @@ if __name__=="__main__":
             context = {
                 'path': repo_data['path'],
                 'name': repo_data['name'],
-                'lastsync': get_last_sync(repo_name),
+                # 'lastsync': get_last_sync(repo_name),
-                # 'lastsync': repo_data.get('lastsync', "Not Synced"),
+                'lastsync': repo_data.get('lastsync', "Not Synced"),
                 'upstream': repo_data['url']
             }
             print(context)

scripts/nginx/nginx.conf

@@ -0,0 +1,91 @@
+user user;
+worker_processes auto;
+pid /run/nginx.pid;
+include /etc/nginx/modules-enabled/*.conf;
+
+events {
+	worker_connections 768;
+	# multi_accept on;
+}
+
+http {
+
+	##
+	# Basic Settings
+	##
+
+	sendfile on;
+	tcp_nopush on;
+	types_hash_max_size 2048;
+	# server_tokens off;
+
+	# server_names_hash_bucket_size 64;
+	# server_name_in_redirect off;
+
+	include /etc/nginx/mime.types;
+	default_type application/octet-stream;
+
+	##
+	# SSL Settings
+	##
+
+	ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; # Dropping SSLv3, ref: POODLE
+	ssl_prefer_server_ciphers on;
+
+	##
+	# Logging Settings
+	##
+
+	log_format main_ext '$remote_addr - $remote_user [$time_local] "$request" '
+                                '$status $body_bytes_sent "$http_referer" '
+                                '"$http_user_agent" "$http_x_forwarded_for" '
+                                '"$host" sn="$server_name" ' 'rt=$request_time '
+                                'ua="$upstream_addr" us="$upstream_status" '
+                                'ut="$upstream_response_time" ul="$upstream_response_length" '
+                                'cs=$upstream_cache_status' ;
+
+	access_log /var/log/nginx/access.log;
+	error_log /var/log/nginx/error.log;
+
+	##
+	# Gzip Settings
+	##
+
+	gzip on;
+
+	# gzip_vary on;
+	# gzip_proxied any;
+	# gzip_comp_level 6;
+	# gzip_buffers 16 8k;
+	# gzip_http_version 1.1;
+	# gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
+
+	##
+	# Virtual Host Configs
+	##
+
+	include /etc/nginx/conf.d/*.conf;
+	include /etc/nginx/sites-enabled/*;
+}
+
+
+#mail {
+#	# See sample authentication script at:
+#	# http://wiki.nginx.org/ImapAuthenticateWithApachePhpScript
+#
+#	# auth_http localhost/auth.php;
+#	# pop3_capabilities "TOP" "USER";
+#	# imap_capabilities "IMAP4rev1" "UIDPLUS";
+#
+#	server {
+#		listen     localhost:110;
+#		protocol   pop3;
+#		proxy      on;
+#	}
+#
+#	server {
+#		listen     localhost:143;
+#		protocol   imap;
+#		proxy      on;
+#	}
+#}

scripts/nginx/pawe.me.conf

@@ -0,0 +1,21 @@
+server {
+    listen 80 ;
+    listen [::]:80 ;
+    server_name repo.city;
+    
+    access_log /var/log/nginx/repo-city-access.log main_ext;
+    error_log /var/log/nginx/repo-city-error.log  warn;
+ 
+    root /srv/mirror;
+
+    location /scripts/ {
+        return 404;
+    }
+
+    location / {
+        autoindex on;
+        fancyindex on;
+    }
+}
+
+

scripts/nginx/stub_status.conf

@@ -0,0 +1,11 @@
+server {
+    listen 127.0.0.1:80;
+    server_name 127.0.0.1;
+    location /nginx_status {
+        stub_status on;
+        access_log off;
+        allow 127.0.0.1;
+        deny all;
+    }
+}
+

scripts/sync.sh

@@ -65,10 +65,10 @@ sync_repo() {
         return
     fi
 
-    path="${BASE_DIR}/$path/"
+    full_path="${BASE_DIR}/$(parse_yaml global.data_dir)/$path/"
     log="${BASE_DIR}/$log"
 
-    echo -e "--------\nRepo:      $name\nType:      $type\nUpstream:  $url\nPath:      $path\nLog:       $log\n--------"
+    echo -e "--------\nRepo:      $name\nType:      $type\nUpstream:  $url\nPath:      $full_path\nLog:       $log\n--------"
 
     if [[ ! -v DRY_RUN ]]; then
 
@@ -82,18 +82,26 @@ sync_repo() {
                 for ex in "${exclude_list[@]}"; do
                     exclude="${exclude} --exclude='${ex}'"
                 done
-                echo rsync ${rsync_options} ${exclude} $url $path >> $log
+                echo rsync ${rsync_options} ${exclude} $url $full_path >> $log
-                rsync ${rsync_options} ${exclude} $url $path >> $log 2>> ${log}-error
+                rsync ${rsync_options} ${exclude} $url $full_path >> $log 2>> ${log}-error
                 ;;
             "ftpsync")
                 cd ${BASE_DIR}/scripts
-                export BASE_DIR=${BASE_DIR}
+
+                export DEB_ADMIN="Morgan <mirror@devpg.net>"
+                export DEB_COUNTRY="Korea"
+                export DEB_LOCATION="Korea/Seoul"
+                export DEB_PATH="$full_path"
+                export DEB_HOST="$(parse_yaml global.hostname)"
+                export DEB_UPSTREAM="$url"
+                export DEB_LOG="${BASE_DIR}/$(parse_yaml global.log_dir)"
+                
                 ./ftpsync >> $log 2>> ${log}-error
                 cd ${BASE_DIR}
                 ;;
             "http")
                 echo ${repo} Fetch >> $log 2>> ${log}-error
-                python3 -u $BASE_DIR/scripts/getFetch.py "${url}" $path $BASE_DIR/scripts/${path}.fetch >> $log 2>> ${log}-error
+                python3 -u $BASE_DIR/scripts/getFetch.py "${url}" $full_path $BASE_DIR/scripts/${path}.fetch >> $log 2>> ${log}-error
                 echo ${repo} Download >> $log 2>> ${log}-error
                 python3 -u $BASE_DIR/scripts/getFile.py $BASE_DIR/scripts/${path}.fetch >> $log 2>> ${log}-error
                 ;;
@@ -277,6 +285,7 @@ for repo in "${repos[@]}"; do
 
         repo_post_scripts=($(get_repo_config ${repo} "scripts.post[]"))
         for script in "${repo_post_scripts[@]}"; do
+            script=${script/\${config\}/$CONFIG}
             execute $BASE_DIR/$script $repo
         done
     fi

scripts/templates/new/base.html

@@ -0,0 +1,99 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>repo.city</title>
+  <link rel="icon" type="image/x-icon" href="favicon.ico">
+  <link href="https://fonts.googleapis.com/css2?family=Noto+Sans:wght@300;400;500;600;700;800;900&display=swap"
+    rel="stylesheet">
+  <script src="https://cdn.tailwindcss.com"></script>
+  <script>
+    tailwind.config = {
+      darkMode: 'class'
+    }
+    // if (localStorage.theme === 'dark' || (!('theme' in localStorage) && window.matchMedia('(prefers-color-scheme: dark)').matches)) {
+    //  document.documentElement.classList.add('dark')
+    if (localStorage.theme === 'dark') {
+      document.documentElement.classList.add('dark')
+    } else {
+      document.documentElement.classList.remove('dark')
+    }
+  </script>
+  <style>
+    .url {
+      font-style: italic;
+      color: rgb(86, 105, 151);
+    }
+  </style>
+</head>
+
+<body class="font-sans text-gray-900 bg-gray-100 py-12 px-6 mx-auto max-w-5xl dark:bg-gray-900">
+  <header class="mb-10 ml-2">
+    <h1 class="text-3xl font-bold mb-2 text-black dark:text-white">Welcome to <a href="https://repo.city"
+                     class="text-blue-900 dark:text-blue-300 font-bold">REPO.CITY</a>!</h1>
+    <h2 class="text-xl text-gray-600 dark:text-gray-300">Simple mirroring &amp; archiving server.</h2>
+  </header>
+  <main>
+    <section class="mb-12">
+      <h2 class="dark:text-gray-200 text-2xl font-semibold mb-5 ml-2">Mirroring List</h2>
+      <div id="mirrorList" class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-2 gap-4">
+
+        {{ repos }}
+
+      </div>
+
+
+    </section>
+    <section class="mb-12">
+      <h2 class="dark:text-gray-200 text-2xl font-semibold mb-5 ml-2">Additional Mirror</h2>
+      <!-- <button onclick="toggleView('httpView')" class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded mb-4">View More</button> -->
+      <div id="httpView" class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-2 gap-4">
+
+        {{ repos_more }}
+
+        <div class="bg-white dark:bg-gray-800 col-span-1 md:col-span-2 shadow-lg rounded-lg p-5">
+          <h3 class="dark:text-white text-lg font-semibold mb-1"><a href="/cd-image"
+              class="text-blue-900 hover:text-blue-600 dark:text-blue-600 dark:hover:text-blue-400">Image Files</a></h3>
+        </div>
+      </div>
+    </section>
+    <section class="ml-2">
+      <h2 class="dark:text-gray-200 text-2xl font-semibold mb-4">Server Details</h2>
+        <p class="mb-2 text-gray-800 dark:text-gray-500"><strong>Location:</strong> Seoul, Korea</p>
+  <p class="mb-2 text-gray-800 dark:text-gray-500"><strong>Bandwidth:</strong> ~1Gbps</p>
+  <p class="mb-2 text-gray-800 dark:text-gray-500"><strong>Storage size:</strong> 32TiB</p>
+  <p class="mb-2 text-gray-800 dark:text-gray-500"><strong>Sync delay:</strong> 4hrs</p>  
+  <p class="mb-2 text-gray-800 dark:text-gray-500"><strong>Contact:</strong> <a href="mailto:mirror@devpg.net">mirror@devpg.net</a></p>
+  <p class="mb-2 text-gray-800 dark:text-gray-500"><strong>Maintainer:</strong> <a href="https://github.com/morgan9e">@morgan9e</a></p>
+      <p class="text-gray-500 dark:text-gray-600 url">This mirror is based on <a href="https://github.com/morgan9e/reposync" class="text-blue-500 dark:text-blue-800">reposync</a>.</p>
+      <p class="text-gray-500 dark:text-gray-600 url">For additional mirroring requests, please contact me at email above.</p>
+    </section>
+    <button
+      class="fixed bottom-6 right-6 bg-blue-200 hover:bg-blue-300 text-blue-700 dark:bg-blue-800 dark:hover:bg-blue-600 dark:text-white w-10 h-10 rounded-full flex items-center justify-center"
+      onclick="toggleDarkMode()">
+      +
+    </button>
+  </main>
+
+  <footer class="mt-10 text-center text-gray-600">
+    <a href="mailto:mirror@devpg.net">&copy; REPO.CITY</a>
+  </footer>
+
+  <script>
+    function toggleView(elementId) {
+      const element = document.getElementById(elementId);
+      element.style.display = element.style.display === "none" ? "block" : "none";
+    }
+    function toggleDarkMode() {
+      if (document.documentElement.classList.contains('dark')) {
+        document.documentElement.classList.remove('dark');
+        localStorage.theme = 'light';
+      } else {
+        document.documentElement.classList.add('dark');
+        localStorage.theme = 'dark';
+      }
+    }
+  </script>
+</body>
+</html>

scripts/templates/new/full.html

@@ -0,0 +1,6 @@
+<div class="bg-white dark:bg-gray-800 col-span-1 md:col-span-2 shadow-lg rounded-lg p-5">
+  <h3 class="dark:text-white text-lg font-semibold mb-1"><a href="/{{path}}"
+      class="text-blue-900 hover:text-blue-600 dark:text-blue-600 dark:hover:text-blue-400">{{name}}</a></h3>
+  <p class="text-gray-600 dark:text-gray-300">Last Updated: {{lastsync}}</p>
+  <p class="text-gray-600 dark:text-gray-300">Upstream: <span class="url">{{upstream}}</span></p>
+</div>

scripts/templates/new/half.html

@@ -0,0 +1,6 @@
+<div class="bg-white dark:bg-gray-800 col-span-1 shadow-lg rounded-lg p-5">
+  <h3 class="dark:text-white text-lg font-semibold mb-1"><a href="/{{path}}"
+      class="text-blue-900 hover:text-blue-600 dark:text-blue-600 dark:hover:text-blue-400">{{name}}</a></h3>
+  <p class="text-gray-600 dark:text-gray-300">Last Updated: {{lastsync}}</p>
+  <p class="text-gray-600 dark:text-gray-300">Upstream: <span class="url">{{upstream}}</span></p>
+</div>