diff --git a/include/mirrors-table.wmi b/include/mirrors-table.wmi
index 9a3545c6..e69de29b 100644
--- a/include/mirrors-table.wmi
+++ b/include/mirrors-table.wmi
@@ -1,595 +0,0 @@
-
-
-
- | FR |
-
- Tor Supporter |
-
- Up to date |
-
- - |
- http |
- http |
- https |
- https |
- - |
- - |
-
-
-
-
- | UA |
-
- IP-Connect LLC |
-
- Up to date |
-
- ftp |
- http |
- http |
- - |
- - |
- rsync |
- rsync |
-
-
-
-
- | DE |
-
- TB-ITF |
-
- Up to date |
-
- - |
- http |
- http |
- https |
- https |
- - |
- - |
-
-
-
-
- | IS |
-
- torproject.is |
-
- Up to date |
-
- - |
- http |
- http |
- - |
- - |
- - |
- - |
-
-
-
-
- | AU |
-
- CoffsWiFi |
-
- Up to date |
-
- - |
- http |
- http |
- - |
- - |
- - |
- - |
-
-
-
-
- | IS |
-
- TheOnionRouter |
-
- Up to date |
-
- - |
- http |
- http |
- - |
- - |
- - |
- - |
-
-
-
-
- | DE |
-
- torservers |
-
- Up to date |
-
- - |
- http |
- http |
- https |
- https |
- - |
- - |
-
-
-
-
- | DE |
-
- Tor Supporter |
-
- Up to date |
-
- - |
- http |
- http |
- https |
- https |
- - |
- - |
-
-
-
-
- | DE |
-
- 0x3d.lu |
-
- Up to date |
-
- - |
- http |
- http |
- https |
- https |
- - |
- - |
-
-
-
-
- | US |
-
- intfxdx.com |
-
- Up to date |
-
- - |
- http |
- http |
- https |
- https |
- - |
- - |
-
-
-
-
- | US |
-
- Lucid Networks |
-
- Up to date |
-
- - |
- http |
- http |
- - |
- - |
- rsync |
- rsync |
-
-
-
-
- | US |
-
- EPRCI |
-
- Up to date |
-
- - |
- http |
- http |
- https |
- https |
- - |
- - |
-
-
-
-
- | FR |
-
- wardsback.org |
-
- Up to date |
-
- - |
- http |
- http |
- - |
- - |
- - |
- - |
-
-
-
-
- | TW |
-
- Department of CSE. Yuan Ze University |
-
- Up to date |
-
- ftp |
- http |
- http |
- https |
- https |
- rsync |
- rsync |
-
-
-
-
- | XX |
-
- TvdW |
-
- Up to date |
-
- - |
- http |
- http |
- - |
- - |
- - |
- - |
-
-
-
-
- | DE |
-
- Chaos Computer Club |
-
- Up to date |
-
- - |
- http |
- http |
- https |
- https |
- - |
- - |
-
-
-
-
- | US |
-
- The Calyx Institute |
-
- Up to date |
-
- - |
- http |
- http |
- https |
- https |
- - |
- - |
-
-
-
-
- | FR |
-
- Michael Armbruster |
-
- Up to date |
-
- - |
- http |
- http |
- https |
- https |
- rsync |
- rsync |
-
-
-
-
- | DE |
-
- TB-ITF |
-
- Up to date |
-
- ftp |
- http |
- http |
- https |
- https |
- - |
- - |
-
-
-
-
- | FR |
-
- stalkr.net |
-
- Up to date |
-
- - |
- http |
- http |
- https |
- https |
- - |
- - |
-
-
-
-
- | DE |
-
- tormirror |
-
- Up to date |
-
- ftp |
- http |
- http |
- - |
- - |
- - |
- - |
-
-
-
-
- | FR |
-
- Standalone Installer Software |
-
- Up to date |
-
- - |
- http |
- http |
- - |
- - |
- rsync |
- rsync |
-
-
-
-
- | IS |
-
- Frenn vun der Enn A.S.B.L. |
-
- Up to date |
-
- - |
- http |
- http |
- - |
- - |
- - |
- - |
-
-
-
-
- | ES |
-
- Tor Supporter |
-
- Up to date |
-
- - |
- http |
- http |
- https |
- https |
- - |
- - |
-
-
-
-
- | AT |
-
- ph3x |
-
- Up to date |
-
- - |
- http |
- http |
- https |
- https |
- - |
- - |
-
-
-
-
- | US |
-
- AskApache |
-
- Up to date |
-
- - |
- http |
- http |
- - |
- - |
- - |
- - |
-
-
-
-
- | DE |
-
- Tor Supporter |
-
- Up to date |
-
- - |
- http |
- http |
- https |
- https |
- - |
- - |
-
-
-
-
- | US |
-
- EFF |
-
- Up to date |
-
- - |
- - |
- - |
- https |
- https |
- - |
- - |
-
-
-
-
- | DE |
-
- Tor Supporter |
-
- Up to date |
-
- - |
- http |
- http |
- https |
- https |
- - |
- - |
-
-
-
-
- | CH |
-
- urown.net |
-
- Up to date |
-
- - |
- http |
- http |
- https |
- https |
- - |
- - |
-
-
-
-
- | DE |
-
- sela Internet |
-
- Up to date |
-
- - |
- http |
- http |
- https |
- https |
- - |
- - |
-
-
-
-
- | DE |
-
- Tor World (torworld.org) |
-
- Up to date |
-
- - |
- http |
- http |
- https |
- https |
- - |
- - |
-
-
-
-
- | US |
-
- Setec Administrator |
-
- Up to date |
-
- - |
- http |
- http |
- https |
- https |
- - |
- - |
-
-
-
-
- | VN |
-
- Freedif |
-
- Up to date |
-
- - |
- http |
- http |
- https |
- https |
- - |
- - |
-
-
-
-
- | DE |
-
- Funkfreunde Landshut e.V. |
-
- Up to date |
-
- - |
- http |
- http |
- - |
- - |
- - |
- - |
-
diff --git a/include/tor-mirrors.csv b/include/tor-mirrors.csv
index 5ff5b3c1..58310330 100644
--- a/include/tor-mirrors.csv
+++ b/include/tor-mirrors.csv
@@ -1,112 +1,29 @@
adminContact, orgName, isoCC, subRegion, region, ipv4, ipv6, loadBalanced, httpWebsiteMirror, httpsWebsiteMirror, rsyncWebsiteMirror, ftpWebsiteMirror, httpDistMirror, httpsDistMirror, rsyncDistMirror, hiddenServiceMirror, updateDate
-Vedran Miletić, University of Rijeka, HR, Croatia, HR, TRUE, FALSE, No, , , , , http://mirrors.uniri.hr/torproject.org/dist/, https://mirrors.uniri.hr/torproject.org/dist/, , ,
-Tor Fan, Tor Supporter, US, United States, US, TRUE, TRUE, No, http://tor.loritsu.com/, , , , http://tor.loritsu.com/dist/, , , ,
-Tor Fan, Tor Supporter, LU, Luxemborg, LU, TRUE, FALSE, No, http://torproject.adamas.ai/, , , , http://torproject.adamas.ai/dist/, , , ,
-Tor Fan, NW Linux, US, WA, US, TRUE, FALSE, No, http://torproject.nwlinux.us/, , rsync://nwlinux.us/tor-web, , http://torproject.nwlinux.us/dist/, , rsync://nwlinux.us/tor-dist, ,
-Tor Fan, Tor Supporter, NL, The Netherlands, NL, TRUE, FALSE, No, , , , , , https://www.coevoet.nl/tor/dist/, , ,
-Tor Fan, LazyTiger, FR, France, FR, TRUE, FALSE, No, http://tor.taiga-san.net/, , , , http://tor.taiga-san.net/dist/, , , ,
-Tor Fan, Tor Supporter, EE, Estonia, EE, TRUE, FALSE, No, http://tor.li/, https://tor.li/, , , http://tor.li/dist/, https://tor.li/dist/, , ,
-mirror-service@netcologne.de, NetCologne GmbH, DE, NRW, , TRUE, TRUE, No, http://mirror.netcologne.de/torproject.org, , rsync://mirror.netcologne.de/torproject.org, ftp://mirror.netcologne.de/torproject.org/, http://mirror.netcologne.de/torproject.org/dist, , rsync://mirror.netcologne.de/torproject.org/dist, , Wed Sep 5 16:16:16 2018
-admin AT netgull DOT com, NetGull, US, United States, North America, TRUE, TRUE, No, , , , , http://www.netgull.com/torproject/, , , ,
-mirrors[at]ip-connect[dot]vn[dot]ua, IP-Connect LLC, UA, VN, , TRUE, TRUE, Yes, http://torproject.ip-connect.vn.ua, , rsync://torproject.ip-connect.vn.ua/torproject, ftp://torproject.ip-connect.vn.ua/mirror/torproject/, http://torproject.ip-connect.vn.ua/dist, , rsync://torproject.ip-connect.vn.ua/torproject/dist, , Wed Sep 5 16:53:44 2018
-torsupport AT tb-itf DOT de, TB-ITF, DE, Germany, Europe, TRUE, TRUE, No, http://tormirror.tb-itf-tor.de, https://tormirror.tb-itf-tor.de, , , http://tormirror.tb-itf-tor.de/dist/, https://tormirror.tb-itf-tor.de/dist/, , , Wed Sep 5 16:53:44 2018
-info AT zentrum-der-gesundheit DOT de, Zentrum der Gesundheit, DK, Denmark, Europe, TRUE, FALSE, No, http://tor.idnr.ws/, , , , http://tor.idnr.ws/dist/, , , , Thu Sep 4 08:16:00 2014
-info /AT enn /DOT lu, Frenn vun der Enn A.S.B.L., IS, Iceland, Europe, TRUE, FALSE, No, http://torproject.lu/, , , , http://torproject.lu/dist/, , , http://btn6gqzqevlhoryd.onion, Wed Sep 5 14:06:00 2018
-Piratenpartei Bayern, Piratenpartei Bayern, DE, Germany, DE, TRUE, FALSE, NO, http://tormirror.piratenpartei-bayern.de, https://tormirror.piratenpartei-bayern.de, , , http://tormirror.piratenpartei-bayern.de/dist/, https://tormirror.piratenpartei-bayern.de/dist/, , , Fri Sep 9 22:46:02 2016
-Tor Fan, Tor Supporter, DE, Germany, DE, TRUE, TRUE, NO, http://tor.hoi-polloi.org, https://tor.hoi-polloi.org/, , , http://tor.hoi-polloi.org/dist/, https://tor.hoi-polloi.org/dist/, , , Fri May 5 15:12:25 2017
-tor@fodt.it // FoDT.it Webteam, FoDT.it, AT, Austria, Europe, TRUE, FALSE, No, http://tor.fodt.it, https://tor.fodt.it, , ftp://ftp.fodt.it/pub/mirrors/torproject.org/, http://tor.fodt.it/dist/, https://tor.fodt.it/dist/, , , Wed Aug 27 07:19:07 2014
-http://www.multinet.no, MultiNet AS, NO, Trondheim, Trondheim, TRUE, TRUE, No, http://tor.multinet.no/, , , , http://tor.multinet.no/dist/, , , , Wed Sep 5 00:49:40 2018
-haskell at gmx.es, Tor Supporter, ES, Spain, Europe, TRUE, TRUE, No, http://tor.zilog.es/, https://tor.zilog.es/, , , http://tor.zilog.es/dist/, https://tor.zilog.es/dist/, , , Wed Sep 5 14:06:00 2018
-Tor Fan, Tor Supporter, US, United States, US, FALSE, TRUE, No, http://2607:8b00:2::6258:5c9/, https://2607:8b00:2::6258:5c9/, , , http://2607:8b00:2::6258:5c9/dist/, https://2607:8b00:2::6258:5c9/dist/, , , Fri Jan 23 09:17:52 2015
-margus.random at mail.ee, CyberSIDE, EE, Estonia, EE, TRUE, FALSE, No, http://cyberside.planet.ee/tor/, , , , http://cyberside.net.ee/tor/, , , , Wed Sep 5 00:49:40 2018
-Tor Fan, torproject.is, IS, Iceland, IS, TRUE, FALSE, No, http://www.torproject.is/, , , , http://www.torproject.is/dist/, , , , Wed Sep 5 16:53:44 2018
-Tor Fan, spline, DE, Germany, DE, TRUE, FALSE, No, http://tor.spline.de/, https://tor.spline.inf.fu-berlin.de/, rsync://ftp.spline.de/tor, ftp://ftp.spline.de/pub/tor, http://tor.spline.de/dist/, https://tor.spline.inf.fu-berlin.de/dist/, rsync://ftp.spline.de/tor/dist, , Wed Sep 5 16:16:16 2018
-hosting AT ph3x DOT at, ph3x, AT, Austria, Europe, TRUE, FALSE, No, http://torproject.ph3x.at, https://torproject.ph3x.at, , , http://torproject.ph3x.at/dist/, https://torproject.ph3x.at/dist/, , , Wed Sep 5 14:06:00 2018
-Tor Fan, Tor Supporter, MX, Mexico, MX, TRUE, FALSE, No, http://fbnaia.homelinux.net/torproject/, https://fbnaia.homelinux.net/torproject/, , , http://fbnaia.homelinux.net/torproject/dist/, https://fbnaia.homelinux.net/torproject/dist/, , , Fri Sep 29 14:11:11 2017
-webmaster AT askapache DOT com, AskApache, US, California, US, TRUE, FALSE, No, http://tor.askapache.com/, , , , http://tor.askapache.com/dist/, , , , Wed Sep 5 14:06:00 2018
-kontakt AT unicorncloud DOT org, UnicornCloud.org, DE, Germany, Falkenstein, TRUE, FALSE, No, http://mirror.unicorncloud.org/torproject.org/, https://mirror.unicorncloud.org/torproject.org/, , , http://mirror.unicorncloud.org/torproject.org/dist, https://mirror.unicorncloud.org/torproject.org/dist, , , Fri Sep 29 11:12:25 2017
-root AT amorphis DOT eu, Amorphis, NL, The Netherlands, Europe, TRUE, FALSE, No, http://tor.amorphis.eu/, , , , http://tor.amorphis.eu/dist/, , , , Wed Mar 18 18:53:03 2015
-hackthissite.org, HackThisSite.org, US, United States, US, TRUE, TRUE, No, http://tor.hackthissite.org/, https://tor.hackthissite.org/, , , http://mirror.hackthissite.org/tor, https://mirror.hackthissite.org/tor, , , Wed May 25 21:09:20 2016
-paul at coffswifi.net, CoffsWiFi, AU, Australia and New Zealand, APNIC, TRUE, FALSE, No, http://torproject.coffswifi.net, , , , http://torproject.coffswifi.net/dist, , , , Wed Sep 5 16:53:44 2018
-Tor Fan, cyberarmy, AT, Austria, AT, TRUE, FALSE, No, http://tor.cyberarmy.at/, , , , , , , , Sat Jan 16 12:17:09 2016
-hostmaster AT example DOT com, TheOnionRouter, IS, Iceland, Iceland, TRUE, FALSE, No, http://www.theonionrouter.com/, , , , http://www.theonionrouter.com/dist/, , , , Wed Sep 5 16:53:44 2018
-Tor Fan, crazyhaze.de, DE, Germany, DE, TRUE, FALSE, No, http://tor.crazyhaze.de/, https://tor.crazyhaze.de/, , , http://tor.crazyhaze.de/dist/, https://tor.crazyhaze.de/dist/, , , Mon Jan 8 07:58:44 2018
-Tor Fan, Soviet Anonymous, RU, Russia, RU, TRUE, FALSE, No, http://creep.im/tor, https://creep.im/tor, rsync://creep.im/tor, ftp://creep.im/mirrors/tor, http://creep.im/tor/dist/, https://creep.im/tor/dist/, rsync://creep.im/tor-dist, , Mon Jan 8 07:58:44 2018
-Tor Fan, torservers, DE, Germany, DE, TRUE, FALSE, No, http://www.torservers.net/mirrors/torproject.org/, https://www.torservers.net/mirrors/torproject.org/, , , http://www.torservers.net/mirrors/torproject.org/dist/, https://www.torservers.net/mirrors/torproject.org/dist/, , http://hbpvnydyyjbmhx6b.onion/mirrors/torproject.org/, Wed Sep 5 16:53:44 2018
-Tor Fan, Lightning-bolt.net, CZ, Czech Republic, CZ, TRUE, FALSE, No, http://torproject.lightning-bolt.net/, , , , http://torproject.lightning-bolt.net/dist/, , , , Wed Mar 18 18:53:03 2015
-IceBear, myRL.net, IS, Iceland, IS, TRUE, FALSE, No, http://tor.myrl.net/, https://tor.myrl.net/, , , http://tor.myrl.net/dist/, https://tor.myrl.net/dist/, , , Sun Jan 7 20:12:24 2018
-kiro AT userzap DOT de, Userzap, DE, Germany, DE, TRUE, FALSE, No, http://torprojekt.userzap.de, https://torprojekt.userzap.de, , , http://torprojekt.userzap.de/dist/, https://torprojekt.userzap.de/dist/, , , Fri Jan 23 09:17:52 2015
-tor@les.net, tor@les.net, CA, Canada, CA, TRUE, FALSE, NO, http://tor.les.net/, , , , http://tor.les.net/dist, , , , Wed Sep 5 16:53:44 2018
-tor@stalkr.net, stalkr.net, FR, France, FR, TRUE, TRUE, NO, http://tor.stalkr.net/, https://tor.stalkr.net/, , , http://tor.stalkr.net/dist/, https://tor.stalkr.net/dist/, , , Wed Sep 5 16:16:16 2018
-doemela[AT]cyberguerrilla[DOT]org, cYbergueRrilLa AnonyMous NeXus, DE, Germany, DE, TRUE, FALSE, NO, , https://tor-mirror.cyberguerrilla.org, , , , https://tor-mirror.cyberguerrilla.org/dist/, , http://6dvj6v5imhny3anf.onion, Wed Sep 5 16:53:44 2018
-contact@gtor.org, Gtor, DE, Germany, DE, TRUE, TRUE, NO, http://torproject.gtor.org/, https://torproject.gtor.org/, rsync://torproject.gtor.org/website-mirror/, , http://torproject.gtor.org/dist/, https://torproject.gtor.org/dist/, rsync://torproject.gtor.org/website-mirror/dist/, , Fri Sep 9 22:46:02 2016
-Tor Fan, Tor Supporter, DE, Germany, DE, TRUE, TRUE, NO, http://mirror.velcommuta.de/tor/, https://mirror.velcommuta.de/tor/, , , http://mirror.velcommuta.de/tor/dist/, https://mirror.velcommuta.de/tor/dist/, , , Wed Sep 5 14:06:00 2018
-EFF, EFF, US, United States, US, TRUE, FALSE, NO, , https://tor.eff.org, , , , https://tor.eff.org/dist/, , , Wed Sep 5 14:06:00 2018
-Tor Fan, Tor Supporter, GR, Greece, GR, TRUE, TRUE, NO, http://tor.void.gr, https://tor.void.gr, , , http://tor.void.gr/dist/, https://tor.void.gr/dist/, , , Wed Jun 27 21:28:08 2018
-Ich Eben, Tor Supporter, DE, Germany, DE, TRUE, TRUE, No, http://reichster.de/mirrors/torproject.org/, https://reichster.de/mirrors/torproject.org, , , http://reichster.de/mirrors/torproject.org/dist/, https://reichster.de/mirrors/torproject.org/dist/, , , Wed Sep 5 14:06:00 2018
-jlgaddis AT gnu DOT org, Evil Routers, US, United States, US, TRUE, FALSE, No, http://tor1.evilrouters.net/, , , , http://tor1.evilrouters.net/dist/, , , , Sun Jan 7 20:12:24 2018
-tor AT miglix DOT eu, Tor Supporter, DE, Germany, Europe, TRUE, TRUE, NO, http://tor.miglix.eu, https://tor.miglix.eu, , , http://tor.miglix.eu/dist/, https://tor.miglix.eu/dist/, , , Fri Sep 9 22:46:02 2016
-tor TA ninurta TOD name, TorNinurtaName, AT, Austria, AT, TRUE, TRUE, no, http://tor.ninurta.name/, , , , http://tor.ninurta.name/dist/, , , , Fri Oct 24 12:02:17 2014
-fr33tux general-changelog-team.fr, Tor Supporter, FR, France, FR, TRUE, TRUE, No, http://tor.fr33tux.org, https://tor.fr33tux.org, , , http://tor.fr33tux.org/dist/, https://tor.fr33tux.org/dist/, , , Sun Jan 7 20:12:24 2018
-sebastian(at)bobrecki(dot)pl, Sebastian M. Bobrecki, PL, Poland, Europe, TRUE, FALSE, No, http://tor.iv.net.pl, https://tor.iv.net.pl, , , http://tor.iv.net.pl/dist/, https://tor.iv.net.pl/dist/, , , Mon Sep 14 14:14:19 2015
-tor-mirror AT rdns DOT cc, d0wn.biz, FR, France, Europe, TRUE, FALSE, No, http://tor.static.lu, https://tor.static.lu, , , http://tor.static.lu/dist/, https://tor.static.lu/dist/, , , Fri Sep 9 22:46:02 2016
-tor@moparisthebest.com, moparisthebest.com, DE, Germany, Europe, TRUE, TRUE, No, http://www.moparisthebest.com/tor/, https://www.moparisthebest.com/tor/, , , http://www.moparisthebest.com/tor/dist/, https://www.moparisthebest.com/tor/dist/, , , Sun Jan 7 20:12:24 2018
-stefano.fenoglio AT gmail DOT com, Tor Supporter, IT, Italy, Europe, TRUE, FALSE, No, http://tor.stefanof.com, , , , http://tor.stefanof.com/dist, , , , Fri Sep 9 22:46:02 2016
-Tor Fan, Ramos Research, US, United States, US, TRUE, TRUE, No, http://tor.ramosresearch.com/, , , , http://tor.ramosresearch.com/dist/, , , , Wed Mar 18 18:53:03 2015
-s7r[at]sky-ip[d0t]org, sky-ip.org, NL, Netherlands, NL, TRUE, FALSE, No, http://beautiful-mind.sky-ip.org/, , , , http://beautiful-mind.sky-ip.org/dist/, , , , Fri Sep 9 22:46:02 2016
-tor#pajonzeck#de, ITsn, DE, Germany, Europe, TRUE, FALSE, No, http://tor.pajonzeck.de/, https://tor.pajonzeck.de/, rsync://tor.pajonzeck.de/tor, , http://tor.pajonzeck.de/dist/, https://tor.pajonzeck.de/dist/, rsync://tor.pajonzeck.de/tor/dist, http://zgfgvob256pffy62.onion, Wed May 31 03:15:41 2017
-peter AT ludikovsky DOT name, Tor Supporter, AT, Austria, Europe, TRUE, TRUE, No, http://tor.ludikovsky.name/, https://tor.ludikovsky.name/, rsync://tor.ludikovsky.name/tor, , http://tor.ludikovsky.name/dist, https://tor.ludikovsky.name/dist, rsync://tor.ludikovsky.name/tor-dist, http://54lnbzjo6xlr4f4j.onion/, Sun Jan 7 20:12:24 2018
-admin AT nuclear DASH weapons DOT net, Setec Administrator, US, Texas, Austin, TRUE, FALSE, No, http://tor.nuclear-weapons.net, https://tor.nuclear-weapons.net, , , http://tor.nuclear-weapons.net/dist, https://tor.nuclear-weapons.net/dist, , , Wed Sep 5 00:49:40 2018
-opi@zeropi.net, Tor Supporter, FR, France, FR, TRUE, TRUE, No, http://tor-mirror.zeropi.net/, , , , http://tor-mirror.zeropi.net/dist/, , , , Thu Dec 4 05:15:20 2014
-alexander AT dietrich DOT cx, Tor Supporter, DE, Germany, Europe, TRUE, TRUE, No, http://tor.ybti.net/, https://tor.ybti.net/, , , http://tor.ybti.net/dist/, https://tor.ybti.net/dist/, , , Wed Sep 5 16:53:44 2018
-tor@0x3d.lu, 0x3d.lu, DE, Germany, Europe, TRUE, FALSE, No, http://tor.0x3d.lu/, https://tor.0x3d.lu/, , , http://tor.0x3d.lu/dist/, https://tor.0x3d.lu/dist/, , , Wed Sep 5 16:53:44 2018
-kontakt@unicorncloud.org, UnicornCloud.org, AT, Favoriten, Wien, TRUE, TRUE, No, http://www.unicorncloud.org/public/torproject.org/, https://www.unicorncloud.org/public/torproject.org/, , , http://www.unicorncloud.org/public/torproject.org/dist, https://www.unicorncloud.org/public/torproject.org/dist, , , Wed Mar 18 18:53:03 2015
-James Murphy, intfxdx.com, US, United States, US, TRUE, TRUE, No, http://108.248.87.242/, https://108.248.87.242/, , , http://108.248.87.242/dist/, https://108.248.87.242/dist/, , , Wed Sep 5 16:53:44 2018
-Sam Whited 4096R/54083AE104EA7AD3 , SamWhited.com, US, GA, United States, TRUE, TRUE, FALSE, http://mirrors.samwhited.net/tor, https://mirrors.samwhited.net/tor, rsync://mirrors.samwhited.net/tor, , http://mirrors.samwhited.net/tor/dist, https://mirrors.samwhited.net/tor/dist, rsync://mirrors.samwhited.net/tor-dist, , Fri Jul 17 15:49:12 2015
-rohit008 AT e DOT ntu DOT edu DOT sg, NTUOSS, SG, Singapore, Asia, TRUE, FALSE, No, http://torproject.ntuoss.com/, , , , http://torproject.ntuoss.com/dist/, , , , Wed Mar 18 18:53:03 2015
-hostmaster@lucidnetworks.net, Lucid Networks, US, United States, US, TRUE, FALSE, No, http://tor.mirrors.lucidnetworks.net, , rsync://tor.mirrors.lucidnetworks.net::tor, , http://tor.mirrors.lucidnetworks.net/dist, , rsync://tor.mirrors.lucidnetworks.net::tor-dist, , Wed Sep 5 16:53:44 2018
-mirror ntzk de, Netzkonstrukt Berlin, DE, Germany, Europe, TRUE, FALSE, No, http://mirror.ntzk.de/torproject.org/, https://mirror.ntzk.de/torproject.org/, , , http://mirror.ntzk.de/torproject.org/dist/, https://mirror.ntzk.de/torproject.org/dist/, , , Wed Sep 5 16:53:44 2018
-mirror@xfree.com.ar, Xfree.com.ar, AR, Argentina, South America, TRUE, FALSE, No, http://tor.xfree.com.ar/, , , , http://tor.xfree.com.ar/dist/, , , , Mon Jan 8 07:58:44 2018
-tor AT eprci NET, EPRCI, US, NH, US, TRUE, FALSE, No, http://tor.eprci.net/, https://www.eprci.com/tor/, , , http://tor.eprci.net/dist/, https://www.eprci.com/tor/dist/, , , Wed Sep 5 16:53:44 2018
-tor@kura.io, KURA IO LIMITED, NL, Netherlands, Europe, TRUE, TRUE, TRUE, http://tor-mirror.kura.io/, https://tor-mirror.kura.io/, rsync://tor-mirror.kura.io/torproject.org, ftp://tor-mirror.kura.io, http://tor-mirror.kura.io/dist/, https://tor-mirror.kura.io/dist/, rsync://tor-mirror.kura.io/torproject.org/dist, , Sun Jan 25 09:27:59 2015
-tor-admin AT wardsback DOT org, wardsback.org, FR, France, FR, TRUE, FALSE, No, http://alliumcepa.wardsback.org/, , , , http://alliumcepa.wardsback.org/dist/, , , , Wed Sep 5 16:53:44 2018
-PW, PW, DE, Germany, DE, TRUE, TRUE, NO, http://tor.pw.is/, https://www.it-sicherheitschannel.de/, , , http://tor.pw.is/dist/, https://www.it-sicherheitschannel.de/dist/, , , Mon Jan 8 07:58:44 2018
-kevin@freedom.press, Freedom of the Press Foundation, , US, US, True, False, No, http://tor.freedom.press, https://tor.freedom.press, , , http://tor.freedom.press/dist/, https://tor.freedom.press/dist/, , , Sat Jan 16 12:17:09 2016
-hsu AT peterdavehellor DOT org, Department of CSE. Yuan Ze University, TW, Taiwan, Asia, TRUE, FALSE, No, http://ftp.yzu.edu.tw/torproject.org/, https://ftp.yzu.edu.tw/torproject.org/, rsync://ftp.yzu.edu.tw/pub/torproject.org/, ftp://ftp.yzu.edu.tw/torproject.org/, http://ftp.yzu.edu.tw/torproject.org/dist/, https://ftp.yzu.edu.tw/torproject.org/dist/, rsync://ftp.yzu.edu.tw/pub/torproject.org/dist/, , Wed Sep 5 16:53:44 2018
-tor at tvdw dot eu, TvdW, XX, Around the world, XX, TRUE, TRUE, Yes, http://tor-exit.network, , , , http://tor-exit.network/dist, , , , Wed Sep 5 16:53:44 2018
-tormaster AT urown DOT net, urown.net, CH, Switzerland, Europe, TRUE, TRUE, No, http://torproject.urown.net/, https://torproject.urown.net/, , , http://torproject.urown.net/dist/, https://torproject.urown.net/dist/, , http://torprowdd64ytmyk.onion, Wed Sep 5 14:06:00 2018
-Stefan, sela Internet, DE, Germany, DE, TRUE, TRUE, No, http://sela.io/mirrors/torproject.org/, https://sela.io/mirrors/torproject.org/, , , http://sela.io/mirrors/torproject.org/dist/, https://sela.io/mirrors/torproject.org/dist/, , , Wed Sep 5 14:06:00 2018
-thomaswhite AT riseup DOT net, TheCthulhu, NL, The Netherlands, NL, True, False, No, http://tor.thecthulhu.com/, https://tor.thecthulhu.com/, , , http://tor.thecthulhu.com/dist/, https://tor.thecthulhu.com/dist/, , , Fri Sep 9 22:46:02 2016
-webmaster AT ccc DOT de, Chaos Computer Club, DE, Germany, Europe, TRUE, FALSE, No, http://tor.ccc.de/, https://tor.ccc.de, , , http://tor.ccc.de/dist/, https://tor.ccc.de/dist/, , , Wed Sep 5 16:53:44 2018
-Tor AT goodeid DOT com, Tor Supporter, CA, Canada, CA, TRUE, FALSE, No, http://goodeid.com/mirrors/tor-project.org/, , , , http://goodeid.com/mirrors/tor-project.org/dist/, , , , Fri Sep 9 22:46:02 2016
-tor@datensicherhe.it, datensicherhe.it, AT, Austria, Europe, TRUE, FALSE, No, http://datensicherhe.it/torproject, https://datensicherhe.it/torproject, , , http://datensicherhe.it/torproject/dist/, https://datensicherhe.it/torproject/dist/, , , Fri Sep 9 22:46:02 2016
-NocturnalFilth, Disciples of Disorder, NL, Netherlands, NL, TRUE, FALSE, NO, http://torproject.mirror.disciplesofdisorder.com, https://torproject.mirror.disciplesofdisorder.com, , , http://torproject.mirror.disciplesofdisorder.com/dist/, https://torproject.mirror.disciplesofdisorder.com/dist/, , , Wed May 17 11:00:42 2017
-tormirror0121.10.swsnyder@spamgourmet.com, tormirror, DE, Germany, TRUE, FALSE, NO, http://tormirror.snydernet.net/, https://tormirror.snydernet.net/, , , http://tormirror.snydernet.net/dist/, https://tormirror.snydernet.net/dist/, , , Mon Dec 28 17:11:31 2015, Wed Sep 5 16:16:16 2018
-justaguy@riseup.net, Justaguy, FR, France, FR, TRUE, FALSE, No, http://tormirror.justaguy.pw/, https://tormirror.justaguy.pw, , , http://tormirror.justaguy.pw/dist/, https://tormirror.justaguy.pw/dist/, , , Sat Jan 16 12:17:09 2016
-Disciples of Disorder, Vargr, NL, Netherlands, NL, TRUE, FALSE, NO, http://tor.mirror.disciplesofdisorder.eu, https://tor.mirror.disciplesofdisorder.eu, , , http://tor.mirror.disciplesofdisorder.eu/dist/, https://tor.mirror.disciplesofdisorder.eu/dist/, , http://vargrevir52vkbte.onion, Thu Dec 24 08:50:00 2015
-nick at calyx dot com, The Calyx Institute, US, United States, North America, TRUE, FALSE, No, http://tor.calyxinstitute.org, https://tor.calyxinstitute.org, , , http://tor.calyxinstitute.org/dist/, https://tor.calyxinstitute.org/dist/, , http://tmdrhl4e4anhsjc5.onion, Wed Sep 5 16:53:44 2018
-tor@armbrust.me, Michael Armbruster, FR, France, FR, TRUE, TRUE, No, http://tor.armbrust.me/, https://tor.armbrust.me/, rsync://tor.armbrust.me/tor, , http://tor.armbrust.me/dist/, https://tor.armbrust.me/dist, rsync://tor.armbrust.me/tor-dist, , Wed Sep 5 16:53:44 2018
-HdO Tor, HdO Tor Supporter, DE, Germany, US, TRUE, TRUE, No, http://tor.hdoev.de/, , , , http://tor.hdoev.de/dist/, , , , Fri Sep 9 22:46:02 2016
-Tor Fan, Tor Supporter, DE, Germany, Germany, TRUE, TRUE, No, http://torproject.mirror.metalgamer.eu/, https://torproject.mirror.metalgamer.eu/, , , http://torproject.mirror.metalgamer.eu/dist/, https://torproject.mirror.metalgamer.eu/dist/, , , Wed Sep 5 16:53:44 2018
-john AT quintex.com, Quintex Alliance Consulting, US, TX, US, TRUE, FALSE, No, http://torproject.quintex.com/, https://torproject.quintex.com, rsync://mirror.quintex.com/torprojectwebsite, ftp://mirror.quintex.com/torproject.org, http://torproject.quintex.com/dist, https://torproject.quintex.com/dist, rsync://mirror.quintex.com/torprojectdist, , Fri Sep 9 22:46:02 2016
-noc AT babylon DOT network, Babylon Network, NL, The Netherlands, Europe, TRUE, TRUE, No, http://nl.mirror.babylon.network/torproject/, https://nl.mirror.babylon.network/torproject/, rsync://nl.mirror.babylon.network/torproject/, ftp://nl.mirror.babylon.network/torproject/, http://nl.mirror.babylon.network/torproject/dist/, https://nl.mirror.babylon.network/torproject/dist/, rsync://nl.mirror.babylon.network/torproject/dist/, , Sun Jan 7 20:12:24 2018
-noc AT babylon DOT network, Babylon Network, FR, France, Europe, TRUE, TRUE, No, http://fr.mirror.babylon.network/torproject/, https://fr.mirror.babylon.network/torproject/, rsync://fr.mirror.babylon.network/torproject/, ftp://fr.mirror.babylon.network/torproject/, http://fr.mirror.babylon.network/torproject/dist/, https://fr.mirror.babylon.network/torproject/dist/, rsync://fr.mirror.babylon.network/torproject/dist/, , Sun Jan 7 20:12:24 2018
-Pool Toys, Pool Toys, SGP, Singapore, SGP, TRUE, TRUE, No, http://mirrors-sg.pooltoys.com/tor/, , rsync://mirrors-sg.pooltoys.com/tor/, ftp://mirrors-sg.pooltoys.com/tor/, http://mirrors-sg.pooltoys.com/tor/, , , ,
-cyberrax at yahoo.com, CyberSiDE, EE, Estonia, Europe, TRUE, FALSE, No, http://cyberside.net.ee/sibul/, https://cyberside.net.ee/sibul/, , , http://cyberside.net.ee/sibul/dist/, https://cyberside.net.ee/sibul/dist/, , , Wed Sep 5 00:49:40 2018
-torsupport AT tb-itf DOT de, TB-ITF, DE, Germany, Europe, TRUE, TRUE, No, http://tormirror.tb-itf-tor.de, https://tormirror.tb-itf-tor.de, , ftp://tormirror.tb-itf-tor.de/, http://tormirror.tb-itf-tor.de/dist/, https://tormirror.tb-itf-tor.de/dist/, , , Wed Sep 5 16:53:44 2018
-Disciples of Disorder, Vargr, NL, Netherlands, NL, TRUE, FALSE, NO, http://tor.mirror.disciplesofdisorder.eu, https://tor.mirror.disciplesofdisorder.eu, , , http://tor.mirror.disciplesofdisorder.eu/dist/, https://tor.mirror.disciplesofdisorder.eu/dist/, , http://vargrevir52vkbte.onion,
-admin @T standaloneinstaler.com, Standalone Installer Software, FR, France, FR, TRUE, TRUE, No, http://mirrors.standaloneinstaller.com/torproject/, , rsync://mirrors.standaloneinstaller.com/torproject/, , http://mirrors.standaloneinstaller.com/torproject/dist, , rsync://mirrors.standaloneinstaller.com/torproject/dist, , Wed Sep 5 16:16:16 2018
-lutz.horn@posteo.de, , DE, Germany, DE, TRUE, FALSE, No, , https://tor.lhorn.de/, , , , https://tor.lhorn.de/dist/, , , Wed Feb 8 12:00:00 2017
-karibu@freedif.org, Freedif, VN, Vietnam, VN, TRUE, FALSE, No, http://mirror.freedif.org/TorProject/, https://mirror.freedif.org/TorProject/, , , http://mirror.freedif.org/TorProject/dist, https://mirror.freedif.org/TorProject/dist, , , Wed Sep 5 00:49:40 2018
-Tor Fan, Tor Supporter, FR, France, FR, TRUE, FALSE, No, http://torproject.xj1.fr/, , , , http://torproject.xj1.fr/dist/, , , javkk6746z7wvigk.onion, Wed May 31 03:15:41 2017
-tienhn@vinahost.vn, VinaHost, VN, Viet Nam, VN, TRUE, FALSE, No, , https://mirror.vinahost.vn/torproject.org, rsync://mirror.vinahost.vn/torproject.org, , , https://mirror.vinahost.vn/torproject.org/dist/, rsync://mirror.vinahost.vn/torproject.org/dist, , Wed Apr 19 18:52:55 2017
-mirror AT funkfreundelandshut DOT de, Funkfreunde Landshut e.V., DE, BY, DE, TRUE, FALSE, No, http://mirror.funkfreundelandshut.de/torproject.org/, , , , http://mirror.funkfreundelandshut.de:81/torproject.org/dist/, , , http://44.225.40.254/torproject.org/dist, Wed Sep 5 00:49:40 2018
-email@heikorichter.name, Tor Supporter, DE, Germany, Europe, TRUE, TRUE, No, http://tor.heikorichter.name, https://tor.heikorichter.name, , , http://tor.heikorichter.name/dist, https://tor.heikorichter.name/dist, rsync://ftp.heikorichter.name/tor, , Sun Jan 7 20:12:24 2018
-tor secure voyage, secure.voyage, CA, Canada, CA, TRUE, FALSE, No, http://tor.secure.voyage, https://tor.secure.voyage, , , http://tor.secure.voyage/dist, https://tor.secure.voyage/dist, , , Mon Jan 8 07:58:44 2018
-mirror-torproject at, , , , , , , , , , , , , , , ,
-razx.cloud, razx, CA, Canada, AB, TRUE, FALSE, No, http://tcejorprot.razx.cloud/, https://tcejorprot.razx.cloud/, , , http://tcejorprot.razx.cloud/dist/, https://tcejorprot.razx.cloud/dist, , http://rsqiscyvxt4qgaiw.onion/torproject.org/, Wed Sep 5 19:32:45 2018
-coby [at ]127001 [dot] ovh, Tor Supporter, DE, Germany, DE, TRUE, FALSE, No, http://tor.127001.ovh/, https://tor.127001.ovh/, , , http://tor.127001.ovh/dist/, https://tor.127001.ovh/dist/, , , Mon Jan 8 07:58:44 2018
-Tor Fan, Tor Supporter, FR, France, Fr, TRUE, TRUE, No, http://mirror.oldsql.cc/tor/, https://mirror.oldsql.cc/tor/, , , http://mirror.oldsql.cc/tor/dist/, https://mirror.oldsql.cc/tor/dist/, , http://oldsqlcbr3aykyta.onion/tor, Wed Sep 5 19:32:45 2018
-Lunar, Tor World (torworld.org), DE, Germany, Europe, TRUE, FALSE, No, http://mirror.torworld.org/, https://mirror.torworld.org/, , , http://mirror.torworld.org/dist/, https://mirror.torworld.org/dist/, , , Wed Sep 5 14:06:00 2018
-, , , , , , , , , , , , , , , ,
-, , , , , , , , , , , , , , , ,
+mirror-service@netcologne.de, NetCologneGmbH, DE, NRW, , TRUE, TRUE, FALSE, http://mirror.netcologne.de/torproject.org, , rsync://mirror.netcologne.de/torproject.org, ftp://mirror.netcologne.de/torproject.org/, , , rsync://mirror.netcologne.de/torproject.org/dist, , Sun Sep 30 11:12:31 2018
+info/ATenn/DOTlu, FrennvunderEnnA.S.B.L., IS, Iceland, Europe, TRUE, FALSE, FALSE, http://torproject.lu/, , , , , , , , Sun Sep 30 15:12:31 2018
+http://www.multinet.no, MultiNetAS, NO, Trondheim, Trondheim, TRUE, TRUE, FALSE, http://tor.multinet.no/, , , , , , , , Sun Sep 30 15:12:31 2018
+haskellatgmx.es, TorSupporter, ES, Spain, Europe, TRUE, TRUE, FALSE, http://tor.zilog.es/, https://tor.zilog.es/, , , , , , , Sun Sep 30 18:48:27 2018
+TorFan, torproject.is, IS, Iceland, IS, TRUE, FALSE, FALSE, http://www.torproject.is/, , , , , , , , Sun Sep 30 18:48:27 2018
+TorFan, spline, DE, Germany, DE, TRUE, FALSE, FALSE, http://tor.spline.de/, https://tor.spline.inf.fu-berlin.de/, rsync://ftp.spline.de/tor, ftp://ftp.spline.de/pub/tor, , , rsync://ftp.spline.de/tor/dist, , Sun Sep 30 18:48:27 2018
+webmasterATaskapacheDOTcom, AskApache, US, California, US, TRUE, FALSE, FALSE, http://tor.askapache.com/, , , , , , , , Sun Sep 30 18:48:27 2018
+hostmasterATexampleDOTcom, TheOnionRouter, IS, Iceland, Iceland, TRUE, FALSE, FALSE, http://www.theonionrouter.com/, , , , , , , , Sun Sep 30 18:48:27 2018
+TorFan, torservers, DE, Germany, DE, TRUE, FALSE, FALSE, http://www.torservers.net/mirrors/torproject.org/, https://www.torservers.net/mirrors/torproject.org/, , , , , , , Sun Sep 30 18:48:27 2018
+tor@les.net, tor@les.net, CA, Canada, CA, TRUE, FALSE, FALSE, http://tor.les.net/, , , , , , , , Sun Sep 30 15:12:31 2018
+tor@stalkr.net, stalkr.net, FR, France, FR, TRUE, TRUE, FALSE, http://tor.stalkr.net/, https://tor.stalkr.net/, , , , , , , Sun Sep 30 15:12:31 2018
+TorFan, TorSupporter, DE, Germany, DE, TRUE, TRUE, FALSE, http://mirror.velcommuta.de/tor/, https://mirror.velcommuta.de/tor/, , , , , , , Sun Sep 30 18:48:27 2018
+alexanderATdietrichDOTcx, TorSupporter, DE, Germany, Europe, TRUE, TRUE, FALSE, http://tor.ybti.net/, https://tor.ybti.net/, , , , , , , Sun Sep 30 15:12:31 2018
+tor@0x3d.lu, 0x3d.lu, DE, Germany, Europe, TRUE, FALSE, FALSE, http://tor.0x3d.lu/, https://tor.0x3d.lu/, , , , , , , Sun Sep 30 18:48:27 2018
+JamesMurphy, intfxdx.com, US, UnitedStates, US, TRUE, TRUE, FALSE, http://108.248.87.242/, , , , , , , , Sun Sep 30 15:12:31 2018
+torATeprciNET, EPRCI, US, NH, US, TRUE, FALSE, FALSE, http://tor.eprci.net/, https://www.eprci.com/tor/, , , , , , , Sun Sep 30 18:48:27 2018
+tor-adminATwardsbackDOTorg, wardsback.org, FR, France, FR, TRUE, FALSE, FALSE, http://alliumcepa.wardsback.org/, , , , , , , , Sun Sep 30 15:12:31 2018
+hsuATpeterdavehellorDOTorg, DepartmentofCSE.YuanZeUniversity, TW, Taiwan, Asia, TRUE, FALSE, FALSE, http://ftp.yzu.edu.tw/torproject.org/, https://ftp.yzu.edu.tw/torproject.org/, rsync://ftp.yzu.edu.tw/pub/torproject.org/, ftp://ftp.yzu.edu.tw/torproject.org/, , , rsync://ftp.yzu.edu.tw/pub/torproject.org/dist/, , Sun Sep 30 18:48:27 2018
+tormasterATurownDOTnet, urown.net, CH, Switzerland, Europe, TRUE, TRUE, FALSE, http://torproject.urown.net/, https://torproject.urown.net/, , , , , , , Sat Sep 29 15:12:33 2018
+Stefan, selaInternet, DE, Germany, DE, TRUE, TRUE, FALSE, http://sela.io/mirrors/torproject.org/, https://sela.io/mirrors/torproject.org/, , , , , , , Sun Sep 30 15:12:31 2018
+webmasterATcccDOTde, ChaosComputerClub, DE, Germany, Europe, TRUE, FALSE, FALSE, http://tor.ccc.de/, https://tor.ccc.de/, , , , , , , Sun Sep 30 18:48:27 2018
+tormirror0121.10.swsnyder@spamgourmet.com, tormirror, DE, Germany, TRUE, FALSE, FALSE, FALSE, http://tormirror.snydernet.net/, https://tormirror.snydernet.net/, , , , , , , Sun Sep 30 18:48:27 2018
+nickatcalyxdotcom, TheCalyxInstitute, US, UnitedStates, NorthAmerica, TRUE, FALSE, FALSE, http://tor.calyxinstitute.org/, https://tor.calyxinstitute.org/, , , , , , , Sun Sep 30 18:48:27 2018
+tor@armbrust.me, MichaelArmbruster, FR, France, FR, TRUE, TRUE, FALSE, http://tor.armbrust.me/, https://tor.armbrust.me/, rsync://tor.armbrust.me/tor, , , , rsync://tor.armbrust.me/tor-dist, , Sun Sep 30 18:48:27 2018
+TorFan, TorSupporter, DE, Germany, Germany, TRUE, TRUE, FALSE, http://torproject.mirror.metalgamer.eu/, https://torproject.mirror.metalgamer.eu/, , , , , , , Sun Sep 30 18:48:27 2018
+admin@Tstandaloneinstaler.com, StandaloneInstallerSoftware, FR, France, FR, TRUE, TRUE, FALSE, http://mirrors.standaloneinstaller.com/torproject/, , rsync://mirrors.standaloneinstaller.com/torproject/, , , , rsync://mirrors.standaloneinstaller.com/torproject/dist, , Sun Sep 30 15:12:31 2018
+razx.cloud, razx, CA, Canada, AB, TRUE, FALSE, FALSE, http://tcejorprot.razx.cloud/, https://tcejorprot.razx.cloud/, , , , , , http://rsqiscyvxt4qgaiw.onion/torproject.org/, Sun Sep 30 18:48:27 2018
+Lunar, TorWorld(torworld.org), DE, Germany, Europe, TRUE, FALSE, FALSE, http://mirror.torworld.org/, https://mirror.torworld.org/, , , , , , , Sun Sep 30 15:12:31 2018
diff --git a/update-mirrors.pl b/update-mirrors.pl
index 088903cd..a66a8661 100755
--- a/update-mirrors.pl
+++ b/update-mirrors.pl
@@ -1,226 +1,361 @@
-#!/usr/bin/perl -w
+#!/usr/bin/env perl
+# This is Free Software (GPLv3) http://www.gnu.org/licenses/gpl-3.0.txt
+# This script is currently being refactored: https://bugs.torproject.org/22182
use warnings;
use strict;
+use Getopt::Long; # https://metacpan.org/pod/Getopt::Long::Modern
+use File::Basename;
use Data::Dumper;
use LWP::Simple;
use HTML::LinkExtor;
-use LWP;
-use Date::Parse;
-use Date::Format;
+use LWP; # https://metacpan.org/pod/distribution/libwww-perl/lwptut.pod
+use Time::Piece; # https://metacpan.org/pod/Time::Piece
use Digest::SHA qw(sha256_hex);
-# This is Free Software (GPLv3)
-# http://www.gnu.org/licenses/gpl-3.0.txt
+# Set Defaults
+my %opts = (
+ 'max-age' => 2,
+ 'socks_proxy' => '127.0.0.1:9050',
+ 'disable_proxy' => 0,
+ 'verify_files' => 0,
+ 'csvfile' => 'include/tor-mirrors.csv',
+ 'wmifile' => 'include/mirrors-table.wmi');
-print "Creating LWP agent ($LWP::VERSION)...\n";
-my $lua = LWP::UserAgent->new(
- keep_alive => 1,
- timeout => 30,
- agent => "Tor MirrorCheck Agent"
-);
+sub print_help {
+ print "
+Check if Tor mirrors are reachable, update $opts{'csvfile'} and regenerate
+$opts{'wmifile'} which is included in getinvolved/en/mirrors.wml and appears at
+https://www.torproject.org/getinvolved/mirrors
+
+Usage: $0 [options]
+\t--max-age\tSet acceptable age for mirrors in days (Default: $opts{'max-age'})
+\t--socks-proxy\tDefine SOCKS proxy to access mirrors (Default: $opts{'socks_proxy'})
+\t--no-proxy\tDirectly connect to mirrors.
+\t--verify-files\tDownload random files and verify their signature
+\t\t\t(depending on connection speed this can take a bit).
+\t--csv\t\tDefine alternative csv file (Default: $opts{'csvfile'})
+\t--wmi\t\tDefine alternative wmi file (Default: $opts{'wmifile'})
+\t--help\t\tShow this help\n";
+ exit 0;
+}
+
+# Parse Options
+GetOptions(
+ "max-age=n" => \$opts{'max-age'},
+ "socks-proxy=s" => \$opts{'socks_proxy'},
+ "no-proxy" => \$opts{'no_proxy'},
+ "verify-files" => \$opts{'verify_files'},
+ "csv=s" => \$opts{'csvfile'},
+ "wmi=s" => \$opts{'wmifile'},
+ "help" => sub { &print_help }
+) or die "Error parsing arguments. Please try again.\n";
+
+# Functions
+
+sub DateString {
+ return map { $_->strftime("%a %b %e %T %Y") } @_;
+}
-sub sanitize {
+sub SanitizeContent { # remove letters, numbers and other characters
my $taintedData = shift;
- my $cleanedData;
my $whitelist = '-a-zA-Z0-9: +';
-
# clean the data, return cleaned data
$taintedData =~ s/[^$whitelist]//go;
- $cleanedData = $taintedData;
-
- return $cleanedData;
+ return $taintedData;
}
+
sub ExtractLinks {
- my $content = shift;
- my $url = shift;
+ my ($content, $url, $ua) = @_;
+ unless ($content) { die "ExtractLinks: Called with empty content.\n"; }
+ unless ($url) { die "ExtractLinks: Called with empty URL.\n"; }
my @links;
-
my $parser = HTML::LinkExtor->new(undef, $url);
$parser->parse($content);
- foreach my $linkarray($parser->links)
- {
- my ($elt_type, $attr_name, $attr_value) = @$linkarray;
- if ($elt_type eq 'a' && $attr_name eq 'href' && $attr_value =~ /\/$/ && $attr_value =~ /^$url/)
- {
- push @links, Fetch($attr_value, \&ExtractLinks);
- }
- elsif ($attr_value =~ /\.(xpi|dmg|exe|tar\.gz)$/)
- #elsif ($attr_value =~ /\.(asc)$/) # small pgp files easier to test with
- {
- push @links, $attr_value;
- }
+ foreach my $linkarray ($parser->links) {
+ my ($elt_type, $attr_name, $attr_value) = @$linkarray;
+ if ($elt_type eq 'a' && $attr_name eq 'href' && $attr_value =~ /\/$/ && $attr_value =~ /^$url/) {
+ push @links, Fetch($ua, $attr_value, \&ExtractLinks);
+ }
+ #elsif ($attr_value =~ /\.(asc)$/) # small pgp files easier to test with
+ elsif ($attr_value =~ /\.(xpi|dmg|exe|tar\.gz)$/) {
+ push @links, $attr_value;
+ }
}
return @links;
}
sub ExtractDate {
- my $content = shift;
- $content = sanitize($content);
- my $date = str2time($content);
+ my $content = SanitizeContent(shift);
+ my $date = Time::Piece->strptime($content, "%a %b %d %T UTC %Y");
if ($date) {
- print "\tExtractDate($content) = $date\n";
+ print "\tExtractDate($content) = $date\n";
return $date;
} else {
- print "\tExtractDate($content) = ?\n";
- return undef;
+ print "\tExtractDate: Failed to parse date, this is no longer a mirror.\n";
+ return -1;
}
}
sub ExtractSig {
- my $content = shift;
- my $url = shift;
+ my ($content, $url) = @_;
my $sig = sha256_hex($content);
print "\tExtractSig($url) = $sig\n";
return $sig;
}
-sub Fetch {
- my ($url, $sub) = @_; # Base url for mirror
- $|++; # unbuffer stdout to show progress
+sub FindVersion {
+ my ($content, $url) = @_;
+ # TODO Parsing the webpage is an unstable solution (#21222).
+ foreach (split "\n", $content) {
+ if (/Version (.+) -/) { return $1; }
+ }
+ return undef;
+}
+sub Fetch {
+ my ($ua, $url, $sub) = @_;
+ if (! $url || $url eq '') { die "Fetch: called with empty URL.\n"; }
+ STDOUT->autoflush(1); # unbuffer stdout to show progress
print "\nGET $url: ";
my $request = new HTTP::Request GET => "$url";
- my $result = $lua->request($request);
+ my $result = $ua->request($request);
my $code = $result->code();
print "$code\n";
- if ($result->is_success && $code eq "200"){
- my $content = $result->content;
- if ($content) {
- return $sub->($content, $url);
- } else {
- print "Unable to fetch $url, empty content returned.\n";
- }
+ if ($result->is_success && $code eq "200") {
+ my $content = $result->content;
+ if ($content) {
+ if ($content =~ /403 Forbidden/) {
+ print "\tFetch: Received '403 Forbidden', removing mirror.";
+ return -1;
+ } elsif ($sub) { return $sub->($content, $url, $ua);
+ } else {
+ # Check header - https://metacpan.org/pod/HTTP::Headers
+ return ExtractDate( $result->header('Last-Modified') );
+ }
+ } else {
+ print "\tFetch: Empty content, no mirror here.\n";
+ return -1;
+ }
+ } elsif ($code eq "403") {
+ print "\tFetch: Received '403 Forbidden', removing mirror.";
+ return -1;
}
-
return undef;
}
-my @columns;
+
sub LoadMirrors {
- open(CSV, "<", "include/tor-mirrors.csv") or die "Cannot open tor-mirrors.csv: $!";
- my $line = ;
+ my $columns = shift;
+ open(my $fh, "<", $opts{'csvfile'}) or die "Cannot open '$opts{'csvfile'}': $!";
+ my $line = <$fh>;
chomp($line);
- @columns = split(/\s*,\s*/, $line);
+ @$columns = split(/\s*,\s*/, $line);
my @mirrors;
- while ($line = )
- {
- chomp($line);
- my @values = split(/\s*,\s*/, $line);
+ while (<$fh>) {
+ chomp;
+ my @values = split(/\s*,\s*/);
my %server;
- for (my $i = 0; $i < scalar(@columns); $i++)
- {
- $server{$columns[$i]} = $values[$i] || '';
+ for (my $i = 0; $i < scalar(@$columns); $i++) {
+ $server{$columns->[$i]} = $values[$i] || '';
}
- $server{updateDate} = str2time($server{updateDate}) if ($server{updateDate});
+ $server{updateDate} = Time::Piece->strptime($server{updateDate}, "%a %b %d %T %Y") if ($server{updateDate}); # alternatice "%c" < Sun Sep 30 18:48:27 2018
push @mirrors, {%server};
}
- close(CSV);
+ close($fh);
+ my $count = scalar(@mirrors);
+ print "Loaded $count mirrors from disk.\n";
return @mirrors;
}
sub DumpMirrors {
- my @m = @_;
- open(CSV, ">", "include/tor-mirrors.csv") or die "Cannot open tor-mirrors.csv: $!";
- print CSV join(", ", @columns) . "\n";
+ my ($columns, $time_barrier, @m) = @_;
+ open(my $csvfh, ">", $opts{'csvfile'}) or die "Cannot open '$opts{'csvfile'}': $!";
+ print $csvfh join(", ", @$columns) . "\n";
+ print "Saving mirrors that responded since ". DateString($time_barrier) ." ($time_barrier).\n";
foreach my $server(@m) {
+ # Drop mirrors that weren't reachable for some time
+ next if (! $server->{updateDate} || $server->{updateDate} < $time_barrier);
+ next unless ($server->{httpWebsiteMirror} || $server->{httpsWebsiteMirror} || $server->{ftpWebsiteMirror} || $server->{httpDistMirror} || $server->{httpsDistMirror} || $server->{hiddenServiceMirror});
+
$server->{updateDate} = gmtime($server->{updateDate}) if ($server->{updateDate});
- print CSV join(", ", map($server->{$_}, @columns));
- print CSV "\n";
+ print $csvfh join(", ", map($server->{$_}, @$columns));
+ print $csvfh "\n";
}
+ close($csvfh);
+ print "Updated $opts{'csvfile'}.\n";
+}
+
+sub PrintServer {
+ my ($server, $fh) = @_;
+ print $fh "\n\n
+ | $server->{isoCC} | \n
+ $server->{orgName} | \n
+ Up to date | \n";
+
+ my %prettyNames = ( # TODO make this accassible
+ httpWebsiteMirror => "http",
+ httpsWebsiteMirror => "https",
+ ftpWebsiteMirror => "ftp",
+ rsyncWebsiteMirror => "rsync",
+ httpDistMirror => "http",
+ httpsDistMirror => "https",
+ rsyncDistMirror => "rsync",
+ hiddenServiceMirror => "onion");
- close(CSV);
+ foreach my $precious ( sort keys %prettyNames ) {
+ if ($server->{$precious}) {
+ print $fh " {$precious} . "\">" .
+ "$prettyNames{$precious} | \n";
+ } else { print $fh " - | \n"; }
+ }
+ print $fh "
\n";
}
-my @m = LoadMirrors();
-my $count = scalar(@m);
-print "We have a total of $count mirrors\n";
-print "Fetching the last updated date for each mirror.\n";
+# Start
+chdir dirname($0);
+die "Could not find 'include' - are we in the webwml directory?.\n" unless (-d 'include');
+my $secperday = 86400;
+my $trace_path = 'project/trace/www-master.torproject.org';
+my $download_path = 'download/download.html.en';
+my (@columns, @torfiles, %randomtorfiles, @notrace);
+my @m = LoadMirrors(\@columns);
-my $tortime = Fetch("https://www.torproject.org/project/trace/www-master.torproject.org", \&ExtractDate);
-my @torfiles = Fetch("https://www.torproject.org/dist/", \&ExtractLinks);
-my %randomtorfiles;
+# Init LWP
+print "=============== Testing mirrors with LWP UserAgent $LWP::VERSION ===============\n";
+my $lua = LWP::UserAgent->new(
+ keep_alive => 1,
+ timeout => 30,
+ agent => "Tor MirrorCheck Agent"
+); # https://metacpan.org/pod/LWP::UserAgent
-for (1 .. 1)
-{
- my $r = int(rand(scalar(@torfiles)));
- my $suffix = $torfiles[$r];
- $suffix =~ s/^https:\/\/www.torproject.org//;
- $randomtorfiles{$suffix} = Fetch($torfiles[$r], \&ExtractSig);
+# Configure Proxy
+unless ($opts{'no_proxy'}) {
+ print "Loading LWP::Protocol::socks. Install module or use --no-proxy if this fails.\n";
+ require LWP::Protocol::socks; # https://metacpan.org/pod/LWP::Protocol::socks
+ $lua->proxy([qw(http https)] => "socks://$opts{'socks_proxy'}");
+ # TODO Maybe someone wants to use separate http or ftp proxy?
}
-print "Using these files for sig matching:\n";
-print join("\n", keys %randomtorfiles);
-print "\n";
+# Test proxy with tpo
+my $tortime = (Fetch($lua, "http://expyuzz4wqqyqhjn.onion/$trace_path", \&ExtractDate)
+ or Fetch($lua, "https://www.torproject.org/$trace_path", \&ExtractDate))
+ or die "Can't extract time from tpo. Are we or they offline?\n";
+$tortime -= $opts{'max-age'} * $secperday;
+print "The time barrier for mirrors to be listed is ". DateString($tortime) ." ($tortime).\n";
+
+# Determine current TB version
+my $tb_version = (Fetch($lua, "http://expyuzz4wqqyqhjn.onion/$download_path", \&FindVersion)
+ or Fetch($lua, "https://www.torproject.org/$download_path", \&FindVersion));
+if ($tb_version) { print "Tor Browser stable: $tb_version.\n"; }
+else { warn "\tFound no Tor Browser version on the download page, this script needs an update.\n"; }
+
+# nit: There's virtually no risk to receive a 403 above but it could cause a bug.
+
+if ($opts{'verify_files'}) {
+ # it is not optimal that we crawl dist, but it might be necessary if we
+ # are asked to thoroughly verify files on the mirrors.
+ @torfiles = Fetch($lua, "https://www.torproject.org/dist/", \&ExtractLinks);
+ die "Found no files on dist.\n" unless (@torfiles > 0);
-# Adjust official Tor time by out-of-date offset: number of days * seconds per day
-$tortime -= 1 * 172800;
-print "The official time for Tor is $tortime. \n";
+ my $r = int(rand(scalar(@torfiles)));
+ my $suffix = $torfiles[$r];
+ $suffix =~ s/^https:\/\/www.torproject.org//;
+ $randomtorfiles{$suffix} = Fetch($lua, $torfiles[$r], \&ExtractSig);
+
+ print "Using these files for sig matching:\n". join("\n", keys %randomtorfiles) ."\n";
+}
+
+for my $server (@m) {
+ foreach my $serverType('httpWebsiteMirror', 'httpsWebsiteMirror', 'ftpWebsiteMirror', 'httpDistMirror', 'httpsDistMirror', 'hiddenServiceMirror') {
+ if ($server->{$serverType}) {
+
+ my $url = "$server->{$serverType}";
+
+ # Retrieve trace URL as most reliable source for the mirror age
+ my $trace_url = "$url/$trace_path";
+ $trace_url =~ s/\/\/project/\/project/; # remove potential double slash
+ my $updateDate = Fetch($lua, $trace_url, \&ExtractDate);
+
+ # Mirrors without trace URL are 2nd class, we use the
+ # Last-Modified header of the base URL instead.
+ if (! $updateDate) {
+ my $updateDate = Fetch($lua, $url, \&ExtractDate);
+ push (@notrace, $url) if ($updateDate && $updateDate > 0);
+ }
+
+ if (! $updateDate || $updateDate == -1) { # TODO find a better solution
+ # Received empty content or 403 and have to remove it.
+ print "\tRemoving server $url.\n";
+ $server->{$serverType} = '';
+ next;
+ }
-for(my $server = 0; $server < scalar(@m); $server++) {
- foreach my $serverType('httpWebsiteMirror', 'httpsWebsiteMirror', 'ftpWebsiteMirror', 'httpDistMirror', 'httpsDistMirror')
- {
- if ($m[$server]->{$serverType}) {
- my $updateDate = Fetch("$m[$server]->{$serverType}/project/trace/www-master.torproject.org", \&ExtractDate);
-
+ # Check offered TB version and (optionally) verify files
if ($updateDate) {
- $m[$server]->{updateDate} = $updateDate;
- $m[$server]->{sigMatched} = 1;
+ $server->{updateDate} = $updateDate;
+
+ # Compare tor browser version
+ if ($serverType =~ /httpWebsiteMirror|httpsWebsiteMirror|hiddenServiceMirror/) {
+ my $version = Fetch($lua, "$server->{$serverType}/$download_path", \&FindVersion);
+ unless ($version) { # TODO Should we remove the mirror if the download page is wrong?
+ print "\tFound no Tor Browser version.\n";
+ } elsif ($tb_version ne $version) {
+ print "\tMirror offers an old Tor Browser version: $version\n";
+ }
+ }
+
+ # #22182: "The current way how the script is checking
+ # the mirror sites, isn't the best (it is looking for
+ # existing .xpi, .dmg, .exe, .tar.gz files)"
+ # Skipping if not requested with --verify-files
+ next unless ($opts{'verify_files'});
+ $server->{sigMatched} = 1;
foreach my $randomtorfile(keys %randomtorfiles) {
- my $sig = Fetch("$m[$server]->{$serverType}/$randomtorfile", \&ExtractSig);
+ my $fileurl = "$url/$randomtorfile";
+ $fileurl =~ /\/\//\//g;
+ my $sig = Fetch($lua, $fileurl, \&ExtractSig);
if (!$sig) {
- $m[$server]->{sigMatched} = 0;
- last;
+ $server->{sigMatched} = 0;
+ last;
} elsif ($sig ne $randomtorfiles{$randomtorfile}) {
- $m[$server]->{sigMatched} = 0;
- last;
+ $server->{sigMatched} = 0;
+ last;
}
}
}
- last;
}
}
}
-sub PrintServer {
- my $server = shift;
-print OUT <<"END";
- \n\n
- | $server->{isoCC} | \n
- $server->{orgName} | \n
- Up to date | \n
-END
-
- my %prettyNames = (
- httpWebsiteMirror => "http",
- httpsWebsiteMirror => "https",
- ftpWebsiteMirror => "ftp",
- rsyncWebsiteMirror => "rsync",
- httpDistMirror => "http",
- httpsDistMirror => "https",
- rsyncDistMirror => "rsync", );
-
- foreach my $precious ( sort keys %prettyNames )
- {
- if ($server->{$precious}) {
- print OUT " {$precious} . "\">" .
- "$prettyNames{$precious} | \n";
- } else { print OUT " - | \n"; }
- }
+# TODO we could also check rsync
- print OUT "
\n";
+# show list of mirrors without trace URL
+if (@notrace > 0) {
+ print "Following mirrors are reachable but give 404 on the timestamp URL:";
+ map { print "\t$_\n" } @notrace;
}
+# open wmi for writing
+open (my $wmifh, '>', $opts{'wmifile'}) or die "Can't write $opts{'wmifile'}: $!";
-my $outFile = "include/mirrors-table.wmi";
-open(OUT, "> $outFile") or die "Can't open $outFile: $!";
-
-# Here's where we open a file and print some wml include goodness
-# This is sorted from last known recent update to unknown update times
+# Print server list sorted from last known recent update to unknown update times
foreach my $server ( sort { $b->{updateDate} <=> $a->{updateDate} } grep {$_->{updateDate} && $_->{updateDate} > $tortime && $_->{sigMatched}} @m ) {
- PrintServer($server);
+ PrintServer($server, $wmifh);
}
+DumpMirrors(\@columns, $tortime - 31*$secperday, @m);
+close($wmifh);
+print "Updated $opts{'wmifile'}.\nWe are done here, enjoy your day!\n";
+
+__END__
-DumpMirrors(@m);
+- TODO you seem to be repeating the various server types (i think) in a few places. centralize that if possible.
+- TODO a perltidy pass never hurt anyone, neither would some consistent variable/function naming, but that's _really_ nitpicky https://metacpan.org/pod/Perl::Tidy
+ https://metacpan.org/pod/distribution/Perl-Tidy/bin/perltidy
+- TODO another thing to consider is to run it through perl critic with the freenode theme. perlbot: perl critic: a highly configurable and extensible Perl source code analyzer. Comes with policies based on Perl Best
+ Practices (PBP); but see http://p3rl.org/Perl::Critic::Freenode for our recommendations.
+- TODO be less verbose per default
+- TODO use Text::CSV (start with the csv() function), Mojo::CSV, Text::xSV, or DBD::CSV. If you 'just want to ...', see http://tburette.github.io/blog/2014/05/25/so-you-want-to-write-your-own-CSV-code/
-close(OUT);
+OK also Fetch is used in a bunch of places with or, yet on failure it will sometimes return -1
+OK line 168 replaces columns with contents 0 to ''