
从 ftp.ripe.net 下载 probes 存档,并使用 PHP 进行预处理:
<?php
set_time_limit(1800) ;
ini_set("memory_limit", "-1");
file_put_contents("last_run.log", date('Y-m-d H:i:s'));
function getFlags($code){
$code = strtoupper($code);
if($code == 'AD') return '🇦🇩';
if($code == 'AE') return '🇦🇪';
if($code == 'AF') return '🇦🇫';
if($code == 'AG') return '🇦🇬';
if($code == 'AI') return '🇦🇮';
if($code == 'AL') return '🇦🇱';
if($code == 'AM') return '🇦🇲';
if($code == 'AO') return '🇦🇴';
if($code == 'AQ') return '🇦🇶';
if($code == 'AR') return '🇦🇷';
if($code == 'AS') return '🇦🇸';
if($code == 'AT') return '🇦🇹';
if($code == 'AU') return '🇦🇺';
if($code == 'AW') return '🇦🇼';
if($code == 'AX') return '🇦🇽';
if($code == 'AZ') return '🇦🇿';
if($code == 'BA') return '🇧🇦';
if($code == 'BB') return '🇧🇧';
if($code == 'BD') return '🇧🇩';
if($code == 'BE') return '🇧🇪';
if($code == 'BF') return '🇧🇫';
if($code == 'BG') return '🇧🇬';
if($code == 'BH') return '🇧🇭';
if($code == 'BI') return '🇧🇮';
if($code == 'BJ') return '🇧🇯';
if($code == 'BL') return '🇧🇱';
if($code == 'BM') return '🇧🇲';
if($code == 'BN') return '🇧🇳';
if($code == 'BO') return '🇧🇴';
if($code == 'BQ') return '🇧🇶';
if($code == 'BR') return '🇧🇷';
if($code == 'BS') return '🇧🇸';
if($code == 'BT') return '🇧🇹';
if($code == 'BV') return '🇧🇻';
if($code == 'BW') return '🇧🇼';
if($code == 'BY') return '🇧🇾';
if($code == 'BZ') return '🇧🇿';
if($code == 'CA') return '🇨🇦';
if($code == 'CC') return '🇨🇨';
if($code == 'CD') return '🇨🇩';
if($code == 'CF') return '🇨🇫';
if($code == 'CG') return '🇨🇬';
if($code == 'CH') return '🇨🇭';
if($code == 'CI') return '🇨🇮';
if($code == 'CK') return '🇨🇰';
if($code == 'CL') return '🇨🇱';
if($code == 'CM') return '🇨🇲';
if($code == 'CN') return '🇨🇳';
if($code == 'CO') return '🇨🇴';
if($code == 'CR') return '🇨🇷';
if($code == 'CU') return '🇨🇺';
if($code == 'CV') return '🇨🇻';
if($code == 'CW') return '🇨🇼';
if($code == 'CX') return '🇨🇽';
if($code == 'CY') return '🇨🇾';
if($code == 'CZ') return '🇨🇿';
if($code == 'DE') return '🇩🇪';
if($code == 'DJ') return '🇩🇯';
if($code == 'DK') return '🇩🇰';
if($code == 'DM') return '🇩🇲';
if($code == 'DO') return '🇩🇴';
if($code == 'DZ') return '🇩🇿';
if($code == 'EC') return '🇪🇨';
if($code == 'EE') return '🇪🇪';
if($code == 'EG') return '🇪🇬';
if($code == 'EH') return '🇪🇭';
if($code == 'ER') return '🇪🇷';
if($code == 'ES') return '🇪🇸';
if($code == 'ET') return '🇪🇹';
if($code == 'FI') return '🇫🇮';
if($code == 'FJ') return '🇫🇯';
if($code == 'FK') return '🇫🇰';
if($code == 'FM') return '🇫🇲';
if($code == 'FO') return '🇫🇴';
if($code == 'FR') return '🇫🇷';
if($code == 'GA') return '🇬🇦';
if($code == 'GB') return '🇬🇧';
if($code == 'GD') return '🇬🇩';
if($code == 'GE') return '🇬🇪';
if($code == 'GF') return '🇬🇫';
if($code == 'GG') return '🇬🇬';
if($code == 'GH') return '🇬🇭';
if($code == 'GI') return '🇬🇮';
if($code == 'GL') return '🇬🇱';
if($code == 'GM') return '🇬🇲';
if($code == 'GN') return '🇬🇳';
if($code == 'GP') return '🇬🇵';
if($code == 'GQ') return '🇬🇶';
if($code == 'GR') return '🇬🇷';
if($code == 'GS') return '🇬🇸';
if($code == 'GT') return '🇬🇹';
if($code == 'GU') return '🇬🇺';
if($code == 'GW') return '🇬🇼';
if($code == 'GY') return '🇬🇾';
if($code == 'HK') return '🇭🇰';
if($code == 'HM') return '🇭🇲';
if($code == 'HN') return '🇭🇳';
if($code == 'HR') return '🇭🇷';
if($code == 'HT') return '🇭🇹';
if($code == 'HU') return '🇭🇺';
if($code == 'ID') return '🇮🇩';
if($code == 'IE') return '🇮🇪';
if($code == 'IL') return '🇮🇱';
if($code == 'IM') return '🇮🇲';
if($code == 'IN') return '🇮🇳';
if($code == 'IO') return '🇮🇴';
if($code == 'IQ') return '🇮🇶';
if($code == 'IR') return '🇮🇷';
if($code == 'IS') return '🇮🇸';
if($code == 'IT') return '🇮🇹';
if($code == 'JE') return '🇯🇪';
if($code == 'JM') return '🇯🇲';
if($code == 'JO') return '🇯🇴';
if($code == 'JP') return '🇯🇵';
if($code == 'KE') return '🇰🇪';
if($code == 'KG') return '🇰🇬';
if($code == 'KH') return '🇰🇭';
if($code == 'KI') return '🇰🇮';
if($code == 'KM') return '🇰🇲';
if($code == 'KN') return '🇰🇳';
if($code == 'KP') return '🇰🇵';
if($code == 'KR') return '🇰🇷';
if($code == 'KW') return '🇰🇼';
if($code == 'KY') return '🇰🇾';
if($code == 'KZ') return '🇰🇿';
if($code == 'LA') return '🇱🇦';
if($code == 'LB') return '🇱🇧';
if($code == 'LC') return '🇱🇨';
if($code == 'LI') return '🇱🇮';
if($code == 'LK') return '🇱🇰';
if($code == 'LR') return '🇱🇷';
if($code == 'LS') return '🇱🇸';
if($code == 'LT') return '🇱🇹';
if($code == 'LU') return '🇱🇺';
if($code == 'LV') return '🇱🇻';
if($code == 'LY') return '🇱🇾';
if($code == 'MA') return '🇲🇦';
if($code == 'MC') return '🇲🇨';
if($code == 'MD') return '🇲🇩';
if($code == 'ME') return '🇲🇪';
if($code == 'MF') return '🇲🇫';
if($code == 'MG') return '🇲🇬';
if($code == 'MH') return '🇲🇭';
if($code == 'MK') return '🇲🇰';
if($code == 'ML') return '🇲🇱';
if($code == 'MM') return '🇲🇲';
if($code == 'MN') return '🇲🇳';
if($code == 'MO') return '🇲🇴';
if($code == 'MP') return '🇲🇵';
if($code == 'MQ') return '🇲🇶';
if($code == 'MR') return '🇲🇷';
if($code == 'MS') return '🇲🇸';
if($code == 'MT') return '🇲🇹';
if($code == 'MU') return '🇲🇺';
if($code == 'MV') return '🇲🇻';
if($code == 'MW') return '🇲🇼';
if($code == 'MX') return '🇲🇽';
if($code == 'MY') return '🇲🇾';
if($code == 'MZ') return '🇲🇿';
if($code == 'NA') return '🇳🇦';
if($code == 'NC') return '🇳🇨';
if($code == 'NE') return '🇳🇪';
if($code == 'NF') return '🇳🇫';
if($code == 'NG') return '🇳🇬';
if($code == 'NI') return '🇳🇮';
if($code == 'NL') return '🇳🇱';
if($code == 'NO') return '🇳🇴';
if($code == 'NP') return '🇳🇵';
if($code == 'NR') return '🇳🇷';
if($code == 'NU') return '🇳🇺';
if($code == 'NZ') return '🇳🇿';
if($code == 'OM') return '🇴🇲';
if($code == 'PA') return '🇵🇦';
if($code == 'PE') return '🇵🇪';
if($code == 'PF') return '🇵🇫';
if($code == 'PG') return '🇵🇬';
if($code == 'PH') return '🇵🇭';
if($code == 'PK') return '🇵🇰';
if($code == 'PL') return '🇵🇱';
if($code == 'PM') return '🇵🇲';
if($code == 'PN') return '🇵🇳';
if($code == 'PR') return '🇵🇷';
if($code == 'PS') return '🇵🇸';
if($code == 'PT') return '🇵🇹';
if($code == 'PW') return '🇵🇼';
if($code == 'PY') return '🇵🇾';
if($code == 'QA') return '🇶🇦';
if($code == 'RE') return '🇷🇪';
if($code == 'RO') return '🇷🇴';
if($code == 'RS') return '🇷🇸';
if($code == 'RU') return '🇷🇺';
if($code == 'RW') return '🇷🇼';
if($code == 'SA') return '🇸🇦';
if($code == 'SB') return '🇸🇧';
if($code == 'SC') return '🇸🇨';
if($code == 'SD') return '🇸🇩';
if($code == 'SE') return '🇸🇪';
if($code == 'SG') return '🇸🇬';
if($code == 'SH') return '🇸🇭';
if($code == 'SI') return '🇸🇮';
if($code == 'SJ') return '🇸🇯';
if($code == 'SK') return '🇸🇰';
if($code == 'SL') return '🇸🇱';
if($code == 'SM') return '🇸🇲';
if($code == 'SN') return '🇸🇳';
if($code == 'SO') return '🇸🇴';
if($code == 'SR') return '🇸🇷';
if($code == 'SS') return '🇸🇸';
if($code == 'ST') return '🇸🇹';
if($code == 'SV') return '🇸🇻';
if($code == 'SX') return '🇸🇽';
if($code == 'SY') return '🇸🇾';
if($code == 'SZ') return '🇸🇿';
if($code == 'TC') return '🇹🇨';
if($code == 'TD') return '🇹🇩';
if($code == 'TF') return '🇹🇫';
if($code == 'TG') return '🇹🇬';
if($code == 'TH') return '🇹🇭';
if($code == 'TJ') return '🇹🇯';
if($code == 'TK') return '🇹🇰';
if($code == 'TL') return '🇹🇱';
if($code == 'TM') return '🇹🇲';
if($code == 'TN') return '🇹🇳';
if($code == 'TO') return '🇹🇴';
if($code == 'TR') return '🇹🇷';
if($code == 'TT') return '🇹🇹';
if($code == 'TV') return '🇹🇻';
if($code == 'TW') return '🇹🇼';
if($code == 'TZ') return '🇹🇿';
if($code == 'UA') return '🇺🇦';
if($code == 'UG') return '🇺🇬';
if($code == 'UM') return '🇺🇲';
if($code == 'US') return '🇺🇸';
if($code == 'UY') return '🇺🇾';
if($code == 'UZ') return '🇺🇿';
if($code == 'VA') return '🇻🇦';
if($code == 'VC') return '🇻🇨';
if($code == 'VE') return '🇻🇪';
if($code == 'VG') return '🇻🇬';
if($code == 'VI') return '🇻🇮';
if($code == 'VN') return '🇻🇳';
if($code == 'VU') return '🇻🇺';
if($code == 'WF') return '🇼🇫';
if($code == 'WS') return '🇼🇸';
if($code == 'XK') return '🇽🇰';
if($code == 'YE') return '🇾🇪';
if($code == 'YT') return '🇾🇹';
if($code == 'ZA') return '🇿🇦';
if($code == 'ZM') return '🇿🇲';
return '🏳';
}
$path = date("Y/m",strtotime("-1 day"));
$fileName = date("Ymd",strtotime("-1 day")) . ".json.bz2";
$jsonFileName = date("Ymd",strtotime("-1 day")) . ".json";
$fileUrl = "https://ftp.ripe.net/ripe/atlas/probes/archive/$path/$fileName";
echo $fileUrl . "\n";
@unlink($jsonFileName);
@unlink($fileName);
echo "Retrieving http header...\n";
$header = get_headers($fileUrl);
$pp = "0";
echo json_encode($header, JSON_PRETTY_PRINT);
$key = key(preg_grep('/\bLength\b/i', $header));
$type = key(preg_grep('/\bType\b/i', $header));
$http = substr($header[0], 9, 3);
$tbytes = @explode(" ", $header[$key])[1];
$type = @explode("/", explode(" ", $header[$type])[1])[1];
echo " Target size: " . floor((($tbytes / 1000) / 1000)) . " Mo || " . floor(($tbytes / 1000)) . " Kb";
$remote = fopen($fileUrl, 'r');
$local = fopen($fileName, 'w');
$read_bytes = 0;
echo PHP_EOL;
while (!feof($remote)) {
$buffer = fread($remote, intval($tbytes));
fwrite($local, $buffer);
$read_bytes += 2048;
$progress = min(100, 100 * $read_bytes / $tbytes);
$progress = substr($progress, 0, 6) * 4;
$shell = 10; /* Progress bar width */
$rt = $shell * $progress / 100;
echo " \033[35;2m\e[0m Downloading: [" . round($progress, 3) . "%] " . floor((($read_bytes / 1000) * 4)) . "Kb ";
if ($pp === $shell) {
$pp = 0;
};
if ($rt === $shell) {
$rt = 0;
};
echo str_repeat("█", $rt) . str_repeat("=", ($pp++)) . ">@\r";
usleep(1000);
}
echo " \033[35;2m\e[0mDone [100%] " . floor((($tbytes / 1000) / 1000)) . " Mo || " . floor(($tbytes / 1000)) . " Kb \r";
echo PHP_EOL;
fclose($remote);
fclose($local);
echo 'bzip2 -d ' . $fileName . "\n";
system('bzip2 -d ' . $fileName);
echo "reading...\n";
$json = json_decode(file_get_contents($jsonFileName), true);
print_r($json['meta']);
@unlink($fileName);
@unlink($jsonFileName);
$probes = [];
echo "convert...\n";
foreach ($json['objects'] as &$item) {
foreach ($item as $key => $value) {
if(!in_array($key, ['id', 'address_v4', 'address_v6', 'asn_v4', 'asn_v6', 'country_code'])){
unset($item[$key]);
}
if($key == 'country_code') {
$item[$key] = getFlags($item[$key]) . " " . $item[$key];
}
}
$probes[$item['id']] = $item;
}
echo "write file: probes.json\n";
$json = json_encode($probes); //, JSON_PRETTY_PRINT);
file_put_contents("probes.json", $json);
echo "done.\n";
修改 ripe\atlas\tools\renderers\ping.py 文件,加入 JSON 解析:
# Copyright (c) 2016 RIPE NCC
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# import sys
# reload(sys)
# sys.setdefaultencoding('utf-8')
import json
import io
from ..helpers.sanitisers import sanitise
from .base import Renderer as BaseRenderer
class Renderer(BaseRenderer):
"""
This is meant to be a stub example for what an aggregate renderer might look
like. If you have ideas as to how to make this better, feel free to send
along a pull request.
"""
RENDERS = [BaseRenderer.TYPE_PING]
def __init__(self, **kwargs):
BaseRenderer.__init__(self, **kwargs)
self.target = ""
self.packet_loss = 0
self.sent_packets = 0
self.received_packets = 0
self.rtts = []
self.rtts_min = []
self.rtts_max = []
self.rtt_types_map = {"min": self.rtts_min, "max": self.rtts_max}
def collect_stats(self, result):
"""
Calculates, stores and collects all stats we want from the given
result.
"""
if not self.target:
self.target = result.destination_name
self.sent_packets += result.packets_sent
self.received_packets += result.packets_received
self.collect_min_max_rtts("min", result.rtt_min)
self.collect_min_max_rtts("max", result.rtt_max)
self.collect_packets_rtt(result.packets)
def collect_min_max_rtts(self, rtt_type, rtt):
"""
Stores the given rtt in the corresponding list (min/max) if rtt is set.
"""
rtt = rtt
if not rtt:
rtt = 0
self.rtt_types_map[rtt_type].append(rtt)
def collect_packets_rtt(self, packets):
"""
Collects all the rrts of given packets and stores them
in our rtts list.
"""
for packet in packets:
rtt = packet.rtt
if not packet.rtt:
rtt = 0
self.rtts.append(rtt)
def calculate_loss(self):
"""Calculates the total loss between received and sent packets."""
if not self.sent_packets:
return 0
return (1 - float(self.received_packets) / self.sent_packets) * 100
def mean(self):
"""Calculates the mean of the collected rtts"""
return round(float(sum(self.rtts)) / max(len(self.rtts), 1), 3)
def median(self):
"""Calculates the median of the collected rtts"""
sorted_rtts = sorted(self.rtts)
index = (len(self.rtts) - 1) // 2
if len(self.rtts) % 2:
return sorted_rtts[index]
else:
return (sorted_rtts[index] + sorted_rtts[index + 1]) / 2.0
def on_result(self, result):
packets = result.packets
if not packets:
return "No packets found\n"
self.collect_stats(result)
# Because the origin value is more reliable as "from" in v4 and as
# "packet.source_address" in v6.
origin = result.origin
if ":" in origin:
origin = packets[0].source_address
times = " ".join(["{:11}".format(str(_.rtt) + " ms") for _ in packets])
f = io.open('/path/to/probes.json','r',encoding='utf-8')
probes = json.load(f)
return (
f"{result.packet_size} bytes from {result.destination_address} via "
f"probe #{result.probe_id:<7} {probes[str(result.probe_id)]['country_code']:<8}({origin:15})"
f": ttl={packets[0].ttl} times={times}\n"
)
def header(self, sample):
resolved_on = (
"server"
if sample.destination_address == sample.destination_name
else "probe"
)
return f"PING {sample.destination_name} (resolved on {resolved_on})\n"
def footer(self):
if not self.sent_packets:
return ""
self.packet_loss = self.calculate_loss()
return self.render_template(
"reports/aggregate_ping.txt",
target=sanitise(self.target),
sent=self.sent_packets,
received=self.received_packets,
packet_loss=self.packet_loss,
min=min(self.rtts_min),
median=self.median(),
mean=self.mean(),
max=max(self.rtts_max),
)
顺便说几句,RIPE 全网 Probes 30885 个,已连接 9347 个,可供 ICMP 测试的 6096 个。中国的一共 118 个,截止目前已连接的才 21 个。
可供 ICMP 测试(system-ipv4-works)稳定在线 1 天(system-ipv4-stable-1d 24*5%=1.2h/d)及以上的有 5 个,30 天及以上的只有 2 个(system-ipv4-stable-30d 30*24*5%=36h/m)。
关于 tags 描述可以看这里,国际互联网链接质量堪忧啊……?
根据朋友描述,申请硬件 Probe 不是很容易,而且因为是平邮,即使在地址上写明手机号,也不一定会收到……
但目前 RIPE 提供了软件 Probes,申请还是比较容易的,之前申请被拒的可以考虑申请软件版本。
软件 Probes 对 CentOS 7 提供了 rpm 的安装方式,其他系统需要自行编译安装或者使用 Docker 运行。
申请地址:
https://labs.ripe.net/Members/alun_davies/ripe-atlas-software-probes