From 3fad0f1a249d3a659919f9406581cd6f503bdf40 Mon Sep 17 00:00:00 2001 From: Dan Poltawski Date: Thu, 26 Nov 2015 13:00:20 +0000 Subject: [PATCH 1/1] MDL-48766 iplookup: Update to geoip2 db to support ipv6 The previous maxmind geoip database is now legacy and the GeoIP2 version supports ipv6. --- admin/settings/location.php | 3 +- iplookup/index.php | 10 ++---- iplookup/lib.php | 49 ++++++++++++++--------------- iplookup/tests/geoip_test.php | 51 +++++++++++++++++++++++-------- iplookup/tests/geoplugin_test.php | 13 ++++++-- lang/en/admin.php | 8 ++--- 6 files changed, 81 insertions(+), 53 deletions(-) diff --git a/admin/settings/location.php b/admin/settings/location.php index 1868f3305d0..504cbb9aa06 100644 --- a/admin/settings/location.php +++ b/admin/settings/location.php @@ -10,7 +10,8 @@ if ($hassiteconfig) { // speedup for non-admins, add all caps used on this page $temp->add(new admin_setting_configtext('defaultcity', new lang_string('defaultcity', 'admin'), new lang_string('defaultcity_help', 'admin'), '')); $temp->add(new admin_setting_heading('iplookup', new lang_string('iplookup', 'admin'), new lang_string('iplookupinfo', 'admin'))); - $temp->add(new admin_setting_configfile('geoipfile', new lang_string('geoipfile', 'admin'), new lang_string('configgeoipfile', 'admin', $CFG->dataroot.'/geoip/'), $CFG->dataroot.'/geoip/GeoLiteCity.dat')); + $temp->add(new admin_setting_configfile('geoip2file', new lang_string('geoipfile', 'admin'), + new lang_string('configgeoipfile', 'admin', $CFG->dataroot.'/geoip/'), $CFG->dataroot.'/geoip/GeoLite2-City.mmdb')); $temp->add(new admin_setting_configtext('googlemapkey3', new lang_string('googlemapkey3', 'admin'), new lang_string('googlemapkey3_help', 'admin'), '', PARAM_RAW, 60)); $temp->add(new admin_setting_configtext('allcountrycodes', new lang_string('allcountrycodes', 'admin'), new lang_string('configallcountrycodes', 'admin'), '', '/^(?:\w+(?:,\w+)*)?$/')); diff --git a/iplookup/index.php b/iplookup/index.php index 07981ed169d..c285ee79169 100644 --- a/iplookup/index.php +++ b/iplookup/index.php @@ -33,7 +33,7 @@ if (isguestuser()) { throw new require_login_exception('Guests are not allowed here.'); } -$ip = optional_param('ip', getremoteaddr(), PARAM_HOST); +$ip = optional_param('ip', getremoteaddr(), PARAM_RAW); $user = optional_param('user', 0, PARAM_INT); if (isset($CFG->iplookup)) { @@ -48,15 +48,11 @@ $PAGE->set_context(context_system::instance()); $info = array($ip); $note = array(); -if (!preg_match('/(^\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/', $ip, $match)) { +if (cleanremoteaddr($ip) === false) { print_error('invalidipformat', 'error'); } -if ($match[1] > 255 or $match[2] > 255 or $match[3] > 255 or $match[4] > 255) { - print_error('invalidipformat', 'error'); -} - -if ($match[1] == '127' or $match[1] == '10' or ($match[1] == '172' and $match[2] >= '16' and $match[2] <= '31') or ($match[1] == '192' and $match[2] == '168')) { +if (!ip_is_public($ip)) { print_error('iplookupprivate', 'error'); } diff --git a/iplookup/lib.php b/iplookup/lib.php index da07be6ffdf..4afeb3f89ef 100644 --- a/iplookup/lib.php +++ b/iplookup/lib.php @@ -36,39 +36,30 @@ function iplookup_find_location($ip) { $info = array('city'=>null, 'country'=>null, 'longitude'=>null, 'latitude'=>null, 'error'=>null, 'note'=>'', 'title'=>array()); - if (!empty($CFG->geoipfile) and file_exists($CFG->geoipfile)) { - require_once('Net/GeoIP.php'); + if (!empty($CFG->geoip2file) and file_exists($CFG->geoip2file)) { + $reader = new GeoIp2\Database\Reader($CFG->geoip2file); + $record = $reader->city($ip); - $geoip = Net_GeoIP::getInstance($CFG->geoipfile, Net_GeoIP::STANDARD); - $location = $geoip->lookupLocation($ip); - $geoip->close(); - - if (empty($location)) { + if (empty($record)) { $info['error'] = get_string('iplookupfailed', 'error', $ip); return $info; } - if (!empty($location->city)) { - $info['city'] = core_text::convert($location->city, 'iso-8859-1', 'utf-8'); - $info['title'][] = $info['city']; - } - if (!empty($location->countryCode)) { - $countries = get_string_manager()->get_list_of_countries(true); - if (isset($countries[$location->countryCode])) { - // prefer our localized country names - $info['country'] = $countries[$location->countryCode]; - } else { - $info['country'] = $location->countryName; - } - $info['title'][] = $info['country']; - - } else if (!empty($location->countryName)) { - $info['country'] = $location->countryName; - $info['title'][] = $info['country']; + $info['city'] = core_text::convert($record->city->name, 'iso-8859-1', 'utf-8'); + $info['title'][] = $info['city']; + + $countrycode = $record->country->isoCode; + $countries = get_string_manager()->get_list_of_countries(true); + if (isset($countries[$countrycode])) { + // Prefer our localized country names. + $info['country'] = $countries[$countrycode]; + } else { + $info['country'] = $record->country->names['en']; } + $info['title'][] = $info['country']; - $info['longitude'] = $location->longitude; - $info['latitude'] = $location->latitude; + $info['longitude'] = $record->location->longitude; + $info['latitude'] = $record->location->latitude; $info['note'] = get_string('iplookupmaxmindnote', 'admin'); return $info; @@ -76,6 +67,12 @@ function iplookup_find_location($ip) { } else { require_once($CFG->libdir.'/filelib.php'); + if (strpos($ip, ':') !== false) { + // IPv6 is not supported by geoplugin.net. + $info['error'] = get_string('invalidipformat', 'error'); + return $info; + } + $ipdata = download_file_content('http://www.geoplugin.net/json.gp?ip='.$ip); if ($ipdata) { $ipdata = preg_replace('/^geoPlugin\((.*)\)\s*$/s', '$1', $ipdata); diff --git a/iplookup/tests/geoip_test.php b/iplookup/tests/geoip_test.php index 3c21394c4e3..ef5e2909fc0 100644 --- a/iplookup/tests/geoip_test.php +++ b/iplookup/tests/geoip_test.php @@ -31,20 +31,20 @@ defined('MOODLE_INTERNAL') || die(); */ class core_iplookup_geoip_testcase extends advanced_testcase { - public function test_geoip() { + public function setUp() { global $CFG; require_once("$CFG->libdir/filelib.php"); require_once("$CFG->dirroot/iplookup/lib.php"); if (!PHPUNIT_LONGTEST) { // this may take a long time - return; + $this->markTestSkipped('PHPUNIT_LONGTEST is not defined'); } $this->resetAfterTest(); // let's store the file somewhere - $gzfile = "$CFG->dataroot/phpunit/geoip/GeoLiteCity.dat.gz"; + $gzfile = "$CFG->dataroot/phpunit/geoip/GeoLite2-City.mmdb.gz"; check_dir_exists(dirname($gzfile)); if (file_exists($gzfile) and (filemtime($gzfile) < time() - 60*60*24*30)) { // delete file if older than 1 month @@ -52,24 +52,35 @@ class core_iplookup_geoip_testcase extends advanced_testcase { } if (!file_exists($gzfile)) { - download_file_content('http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz', null, null, false, 300, 20, false, $gzfile); + download_file_content('http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz', + null, null, false, 300, 20, false, $gzfile); } $this->assertTrue(file_exists($gzfile)); - $zd = gzopen($gzfile, "r"); - $contents = gzread($zd, 50000000); - gzclose($zd); + $geoipfile = str_replace('.gz', '', $gzfile); - $geoipfile = "$CFG->dataroot/geoip/GeoLiteCity.dat"; - check_dir_exists(dirname($geoipfile)); - $fp = fopen($geoipfile, 'w'); - fwrite($fp, $contents); - fclose($fp); + // Open our files (in binary mode). + $file = gzopen($gzfile, 'rb'); + $geoipfilebuf = fopen($geoipfile, 'wb'); + + // Keep repeating until the end of the input file. + while (!gzeof($file)) { + // Read buffer-size bytes. + // Both fwrite and gzread and binary-safe. + fwrite($geoipfilebuf, gzread($file, 4096)); + } + + // Files are done, close files. + fclose($geoipfilebuf); + gzclose($file); $this->assertTrue(file_exists($geoipfile)); - $CFG->geoipfile = $geoipfile; + $CFG->geoip2file = $geoipfile; + } + + public function test_ipv4() { $result = iplookup_find_location('147.230.16.1'); @@ -82,5 +93,19 @@ class core_iplookup_geoip_testcase extends advanced_testcase { $this->assertEquals('Liberec', $result['title'][0]); $this->assertEquals('Czech Republic', $result['title'][1]); } + + public function test_ipv6() { + + $result = iplookup_find_location('2a01:8900:2:3:8c6c:c0db:3d33:9ce6'); + + $this->assertEquals('array', gettype($result)); + $this->assertEquals('Lancaster', $result['city']); + $this->assertEquals(-2.79970, $result['longitude'], '', 0.001); + $this->assertEquals(54.04650, $result['latitude'], '', 0.001); + $this->assertNull($result['error']); + $this->assertEquals('array', gettype($result['title'])); + $this->assertEquals('Lancaster', $result['title'][0]); + $this->assertEquals('United Kingdom', $result['title'][1]); + } } diff --git a/iplookup/tests/geoplugin_test.php b/iplookup/tests/geoplugin_test.php index bf546970eb5..289058e551c 100644 --- a/iplookup/tests/geoplugin_test.php +++ b/iplookup/tests/geoplugin_test.php @@ -31,20 +31,22 @@ defined('MOODLE_INTERNAL') || die(); */ class core_iplookup_geoplugin_testcase extends advanced_testcase { - public function test_geoip() { + public function setUp() { global $CFG; require_once("$CFG->libdir/filelib.php"); require_once("$CFG->dirroot/iplookup/lib.php"); if (!PHPUNIT_LONGTEST) { // we do not want to DDOS their server, right? - return; + $this->markTestSkipped('PHPUNIT_LONGTEST is not defined'); } $this->resetAfterTest(); $CFG->geoipfile = ''; + } + public function test_geoip_ipv4() { $result = iplookup_find_location('147.230.16.1'); $this->assertEquals('array', gettype($result)); @@ -56,5 +58,12 @@ class core_iplookup_geoplugin_testcase extends advanced_testcase { $this->assertEquals('Liberec', $result['title'][0]); $this->assertEquals('Czech Republic', $result['title'][1]); } + + public function test_geoip_ipv6() { + $result = iplookup_find_location('2a01:8900:2:3:8c6c:c0db:3d33:9ce6'); + + $this->assertNotNull($result['error']); + $this->assertEquals($result['error'], get_string('invalidipformat', 'error')); + } } diff --git a/lang/en/admin.php b/lang/en/admin.php index 52f555bfb2b..e3fd072f699 100644 --- a/lang/en/admin.php +++ b/lang/en/admin.php @@ -230,7 +230,7 @@ $string['configfrontpageloggedin'] = 'The items selected above will be displayed $string['configfullnamedisplay'] = 'This defines how names are shown when they are displayed in full. The default value, "language", leaves it to the string "fullnamedisplay" in the current language pack to decide. Some languages have different name display conventions. For most mono-lingual sites the most efficient setting is "firstname lastname", but you may choose to hide surnames altogether. Placeholders that can be used are: firstname, lastname, firstnamephonetic, lastnamephonetic, middlename, and alternatename.'; -$string['configgeoipfile'] = 'Location of GeoIP City binary data file. This file is not part of Moodle distribution and must be obtained separately from MaxMind. You can either buy a commercial version or use the free version. Simply download http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz and extract it into "{$a}" directory on your server.'; +$string['configgeoipfile'] = 'Location of GeoLite2 City binary data file. This file is not part of Moodle distribution and must be obtained separately from MaxMind. You can either buy a commercial version or use the free version. Simply download http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz and extract it into "{$a}" directory on your server.'; $string['configgetremoteaddrconf'] = 'If your server is behind a reverse proxy, you can use this setting to specify which HTTP headers can be trusted to contain the remote IP address. The headers are read in order, using the first one that is available.'; $string['configgradebookroles'] = 'This setting allows you to control who appears on the gradebook. Users need to have at least one of these roles in a course to be shown in the gradebook for that course.'; $string['configgradeexport'] = 'Choose which gradebook export formats are your primary methods for exporting grades. Chosen plugins will then set and use a "last exported" field for every grade. For example, this might result in exported records being identified as being "new" or "updated". If you are not sure about this then leave everything unchecked.'; @@ -540,7 +540,7 @@ $string['fullnamedisplay'] = 'Full name format'; $string['fullnamedisplayprivate'] = 'Full name format - private'; $string['gdrequired'] = 'The GD extension is now required by Moodle for image conversion.'; $string['generalsettings'] = 'General settings'; -$string['geoipfile'] = 'GeoIP city data file'; +$string['geoipfile'] = 'GeoLite2 City MaxMind DB'; $string['getremoteaddrconf'] = 'Logged IP address source'; $string['globalsearch'] = 'Global search'; $string['globalsearchmanage'] = 'Manage global search'; @@ -598,9 +598,9 @@ $string['ipblockersyntax'] = 'Put every entry on one line. Valid entries are eit $string['iplookup'] = 'IP address lookup'; $string['iplookupgeoplugin'] = 'geoPlugin service is currently being used to look up geographical information. For more accurate results we recommend installing a local copy of the MaxMind GeoLite database.'; $string['iplookupinfo'] = 'By default Moodle uses the free online NetGeo (The Internet Geographic Database) server to lookup location of IP addresses, unfortunately this database is not maintained anymore and may return wildly incorrect data. -It is recommended to install local copy of free GeoLite City database from MaxMind.
+It is recommended to install local copy of free GeoLite2 City database from MaxMind.
IP address location is displayed on simple map or using Google Maps. Please note that you need to have a Google account and apply for free Google Maps API key to enable interactive maps.'; -$string['iplookupmaxmindnote'] = 'This product includes GeoLite data created by MaxMind, available from http://www.maxmind.com/.'; +$string['iplookupmaxmindnote'] = 'This product includes GeoLite2 data created by MaxMind, available from http://www.maxmind.com.'; $string['keeptagnamecase'] = 'Keep tag name casing'; $string['lang'] = 'Default language'; $string['langcache'] = 'Cache language menu'; -- 2.17.1