<?php require_once(dirname(__FILE__) . '/wfUtils.php'); class wfCrawl { const GOOGLE_BOT_VERIFIED = 'verified'; const GOOGLE_BOT_FAKE = 'fakeBot'; const GOOGLE_BOT_UNDETERMINED = 'undetermined'; public static function isCrawler($UA){ $browscap = new wfBrowscap(); $b = $browscap->getBrowser($UA); if (!$b || $b['Parent'] == 'DefaultProperties') { $IP = wfUtils::getIP(); return !wfLog::isHumanRequest($IP, $UA); } else if (isset($b['Crawler']) && $b['Crawler']) { return true; } return false; } public static function verifyCrawlerPTR($hostPattern, $IP){ $table = wfDB::networkTable('wfCrawlers'); $db = new wfDB(); $IPn = wfUtils::inet_pton($IP); $ipHex = wfDB::binaryValueToSQLHex(wfUtils::inet_pton($IPn)); $status = $db->querySingle("select status from $table where IP={$ipHex} and patternSig=UNHEX(MD5('%s')) and lastUpdate > unix_timestamp() - %d", $hostPattern, WORDFENCE_CRAWLER_VERIFY_CACHE_TIME); if($status){ if($status == 'verified'){ return true; } else { return false; } } $host = wfUtils::reverseLookup($IP); if(! $host){ $db->queryWrite("insert into $table (IP, patternSig, status, lastUpdate, PTR) values ({$ipHex}, UNHEX(MD5('%s')), '%s', unix_timestamp(), '%s') ON DUPLICATE KEY UPDATE status='%s', lastUpdate=unix_timestamp(), PTR='%s'", $hostPattern, 'noPTR', '', 'noPTR', ''); return false; } if(preg_match($hostPattern, $host)){ $resultIPs = wfUtils::resolveDomainName($host); $addrsMatch = false; foreach($resultIPs as $resultIP){ if($resultIP == $IP){ $addrsMatch = true; break; } } if($addrsMatch){ $db->queryWrite("insert into $table (IP, patternSig, status, lastUpdate, PTR) values ({$ipHex}, UNHEX(MD5('%s')), '%s', unix_timestamp(), '%s') ON DUPLICATE KEY UPDATE status='%s', lastUpdate=unix_timestamp(), PTR='%s'", $hostPattern, 'verified', $host, 'verified', $host); return true; } else { $db->queryWrite("insert into $table (IP, patternSig, status, lastUpdate, PTR) values ({$ipHex}, UNHEX(MD5('%s')), '%s', unix_timestamp(), '%s') ON DUPLICATE KEY UPDATE status='%s', lastUpdate=unix_timestamp(), PTR='%s'", $hostPattern, 'fwdFail', $host, 'fwdFail', $host); return false; } } else { $db->queryWrite("insert into $table (IP, patternSig, status, lastUpdate, PTR) values ({$ipHex}, UNHEX(MD5('%s')), '%s', unix_timestamp(), '%s') ON DUPLICATE KEY UPDATE status='%s', lastUpdate=unix_timestamp(), PTR='%s'", $hostPattern, 'badPTR', $host, 'badPTR', $host); return false; } } public static function isGooglebot($userAgent = null){ if ($userAgent === null) { $userAgent = isset($_SERVER['HTTP_USER_AGENT']) ? $_SERVER['HTTP_USER_AGENT'] : ''; } return (bool) preg_match('/Googlebot\/\d\.\d/', $userAgent); } public static function isGoogleCrawler($userAgent = null){ if ($userAgent === null) { $userAgent = isset($_SERVER['HTTP_USER_AGENT']) ? $_SERVER['HTTP_USER_AGENT'] : ''; } foreach (self::$googPat as $pat) { if (preg_match($pat . 'i', $userAgent)) { return true; } } return false; } private static $googPat = array( '@^Mozilla/5\\.0 \\(.*Google Keyword Tool.*\\)$@', '@^Mozilla/5\\.0 \\(.*Feedfetcher\\-Google.*\\)$@', '@^Feedfetcher\\-Google\\-iGoogleGadgets.*$@', '@^searchbot admin\\@google\\.com$@', '@^Google\\-Site\\-Verification.*$@', '@^Google OpenSocial agent.*$@', '@^.*Googlebot\\-Mobile/2\\..*$@', '@^AdsBot\\-Google\\-Mobile.*$@', '@^google \\(.*Enterprise.*\\)$@', '@^Mediapartners\\-Google.*$@', '@^GoogleFriendConnect.*$@', '@^googlebot\\-urlconsole$@', '@^.*Google Web Preview.*$@', '@^Feedfetcher\\-Google.*$@', '@^AppEngine\\-Google.*$@', '@^Googlebot\\-Video.*$@', '@^Googlebot\\-Image.*$@', '@^Google\\-Sitemaps.*$@', '@^Googlebot/Test.*$@', '@^Googlebot\\-News.*$@', '@^.*Googlebot/2\\.1.*$@', '@^AdsBot\\-Google.*$@', '@^Google$@' ); /** * Has correct user agent and PTR record points to .googlebot.com domain. * * @param string|null $ip * @param string|null $ua * @return bool */ public static function isVerifiedGoogleCrawler($ip = null, $ua = null) { static $verified; if (!isset($verified)) { $verified = array(); } if ($ip === null) { $ip = wfUtils::getIP(); } if ($ip === null || $ip === false) { //Likely a CLI execution return false; } if (array_key_exists($ip, $verified)) { return $verified[$ip]; } if (self::isGoogleCrawler($ua)) { if (self::verifyCrawlerPTR(wordfence::getLog()->getGooglePattern(), $ip)) { $verified[$ip] = true; return $verified[$ip]; } $noc1Status = self::verifyGooglebotViaNOC1($ip); if ($noc1Status == self::GOOGLE_BOT_VERIFIED) { $verified[$ip] = true; return $verified[$ip]; } else if ($noc1Status == self::GOOGLE_BOT_FAKE) { $verified[$ip] = false; return $verified[$ip]; } return true; //We were unable to successfully validate Googlebot status so default to being permissive } $verified[$ip] = false; return $verified[$ip]; } /** * Attempts to verify whether an IP claiming to be Googlebot is actually Googlebot. * * @param string|null $ip * @return string */ public static function verifyGooglebotViaNOC1($ip = null) { $table = wfDB::networkTable('wfCrawlers'); if ($ip === null) { $ip = wfUtils::getIP(); } $db = new wfDB(); $IPn = wfUtils::inet_pton($ip); $ipHex = wfDB::binaryValueToSQLHex($IPn); $patternSig = 'googlenoc1'; $status = $db->querySingle("select status from $table where IP={$ipHex} and patternSig=UNHEX(MD5('%s')) and lastUpdate > unix_timestamp() - %d", $patternSig, WORDFENCE_CRAWLER_VERIFY_CACHE_TIME); if ($status === 'verified') { return self::GOOGLE_BOT_VERIFIED; } else if ($status === 'fakeBot') { return self::GOOGLE_BOT_FAKE; } $api = new wfAPI(wfConfig::get('apiKey'), wfUtils::getWPVersion()); try { $data = $api->call('verify_googlebot', array( 'ip' => $ip, )); if (is_array($data) && !empty($data['verified'])) { // Cache results $db->queryWrite("INSERT INTO {$table} (IP, patternSig, status, lastUpdate) VALUES ({$ipHex}, UNHEX(MD5('%s')), '%s', unix_timestamp()) ON DUPLICATE KEY UPDATE status = VALUES(status), lastUpdate = VALUES(lastUpdate)", $patternSig, 'verified'); return self::GOOGLE_BOT_VERIFIED; } else { $db->queryWrite("INSERT INTO {$table} (IP, patternSig, status, lastUpdate) VALUES ({$ipHex}, UNHEX(MD5('%s')), '%s', unix_timestamp()) ON DUPLICATE KEY UPDATE status = VALUES(status), lastUpdate = VALUES(lastUpdate)", $patternSig, 'fakeBot'); self::GOOGLE_BOT_FAKE; } } catch (Exception $e) { // Do nothing, bail } return self::GOOGLE_BOT_UNDETERMINED; } }
Name | Type | Size | Permission | Actions |
---|---|---|---|---|
Diff | Folder | 0755 |
|
|
audit-log | Folder | 0755 |
|
|
dashboard | Folder | 0755 |
|
|
rest-api | Folder | 0755 |
|
|
.htaccess | File | 354 B | 0644 |
|
Diff.php | File | 5.63 KB | 0644 |
|
GeoLite2-Country.mmdb | File | 7.46 MB | 0644 |
|
IPTraf.php | File | 1.17 KB | 0644 |
|
IPTrafList.php | File | 2.98 KB | 0644 |
|
WFLSPHP52Compatability.php | File | 1.27 KB | 0644 |
|
compat.php | File | 425 B | 0644 |
|
diffResult.php | File | 2.81 KB | 0644 |
|
email_genericAlert.php | File | 1.39 KB | 0644 |
|
email_newIssues.php | File | 8.82 KB | 0644 |
|
email_unlockRequest.php | File | 2.34 KB | 0644 |
|
email_unsubscribeRequest.php | File | 1.05 KB | 0644 |
|
flags.php | File | 6.62 KB | 0644 |
|
live_activity.php | File | 580 B | 0644 |
|
menu_dashboard.php | File | 28 KB | 0644 |
|
menu_dashboard_options.php | File | 15.21 KB | 0644 |
|
menu_firewall.php | File | 2.12 KB | 0644 |
|
menu_firewall_blocking.php | File | 10.25 KB | 0644 |
|
menu_firewall_blocking_options.php | File | 4.63 KB | 0644 |
|
menu_firewall_waf.php | File | 19.96 KB | 0644 |
|
menu_firewall_waf_options.php | File | 11.09 KB | 0644 |
|
menu_install.php | File | 1.73 KB | 0644 |
|
menu_options.php | File | 24.7 KB | 0644 |
|
menu_scanner.php | File | 21.53 KB | 0644 |
|
menu_scanner_credentials.php | File | 2.77 KB | 0644 |
|
menu_scanner_options.php | File | 8.41 KB | 0644 |
|
menu_support.php | File | 17.82 KB | 0644 |
|
menu_tools.php | File | 1.49 KB | 0644 |
|
menu_tools_auditlog.php | File | 16.43 KB | 0644 |
|
menu_tools_diagnostic.php | File | 49.35 KB | 0644 |
|
menu_tools_importExport.php | File | 1.28 KB | 0644 |
|
menu_tools_livetraffic.php | File | 39.43 KB | 0644 |
|
menu_tools_twoFactor.php | File | 19.6 KB | 0644 |
|
menu_tools_whois.php | File | 4.61 KB | 0644 |
|
menu_wordfence_central.php | File | 9.66 KB | 0644 |
|
noc1.key | File | 1.64 KB | 0644 |
|
sodium_compat_fast.php | File | 185 B | 0644 |
|
sysinfo.php | File | 1.47 KB | 0644 |
|
viewFullActivityLog.php | File | 1.47 KB | 0644 |
|
wf503.php | File | 9.63 KB | 0644 |
|
wfAPI.php | File | 9.73 KB | 0644 |
|
wfActivityReport.php | File | 20.45 KB | 0644 |
|
wfAdminNoticeQueue.php | File | 5.2 KB | 0644 |
|
wfAlerts.php | File | 7.37 KB | 0644 |
|
wfArray.php | File | 1.77 KB | 0644 |
|
wfAuditLog.php | File | 47.13 KB | 0644 |
|
wfBrowscap.php | File | 3.9 KB | 0644 |
|
wfBrowscapCache.php | File | 256.83 KB | 0644 |
|
wfBulkCountries.php | File | 9.77 KB | 0644 |
|
wfCache.php | File | 6.02 KB | 0644 |
|
wfCentralAPI.php | File | 25.8 KB | 0644 |
|
wfConfig.php | File | 122.49 KB | 0644 |
|
wfCrawl.php | File | 6.56 KB | 0644 |
|
wfCredentialsController.php | File | 5.16 KB | 0644 |
|
wfCrypt.php | File | 4.05 KB | 0644 |
|
wfCurlInterceptor.php | File | 1.02 KB | 0644 |
|
wfDB.php | File | 11.49 KB | 0644 |
|
wfDashboard.php | File | 8.2 KB | 0644 |
|
wfDateLocalization.php | File | 352.13 KB | 0644 |
|
wfDeactivationOption.php | File | 2.13 KB | 0644 |
|
wfDiagnostic.php | File | 66.87 KB | 0644 |
|
wfDict.php | File | 738 B | 0644 |
|
wfDirectoryIterator.php | File | 1.89 KB | 0644 |
|
wfFileUtils.php | File | 2.72 KB | 0644 |
|
wfHelperBin.php | File | 1.97 KB | 0644 |
|
wfHelperString.php | File | 2.13 KB | 0644 |
|
wfIPWhitelist.php | File | 1.56 KB | 0644 |
|
wfImportExportController.php | File | 3.23 KB | 0644 |
|
wfInaccessibleDirectoryException.php | File | 303 B | 0644 |
|
wfInvalidPathException.php | File | 266 B | 0644 |
|
wfIpLocation.php | File | 1.73 KB | 0644 |
|
wfIpLocator.php | File | 2.74 KB | 0644 |
|
wfIssues.php | File | 27.91 KB | 0644 |
|
wfJWT.php | File | 5.33 KB | 0644 |
|
wfLicense.php | File | 10.43 KB | 0644 |
|
wfLockedOut.php | File | 9.73 KB | 0644 |
|
wfLog.php | File | 57.1 KB | 0644 |
|
wfMD5BloomFilter.php | File | 5.2 KB | 0644 |
|
wfModuleController.php | File | 754 B | 0644 |
|
wfNotification.php | File | 6.41 KB | 0644 |
|
wfOnboardingController.php | File | 9.22 KB | 0644 |
|
wfPersistenceController.php | File | 819 B | 0644 |
|
wfRESTAPI.php | File | 377 B | 0644 |
|
wfScan.php | File | 15.92 KB | 0644 |
|
wfScanEngine.php | File | 133.31 KB | 0644 |
|
wfScanEntrypoint.php | File | 1.04 KB | 0644 |
|
wfScanFile.php | File | 1.01 KB | 0644 |
|
wfScanFileLink.php | File | 403 B | 0644 |
|
wfScanFileListItem.php | File | 408 B | 0644 |
|
wfScanFileProperties.php | File | 1.07 KB | 0644 |
|
wfScanMonitor.php | File | 4.05 KB | 0644 |
|
wfScanPath.php | File | 1.77 KB | 0644 |
|
wfSchema.php | File | 10.91 KB | 0644 |
|
wfStyle.php | File | 1.21 KB | 0644 |
|
wfSupportController.php | File | 24.18 KB | 0644 |
|
wfUnlockMsg.php | File | 1.14 KB | 0644 |
|
wfUpdateCheck.php | File | 27.23 KB | 0644 |
|
wfUtils.php | File | 124.11 KB | 0644 |
|
wfVersionCheckController.php | File | 19.27 KB | 0644 |
|
wfVersionSupport.php | File | 535 B | 0644 |
|
wfView.php | File | 2.22 KB | 0644 |
|
wfViewResult.php | File | 1.42 KB | 0644 |
|
wfWebsite.php | File | 1.75 KB | 0644 |
|
wordfenceClass.php | File | 435.98 KB | 0644 |
|
wordfenceConstants.php | File | 3.56 KB | 0644 |
|
wordfenceHash.php | File | 42.7 KB | 0644 |
|
wordfenceScanner.php | File | 30.47 KB | 0644 |
|
wordfenceURLHoover.php | File | 18.36 KB | 0644 |
|