PHP 爬虫记录

数据库设计

 

create table crawler      
(      
    crawler_ID bigint(20) unsigned not null auto_increment primary key, 
    crawler_category varchar(20) not null, 
    crawler_date datetime not null default '0000-00-00 00:00:00',
    crawler_url varchar(50) not null, 
    crawler_IP varchar(50) not null
)default charset=utf8; 

 

 以下文件 robot.php 记录来访的爬虫，并将信息写入数据库：

 

<?php
    $ServerName = $_SERVER["SERVER_NAME"] ;     
	$ServerPort = $_SERVER["SERVER_PORT"] ;     
	$ScriptName = $_SERVER["SCRIPT_NAME"] ;     
	$QueryString = $_SERVER["QUERY_STRING"];    
	$serverip = $_SERVER["REMOTE_ADDR"] ;   
	$Url="http://".$ServerName;
	if ($ServerPort != "80") 
	{
		$Url = $Url.":".$ServerPort ; 
	}    
	$Url=$Url.$ScriptName;
	if ($QueryString !="")
	{
		$Url=$Url."?".$QueryString; 
	}      
	$GetLocationURL=$Url ;
    $agent1 = $_SERVER["HTTP_USER_AGENT"];  
	$agent=strtolower($agent1);
	$Bot ="";
    if (strpos($agent,"bot")>-1)
	{
		$Bot = "Other Crawler";
	}
	if (strpos($agent,"googlebot")>-1)
	{
		$Bot = "Google";
	}           
    if (strpos($agent,"mediapartners-google")>-1)
	{
		$Bot = "Google Adsense";
	}
	if (strpos($agent,"baiduspider")>-1)
	{
		$Bot = "Baidu";
	}
	if (strpos($agent,"sogou spider")>-1)
	{
		$Bot = "Sogou";
	}
	if (strpos($agent,"yahoo")>-1)
	{
		$Bot = "Yahoo!";
	}
	if (strpos($agent,"msn")>-1)
	{
		$Bot = "MSN";
	}
	if (strpos($agent,"ia_archiver")>-1)
	{
		$Bot = "Alexa";
	}
	if (strpos($agent,"iaarchiver")>-1)
	{
		$Bot = "Alexa";
	}
	if (strpos($agent,"sohu")>-1)
	{
		$Bot = "Sohu";
	}
	if (strpos($agent,"sqworm")>-1) 
	{
		$Bot = "AOL";
	}
	if (strpos($agent,"yodaoBot")>-1)
	{
		$Bot = "Yodao";
	}
	if (strpos($agent,"iaskspider")>-1)
	{
		$Bot = "Iask";
	}
	   
	require("./dbinfo.php");
	
    date_default_timezone_set('PRC');  
	$shijian=date("Y-m-d h:i:s", time());
	
	// 连接到 MySQL 服务器
	$connection = mysql_connect ($host, $username, $password);
	if (!$connection) 
	{
  		die('Not connected : ' . mysql_error());
	}
	
	// 设置活动的 MySQL 数据库
	$db_selected = mysql_select_db($database, $connection);
	if (!$db_selected) 
	{
  		die ('Can\'t use db : ' . mysql_error());
	}
	
	// 向数据库插入数据
	$query = "insert into crawler (crawler_category, crawler_date, crawler_url, crawler_IP) values ('$Bot','$shijian','$GetLocationURL','$serverip')"; 
	
	$result = mysql_query($query);
	if (!$result) 
	{
  		die('Invalid query: ' . mysql_error());
	}
	 
?>

 

 成功了，现在访问数据库即可得知什么时候哪里的蜘蛛爬过你的什么页面。

 

<?php
include './robot.php';
include '../library/page.Class.php';
$page = $_GET['page'];


include '../library/conn_new.php';
		
$count = $mysql -> num_rows($mysql -> query("select *  from crawler"));
$pages = new PageClass($count,25,$_GET['page'],$_SERVER['PHP_SELF'].'?page={page}');
$sql  = "select * from crawler order by ";
$sql .= "crawler_date desc limit ".$pages -> page_limit.",".$pages -> myde_size;
$result = $mysql -> query($sql);
?>

<table width="700">
	<thead>
        <tr>  
        	<td bgcolor="#CCFFFF"></td>  
        	<td bgcolor="#CCFFFF" align="center" style="color:#555">爬虫访问时间</td>  
        	<td bgcolor="#CCFFFF" align="center" style="color:#555">爬虫分类</td> 
        	<td bgcolor="#CCFFFF" align="center" style="color:#555">爬虫IP</td> 
            <td bgcolor="#CCFFFF" align="center" style="color:#555">爬虫访问的URL</td> 
 		</tr> 
	</thead>
      
<?php
while($myrow = $mysql -> fetch_array($result)){
?>
<tr>
    <td width="30"><img src="../images/topicnew.gif" /></td>
    <td width="150" style="font-family:Georgia"><? echo $myrow["crawler_date"] ?></td>
    <td width="100" style="color:#5F7A77"><? echo $myrow["crawler_category"] ?></td>
    <td width="100"><? echo $myrow["crawler_IP"] ?></td>
    <td width="320"><? echo $myrow["crawler_url"] ?></td>
</tr>
<?php
	}
?>
    </table>
<?php
  	echo $pages -> myde_write();
?>