snoopy采集phpchina示例

snoopy采集phpchina示例
<?php
//采集phpchina

set_time_limit(0);

require_once("Snoopy.class.php");

$snoopy=new Snoopy();

//登陆论坛

$submit_url = "http://www.phpchina.com/bbs/logging.php?action=lo gin";

$submit_vars["loginmode"] = "normal";

$submit_vars["styleid"] = "1";

$submit_vars["cookietime"] = "";

$submit_vars["loginfield"] = "username";

$submit_vars["username"] = "***"; //你的用户名

$submit_vars["password"] = "*****"; //你的密码

$submit_vars["questionid"] = "0";

$submit_vars["answer"] = "";

$submit_vars["loginsubmit"] = "提交";

$snoopy->submit($submit_url,$submit_vars);

$snoopy->fetchlinks("http://www.phpchina.com/bbs");

$url=array();

$url=$snoopy->results;

//print_r($url);

foreach ($url as $key=>$value)

{

//匹配http://www.phpchina.com/bbs/forumdisplay.php?fid=1 56&sid=VfcqTR地址即论坛板块地址

if (!preg_match("/^(http:\/\/www\.phpchina\.com\/bbs\ /forumdisplay\.php\?fid=)[0-9]*&sid=[a-zA-Z]{6}/i", $value))

{

unset($url[$key]);

}

}

//print_r($url);

//获取到板块数组$url，循环访问，此处获取第一个模块第一页的数据

$i=0;

foreach ($url as $key=>$value)

{

if ($i>=1)

{

//测试限制

break;

{

//匹配http://www.phpchina.com/bbs/viewthread.php?tid=681 27&extra=page%3D1&page=1&sid=iBLZfK

if (!preg_match("/^(http:\/\/www\.phpchina\.com\/bbs\ /viewthread\.php\?tid=)[0-9]*&extra=page\%3D1&a mp;page=[0-9]*&sid=[a-zA-Z]{6}/i",$value))

{

unset($tie[$i][$key]);

}

}

//print_r($tie[$i]);

//归类数组，将同一个帖子不同页面的内容放一个数组里

$left='';//连接左边公用地址

$j=0;

$page=array();

foreach ($tie[$i] as $key=>$value)

{

$left=substr($value,0,52);

$m=0;

foreach ($tie[$i] as $pkey=>$pvalue)

{

//重组数组

if (substr($pvalue,0,52)==$left)

{

for ($n=1;$n<$nums;$n++)

{

$paget[$n]=array_diff($page[$n],$page[$n-1]);

}

//去除多维数组重复值结束

//去除数组空值

unset($page);

$page=array();//重新定义page数组

$page=array_filter($paget);

//print_r($page);

$u=0;

$title=array();

$content=array();

$temp='';

$tt=array();

foreach ($page as $key=>$value)

{

//外围循环，针对一个帖子

if (is_array($value))

{

foreach ($value as $k1=>$v1)

{

//页内循环，针对一个帖子的N页

$snoopy=new Snoopy();

$snoopy->fetch($v1);

$temp=$snoopy->results;

//读取标题

if (!preg_match_all("/<h2>(.*)<\/h2>/i",$temp,$tt))

{

echo "no title";

exit;

}

else

{

$title[$u]=$tt[1][1];

}

unset($tt);

//读取内容

if (!preg_match_all("/<div id=\"postmessage_[0-9]{1,8}\" class=\"t_msgfont\">(.*)<\/div>/i",$temp,$tt))

{

print_r($tt);

echo "no content1";

exit;

}

else

{

foreach ($tt[1] as $c=>$c2)

{

$content[$u].=$c2;

}

}

}

}

else

{

//直接取页内容

$snoopy=new Snoopy();

$snoopy->fetch($value);

$temp=$snoopy->results;

//读取标题

if (!preg_match_all("/<h2>(.*)<\/h2>/i",$temp,$tt))

{

echo "no title";

exit;

}

else

{

$title[$u]=$tt[1][1];

}

unset($tt);

//读取内容

if (!preg_match_all("/<div id=\"postmessage_[0-9]*\" class=\"t_msgfont\">(.*)<\/div>/i",$temp,$tt))

{

echo "no content2";

exit;

}

else

{

foreach ($tt[1] as $c=>$c2)

{

$content[$u].=$c2;

}

}

}

<?php

//采集phpchina

set_time_limit(0);

require_once("Snoopy.class.php");

$snoopy=new Snoopy();

//登陆论坛

$submit_url = "http://www.phpchina.com/bbs/logging.php?action=lo gin";

$submit_vars["loginmode"] = "normal";

$submit_vars["styleid"] = "1";

$submit_vars["cookietime"] = "";

$submit_vars["loginfield"] = "username";

$submit_vars["username"] = "***"; //你的用户名

$submit_vars["password"] = "*****"; //你的密码

$submit_vars["questionid"] = "0";

$submit_vars["answer"] = "";

$submit_vars["loginsubmit"] = "提交";

$snoopy->submit($submit_url,$submit_vars);

if ($snoopy->results)

{

//获取连接地址

$snoopy->fetchlinks("http://www.phpchina.com/bbs");

$url=array();

$url=$snoopy->results;

//print_r($url);

foreach ($url as $key=>$value)

{

//匹配http://www.phpchina.com/bbs/forumdisplay.php?fid=1 56&sid=VfcqTR地址即论坛板块地址

if (!preg_match("/^(http:\/\/www\.phpchina\.com\/bbs\ /forumdisplay\.php\?fid=)[0-9]*&sid=[a-zA-Z]{6}/i", $value))

{

unset($url[$key]);

}

}

//print_r($url);

//获取到板块数组$url，循环访问，此处获取第一个模块第一页的数据

$i=0;

foreach ($url as $key=>$value)

{

if ($i>=1)

{

//测试限制

break;

}

else

{

//访问该模块，提取帖子的连接地址，正式访问里需要提取帖子分页的数据，然后根据分页数据提取帖子数据

$snoopy=new Snoopy();

$snoopy->fetchlinks($value);

$tie=array();

$tie[$i]=$snoopy->results;

//print_r($tie);

//转换数组

foreach ($tie[$i] as $key=>$value)

{

//匹配http://www.phpchina.com/bbs/viewthread.php?tid=681 27&extra=page%3D1&page=1&sid=iBLZfK

if (!preg_match("/^(http:\/\/www\.phpchina\.com\/bbs\ /viewthread\.php\?tid=)[0-9]*&extra=page\%3D1&a mp;page=[0-9]*&sid=[a-zA-Z]{6}/i",$value))

{

unset($tie[$i][$key]);

}

}

//print_r($tie[$i]);

//归类数组，将同一个帖子不同页面的内容放一个数组里

$left='';//连接左边公用地址

$j=0;

$page=array();

foreach ($tie[$i] as $key=>$value)

{

$left=substr($value,0,52);

$m=0;

foreach ($tie[$i] as $pkey=>$pvalue)

{

//重组数组

if (substr($pvalue,0,52)==$left)

{

$page[$j][$m]=$pvalue;

$m++;

}

}

$j++;

}

//去除重复项开始

//$page=array_unique($page);只能用于一维数组

$paget[0]=$page[0];

$nums=count($page);

for ($n=1;$n<$nums;$n++)

{

$paget[$n]=array_diff($page[$n],$page[$n-1]);

}

//去除多维数组重复值结束

//去除数组空值

unset($page);

$page=array();//重新定义page数组

$page=array_filter($paget);

//print_r($page);

$u=0;

$title=array();

$content=array();

$temp='';

$tt=array();

foreach ($page as $key=>$value)

{

//外围循环，针对一个帖子

if (is_array($value))

{

foreach ($value as $k1=>$v1)

{

//页内循环，针对一个帖子的N页

$snoopy=new Snoopy();

$snoopy->fetch($v1);

$temp=$snoopy->results;

//读取标题

if (!preg_match_all("/<h2>(.*)<\/h2>/i",$temp,$tt))

{

echo "no title";

exit;

}

else

{

$title[$u]=$tt[1][1];

}

unset($tt);

//读取内容

if (!preg_match_all("/<div id=\"postmessage_[0-9]{1,8}\" class=\"t_msgfont\">(.*)<\/div>/i",$temp,$tt))

{

print_r($tt);

echo "no content1";

exit;

}

else

{

foreach ($tt[1] as $c=>$c2)

{

$content[$u].=$c2;

}

}

}

}

else

{

//直接取页内容

$snoopy=new Snoopy();

$snoopy->fetch($value);

$temp=$snoopy->results;

//读取标题

if (!preg_match_all("/<h2>(.*)<\/h2>/i",$temp,$tt))

{

echo "no title";

exit;

}

else

{

$title[$u]=$tt[1][1];

}

unset($tt);

//读取内容

if (!preg_match_all("/<div id=\"postmessage_[0-9]*\" class=\"t_msgfont\">(.*)<\/div>/i",$temp,$tt))

{

echo "no content2";

exit;

}

else

{

foreach ($tt[1] as $c=>$c2)

{

$content[$u].=$c2;

}

}

}

$u++;

}

print_r($content);

}

$i++;

}

}

else

{

echo "login failed";

exit;

}

?>