定向爬虫 瓦力

我有一个通用的爬虫(UI)。
可定制(服务搭建暂不提供)。

地址:http://www.box.net/shared/c9gub1hk8e

抓取说明书:http://www.box.net/shared/horu7hybcq

如有搭建服务器的，我可以配合。

需要jdk 1.6

package org.xuancheng.service.security; 

import java.io.BufferedReader; 
import java.io.File; 

import java.io.FileOutputStream; 
import java.io.InputStreamReader; 

import java.net.URL; 
import java.util.regex.Matcher; 
import java.util.regex.Pattern; 

public class GetSina { 

/** 
* @param args 
*/ 
public static void main(String[] args) { 

try { 
//参数：1、小说分类号；2、小说起号；3、小说章节数 
getbook(82036,50202,55183); 
} catch (Exception e) { 
// TODO Auto-generated catch block 
e.printStackTrace(); 
} 
} 

private static void getbook(int fenlei,int zhangjie,int zongzhangjie){ 
String charset = "GBK"; 
StringBuffer bf = new StringBuffer(); 
String FileName = "C:\\"+fenlei+".txt"; 
File myFile = new File(FileName); 
if (myFile.exists()) { 
myFile.delete(); 
} 

for (int i = 0; i <= (zongzhangjie-zhangjie); i++) { 
System.out.println("正在抓取小说第【" + i + "】章内容......http://vip.book.sina.com.cn/book/chapter_" + fenlei + "_"+(i+zhangjie)+".html"); 

bf.append(getParagraph("http://vip.book.sina.com.cn/book/chapter_" + fenlei + "_", i+zhangjie)); 


} 

try { 
bf.append("\n\n -----------------本文由【新浪网小说抓取程序改进版0.1】获取\n\n"); 
FileOutputStream fos = new FileOutputStream(FileName); 
fos.write(bf.toString().getBytes(charset)); 
fos.flush(); 
fos.close(); 

System.out.println("文件书写完毕"); 
} catch (Exception e) { 
// TODO Auto-generated catch block 
e.printStackTrace(); 
} 


} 

private static String getParagraph(String url, int index) { 
int status = 0; 
String paragraph = ""; 
try { 
URL ebook = new URL(url + index + ".html"); 
BufferedReader reader = new BufferedReader(new InputStreamReader(ebook.openStream())); 
String line; 
while ((line = reader.readLine()) != null) { 

if (status == 0) { 
//还没有碰到标题 

Pattern pattern = Pattern 
.compile("(.*)<title>(.*)</title>(.*)"); 
Matcher matcher = pattern.matcher(line); 
if (matcher.matches()) { 
paragraph += matcher.group(2); 
paragraph += "\n\n"; 
status = 1; 
} 
} 
if (status == 1) { 
//还没有碰到文章的开头 
//System.out.print(line); 
Pattern pattern = Pattern 
.compile("(.*)<div id=\"contTxt\" class=\"contTxt1\"><p>(.*)"); 
Matcher matcher = pattern.matcher(line); 
if (matcher.matches()) { 
paragraph += matcher.group(2); 
status = 2; //碰到了正文中的画中画 
} 
} 
if (status == 2) { 

Pattern pattern = Pattern.compile("(.*)</p><p></p></div>(.*)"); 
Matcher matcher = pattern.matcher(line); 
if (matcher.matches()) { 
paragraph += matcher.group(2); 
status = 3; 
} 
} 
} 

//替换掉</p><p> 
paragraph=paragraph.replaceAll("</p><p>", "\n\n"); 
return paragraph.replaceAll("</p></div>", "\n\n"); 
} catch (Exception e) { 
System.out.println(e.toString()); 
return "抓取出错"; 
} 
} 
}