spider简单的爬虫程序

spider简单的爬虫程序

1、基础准备
htmlparser
首页：http://sourceforge.net/projects/htmlparser/
下载：http://sourceforge.net/project/showfiles.php?group_id=24399
文件：htmlparser1_6_20060610.zip
<dependency>
<groupId>org.htmlparser</groupId>
<artifactId>htmlparser</artifactId>
<version>1.6</version>
</dependency>

cpdetector
首页：http://cpdetector.sourceforge.net/
下载：http://sourceforge.net/project/showfiles.php?group_id=114421
文件：cpdetector_eclipse_project_1.0.7.zip

<dependency>
<groupId>cpdetector</groupId>
<artifactId>cpdetector</artifactId>
<version>1.0.5</version>
</dependency>

spindle
首页：http://www.bitmechanic.com/projects/spindle/ （但是已经无法访问）

2 修改spindle代码得到的spider
简单的将URL打印出来了，解析的内容等等都没有处理

解析HTML的基类HtmlParserUtil.java
package com.sillycat.api.commons.utils.html;

import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.SocketException;
import java.net.SocketTimeoutException;
import java.net.URL;
import java.net.UnknownHostException;
import java.nio.charset.Charset;

import org.htmlparser.Parser;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;
import org.htmlparser.visitors.HtmlPage;

import cpdetector.io.ASCIIDetector;
import cpdetector.io.CodepageDetectorProxy;
import cpdetector.io.JChardetFacade;
import cpdetector.io.ParsingDetector;
import cpdetector.io.UnicodeDetector;

public class HtmlParserUtil {

/* StringBuffer的缓冲区大小 */
public static int TRANSFER_SIZE = 4096;

/* 当前平台的行分隔符 */
public static String lineSep = System.getProperty("line.separator");

/* 自动探测页面编码，避免中文乱码的出现 */
public static String autoDetectCharset(URL url) {

   CodepageDetectorProxy detector = CodepageDetectorProxy.getInstance();
   /**
   * ParsingDetector可用于检查HTML、XML等文件或字符流的编码 构造方法中的参数用于指示是否显示探测过程的详细信息
   * 为false则不显示
   */
   detector.add(new ParsingDetector(false));
   detector.add(JChardetFacade.getInstance());
   detector.add(ASCIIDetector.getInstance());
   detector.add(UnicodeDetector.getInstance());

   Charset charset = null;
   try {
    charset = detector.detectCodepage(url);
   } catch (MalformedURLException mue) {
    mue.printStackTrace();
   } catch (IOException ie) {
    ie.printStackTrace();
   }
   if (charset == null)
    charset = Charset.defaultCharset();
   return charset.name();
}

/* 按照指定编码解析标准的html页面，为建立索引做准备 */
public static String[] parseHtml(String url, String charset) {

   String result[] = null;
   String content = null;

   try {
    URL source = new URL(url);
    InputStream in = source.openStream();
    BufferedReader reader = new BufferedReader(new InputStreamReader(
      in, charset));
    String line = new String();
    StringBuffer temp = new StringBuffer(TRANSFER_SIZE);
    while ((line = reader.readLine()) != null) {
     temp.append(line);
     temp.append(lineSep);
    }
    reader.close();
    in.close();
    content = temp.toString();
   } catch (UnsupportedEncodingException uee) {
    uee.printStackTrace();
   } catch (MalformedURLException mue) {
    System.err.println("Invalid URL : " + url);
   } catch (UnknownHostException uhe) {
    System.err.println("UnknowHost : " + url);
   } catch (SocketException se) {
    System.err.println("Socket Error : " + se.getMessage() + " " + url);
   } catch (SocketTimeoutException ste) {
    System.err.println("Socket Connection Time Out : " + url);
   } catch (FileNotFoundException fnfe) {
    System.err.println("broken link "
      + ((FileNotFoundException) fnfe.getCause()).getMessage()
      + " ignored");
   } catch (IOException ie) {
    ie.printStackTrace();
   }

   if (content != null) {
    Parser myParser = Parser.createParser(content, charset);
    HtmlPage visitor = new HtmlPage(myParser);
    try {
     myParser.visitAllNodesWith(visitor);
     String body = null;
     String title = "Untitled";
     if (visitor.getBody() != null) {
      NodeList nodelist = visitor.getBody();
      body = nodelist.asString().trim();
     }
     if (visitor.getTitle() != null){
      title = visitor.getTitle();
     }
     result = new String[] { body, title };
    } catch (ParserException pe) {
     pe.printStackTrace();
    }
   }
   return result;
}
}


多线程爬虫类    HtmlCaptureRunner.java

package com.sillycat.api.thread.runner;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.SocketException;
import java.net.SocketTimeoutException;
import java.net.URL;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.HashSet;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.htmlparser.Parser;
import org.htmlparser.PrototypicalNodeFactory;
import org.htmlparser.filters.AndFilter;
import org.htmlparser.filters.HasAttributeFilter;
import org.htmlparser.filters.NodeClassFilter;
import org.htmlparser.tags.BaseHrefTag;
import org.htmlparser.tags.FrameTag;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.tags.MetaTag;
import org.htmlparser.util.EncodingChangeException;
import org.htmlparser.util.NodeIterator;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;

import com.sillycat.api.commons.utils.StringUtil;
import com.sillycat.api.commons.utils.html.HtmlParserUtil;

public class HtmlCaptureRunner implements Runnable {

public Log logger = LogFactory.getLog(getClass());

/* 基准(初始)URL */
protected String baseURL = null;

private String contentPath = null;

/**
* 待解析的URL地址集合，所有新检测到的链接均存放于此； 解析时按照先入先出（First-In First-Out）法则线性取出
*/
protected ArrayList URLs = new ArrayList();

/* 已存储的URL地址集合，避免链接的重复抓取 */
protected HashSet indexedURLs = new HashSet();

protected Parser parser = new Parser();;

/* 程序运行线程数，默认2个线程 */
protected int threads = 2;

/* 解析页面时的字符编码 */
protected String charset;

/* 基准端口 */
protected int basePort;

/* 基准主机 */
protected String baseHost;

/* 是否存储,默认true */
protected boolean justDatabase = true;

/* 检测索引中是否存在当前URL信息,避免重复抓取 */
protected boolean isRepeatedCheck = false;

public HtmlCaptureRunner() {
   PrototypicalNodeFactory factory = new PrototypicalNodeFactory();
   factory.registerTag(new LocalLinkTag());
   factory.registerTag(new LocalFrameTag());
   factory.registerTag(new LocalBaseHrefTag());
   parser.setNodeFactory(factory);
}

public void capture() {
   URLs.clear();
   URLs.add(getBaseURL());

   int responseCode = 0;
   String contentType = "";

   try {
    HttpURLConnection uc = (HttpURLConnection) new URL(baseURL)
      .openConnection();
    responseCode = uc.getResponseCode();
    contentType = uc.getContentType();
   } catch (MalformedURLException mue) {
    logger.error("Invalid URL : " + getBaseURL());
   } catch (UnknownHostException uhe) {
    logger.error("UnknowHost : " + getBaseURL());
   } catch (SocketException se) {
    logger.error("Socket Error : " + se.getMessage() + " "
      + getBaseURL());
   } catch (IOException ie) {
    logger.error("IOException : " + ie);
   }

   if (responseCode == HttpURLConnection.HTTP_OK
     && contentType.startsWith("text/html")) {
    try {
     charset = HtmlParserUtil.autoDetectCharset(new URL(baseURL));

     basePort = new URL(baseURL).getPort();
     baseHost = new URL(baseURL).getHost();
     if (charset.equals("windows-1252"))
      charset = "GBK";

     long start = System.currentTimeMillis();
     ArrayList threadList = new ArrayList();
     for (int i = 0; i < threads; i++) {
      Thread t = new Thread(this, "Spider Thread #" + (i + 1));
      t.start();
      threadList.add(t);
     }
     while (threadList.size() > 0) {
      Thread child = (Thread) threadList.remove(0);
      try {
       child.join();
      } catch (InterruptedException ie) {
       logger.error("InterruptedException : " + ie);
      }
     }
     // for (int i = 0; i < threads; i++) {
     // threadPool.getThreadPoolExcutor().execute(new
     // Thread(this,"Spider Thread #" + (i + 1)));
     // }
     long elapsed = System.currentTimeMillis() - start;
     logger.info("Finished in " + (elapsed / 1000) + " seconds");
     logger.info("The Count of the Links Captured is "
       + indexedURLs.size());
    } catch (MalformedURLException e) {
     e.printStackTrace();
    }
   }
}

public void run() {
   String url;
   while ((url = dequeueURL()) != null) {
    if (justDatabase) {
     process(url);
    }
   }
   threads--;
}

/**
* 处理单独的URL地址，解析页面并加入到lucene索引中；通过自动探测页面编码保证抓取工作的顺利执行
*/
protected void process(String url) {

   String result[];
   String content = null;
   String title = null;

   result = HtmlParserUtil.parseHtml(url, charset);
   content = result[0];
   title = result[1];

   if (content != null && content.trim().length() > 0) {
    // content
    System.out.println(url);
    // title
    // DateTools.timeToString(System.currentTimeMillis()
   }
}

/* 从URL队列mPages里取出单个的URL */
public synchronized String dequeueURL() {
   while (true)
    if (URLs.size() > 0) {
     String url = (String) URLs.remove(0);
     indexedURLs.add(url);
     if (isToBeCaptured(url)) {
      NodeList list;
      try {
       int bookmark = URLs.size();
       /* 获取页面所有节点 */
       parser.setURL(url);
       try {
        list = new NodeList();
        for (NodeIterator e = parser.elements(); e
          .hasMoreNodes();)
         list.add(e.nextNode());
       } catch (EncodingChangeException ece) {
        /* 解码出错的异常处理 */
        parser.reset();
        list = new NodeList();
        for (NodeIterator e = parser.elements(); e
          .hasMoreNodes();)
         list.add(e.nextNode());
       }
       /**
       * 依据 http://www.robotstxt.org/wc/meta-user.html 处理
       * Robots <META> tag
       */
       NodeList robots = list
         .extractAllNodesThatMatch(
           new AndFilter(new NodeClassFilter(
             MetaTag.class),
             new HasAttributeFilter("name",
               "robots")), true);
       if (0 != robots.size()) {
        MetaTag robot = (MetaTag) robots.elementAt(0);
        String content = robot.getAttribute("content")
          .toLowerCase();
        if ((-1 != content.indexOf("none"))
          || (-1 != content.indexOf("nofollow")))
         for (int i = bookmark; i < URLs.size(); i++)
          URLs.remove(i);
       }
      } catch (ParserException pe) {
       logger.error("ParserException : " + pe);
      }
      return url;
     }
    } else {
     threads--;
     if (threads > 0) {
      try {
       wait();
       threads++;
      } catch (InterruptedException ie) {
       logger.error("InterruptedException : " + ie);
      }
     } else {
      notifyAll();
      return null;
     }
    }
}

private boolean isHTML(String url) {
   if (!url.endsWith(".html")) {
    return false;
   }
   if (StringUtil.isNotBlank(contentPath)) {
    if (!url.startsWith(baseURL + "/" + contentPath)) {
     return false;
    }
   }
   return true;
}

/**
* 判断提取到的链接是否符合解析条件；标准为Port及Host与基准URL相同且类型为text/html或text/plain
*/
public boolean isToBeCaptured(String url) {

   boolean flag = false;

   HttpURLConnection uc = null;
   int responseCode = 0;
   String contentType = "";
   String host = "";
   int port = 0;

   try {
    URL source = new URL(url);
    String protocol = source.getProtocol();
    if (protocol != null && protocol.equals("http")) {
     host = source.getHost();
     port = source.getPort();
     uc = (HttpURLConnection) source.openConnection();
     uc.setConnectTimeout(8000);
     responseCode = uc.getResponseCode();
     contentType = uc.getContentType();
    }
   } catch (MalformedURLException mue) {
    logger.error("Invalid URL : " + url);
   } catch (UnknownHostException uhe) {
    logger.error("UnknowHost : " + url);
   } catch (SocketException se) {
    logger.error("Socket Error : " + se.getMessage() + " " + url);
   } catch (SocketTimeoutException ste) {
    logger.error("Socket Connection Time Out : " + url);
   } catch (FileNotFoundException fnfe) {
    logger.error("broken link " + url + " ignored");
   } catch (IOException ie) {
    logger.error("IOException : " + ie);
   }
   if (port == basePort
     && responseCode == HttpURLConnection.HTTP_OK
     && host.equals(baseHost)
     && (contentType.startsWith("text/html") || contentType
       .startsWith("text/plain")))
    flag = true;
   return flag;
}

class LocalLinkTag extends LinkTag {
   public void doSemanticAction() {
    String link = getLink();
    if (link.endsWith("/"))
     link = link.substring(0, link.length() - 1);
    int pos = link.indexOf("#");
    if (pos != -1)
     link = link.substring(0, pos);
    /* 将链接加入到处理队列中 */
    if (!(indexedURLs.contains(link) || URLs.contains(link))) {
     if (isHTML(link)) {
      URLs.add(link);
     }
    }
    setLink(link);
   }
}

/**
* Frame tag that rewrites the SRC URLs. The SRC URLs are mapped to local
* targets if they match the source.
*/
class LocalFrameTag extends FrameTag {
   public void doSemanticAction() {
    String link = getFrameLocation();
    if (link.endsWith("/"))
     link = link.substring(0, link.length() - 1);
    int pos = link.indexOf("#");
    if (pos != -1)
     link = link.substring(0, pos);
    /* 将链接加入到处理队列中 */
    if (!(indexedURLs.contains(link) || URLs.contains(link))) {
     if (isHTML(link)) {
      URLs.add(link);
     }
    }
    setFrameLocation(link);
   }
}

/**
* Base tag that doesn't show. The toHtml() method is overridden to return
* an empty string, effectively shutting off the base reference.
*/
class LocalBaseHrefTag extends BaseHrefTag {
   public String toHtml() {
    return ("");
   }
}

public String getBaseURL() {
   return baseURL;
}

public void setBaseURL(String baseURL) {
   this.baseURL = baseURL;
}

public int getThreads() {
   return threads;
}

public void setThreads(int threads) {
   this.threads = threads;
}

public String getCharset() {
   return charset;
}

public void setCharset(String charset) {
   this.charset = charset;
}

public int getBasePort() {
   return basePort;
}

public void setBasePort(int basePort) {
   this.basePort = basePort;
}

public String getBaseHost() {
   return baseHost;
}

public void setBaseHost(String baseHost) {
   this.baseHost = baseHost;
}

public boolean isJustDatabase() {
   return justDatabase;
}

public void setJustDatabase(boolean justDatabase) {
   this.justDatabase = justDatabase;
}

public String getContentPath() {
   return contentPath;
}

public void setContentPath(String contentPath) {
   this.contentPath = contentPath;
}

}

spring上的配置文件applicationContext-bean.xml:
<bean id="productCapture"
   class="com.sillycat.api.thread.runner.HtmlCaptureRunner" >
   <property name="contentPath" value="${product.contentPath}" />
   <property name="basePort" value="${product.base.port}" />
   <property name="baseURL" value="${product.base.url}" />
   <property name="charset" value="${product.base.code}" />
   <property name="threads" value="${product.base.threads}"/>
</bean>

<bean id="messageCapture"
   class="com.sillycat.api.thread.runner.HtmlCaptureRunner" >
   <property name="contentPath" value="${message.contentPath}" />
   <property name="basePort" value="${message.base.port}" />
   <property name="baseURL" value="${message.base.url}" />
   <property name="charset" value="${message.base.code}" />
   <property name="threads" value="${message.base.threads}"/>
</bean>

easySearch.properties配置文件：
#==========================================
# spider configration
#=========================================
product.contentPath=product
product.base.port=80
product.base.url=http://www.safedv.com
product.base.code=UTF-8
product.base.threads=3

message.contentPath=message
message.base.port=80
message.base.url=http://www.safedv.com
message.base.code=UTF-8
message.base.threads=3

单元测试类HtmlRunnerTest.java文件：

package com.sillycat.api.thread;

import com.sillycat.api.commons.base.BaseManagerTest;
import com.sillycat.api.thread.runner.HtmlCaptureRunner;

public class HtmlRunnerTest extends BaseManagerTest {

private HtmlCaptureRunner productCapture;

private HtmlCaptureRunner messageCapture;

protected void setUp() throws Exception {
   super.setUp();
   productCapture = (HtmlCaptureRunner) appContext.getBean("productCapture");
   messageCapture = (HtmlCaptureRunner) appContext.getBean("messageCapture");
}

protected void tearDown() throws Exception {
   super.tearDown();
}

public void testDumy() {
   assertTrue(true);
}

public void ntestProductCapture() {
   productCapture.capture();
}

public void testMessageCapture(){
   messageCapture.capture();
}
}