jeecms 采集功能优化,基于htmlparser实现,多线程版

为了熟悉一下多线程相关知识，把jeecms采集器类，改成了多线程版，还不是很完善，帖出来大家一起完善，改进。
说明：暂不支持暂停，停止功能。
用法：和我上一篇jeecms 采集功能优化,基于htmlparser实现里面的用法一样。
思路：想法很简单，在主线程处理类中，先取得当前采集任务下所有URL，并放入队列中，然后开启指定数目的线程（默认是2）采集内容


代码清单：
采集器主类：MultiThreadAcquisitionSvcImpl.java
HTML解析工具类接口：ParseHtmlTool.java
HTML解析工具,HtmlParser实现类:HtmlParserImpl.java
采集参数封装bean:ParamBean.java
队列类：Queue.java
URL队列：UrlQueue.java

代码如下：


    采集器主类：MultiThreadAcquisitionSvcImpl.java

    package com.jeecms.cms.service;

    import java.io.IOException;
    import java.net.URI;
    import java.net.URISyntaxException;
    import java.util.List;
    import java.util.Map;
    import java.util.concurrent.CountDownLatch;
    import java.util.concurrent.ExecutorService;
    import java.util.concurrent.Executors;

    import org.apache.commons.lang.StringUtils;
    import org.apache.http.HttpEntity;
    import org.apache.http.HttpHost;
    import org.apache.http.HttpResponse;
    import org.apache.http.StatusLine;
    import org.apache.http.client.ClientProtocolException;
    import org.apache.http.client.HttpClient;
    import org.apache.http.client.HttpResponseException;
    import org.apache.http.client.ResponseHandler;
    import org.apache.http.client.methods.HttpGet;
    import org.apache.http.conn.params.ConnRoutePNames;
    import org.apache.http.impl.client.DefaultHttpClient;
    import org.apache.http.util.EntityUtils;
    import org.slf4j.Logger;
    import org.slf4j.LoggerFactory;
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.stereotype.Service;

    import com.jeecms.cms.entity.assist.CmsAcquisition;
    import com.jeecms.cms.entity.main.Content;
    import com.jeecms.cms.manager.assist.CmsAcquisitionMng;
    /**
     * 采集器-多线程版
     * @author javacoo
     * @since 2011-11-02
     * @version 1.0
     */
    @Service
    public class MultiThreadAcquisitionSvcImpl implements AcquisitionSvc {
    private Logger log = LoggerFactory.getLogger(MultiThreadAcquisitionSvcImpl.class);
    /**开启线程数*/
    private static int THREAD_NUM = 2;
    /**每个线程休眠毫秒数*/
    private static int SLEEP_TIME = 100;
    /**连接集合标志*/
        private static String LINK_KEY = "linkKey";
        /**标题集合标志*/
    private static String TITLE_KEY = "titleKey";
    /**采集管理对象*/
    private CmsAcquisitionMng cmsAcquisitionMng;
    /**存放HttpClient的ThreadLocal对象*/
    private static ThreadLocal<HttpClient> httpClientThreadLocal = new ThreadLocal<HttpClient>();
    /**存放ParseHtmlTool的ThreadLocal对象*/
    private static ThreadLocal<ParseHtmlTool> parseHtmlToolThreadLocal = new ThreadLocal<ParseHtmlTool>();
    /**存放UrlQueue的ThreadLocal对象*/
    private static ThreadLocal<UrlQueue> urlQueueThreadLocal = new ThreadLocal<UrlQueue>();

    @Autowired
    public void setCmsAcquisitionMng(CmsAcquisitionMng cmsAcquisitionMng) {
    this.cmsAcquisitionMng = cmsAcquisitionMng;
    }

    /**
     * 开始执行采集任务
     */
    public boolean start(Integer id) {
    CmsAcquisition acqu = cmsAcquisitionMng.findById(id);
    if (acqu == null || acqu.getStatus() == CmsAcquisition.START) {
    return false;
    }
    new Thread(new MainThreadProcesser(this,acqu)).start();
    return true;
    }
    /**
     * 主线程处理类
     * @author javacoo
     * @since 2011-11-02
     */
    private class MainThreadProcesser implements Runnable {
    private CmsAcquisition acqu;
    private AcquisitionSvc acquisitionSvc;
    public MainThreadProcesser(AcquisitionSvc acquisitionSvc,CmsAcquisition acqu) {
    this.acqu = acqu;
    this.acquisitionSvc = acquisitionSvc;
    }
    public void run() {
    long tStart = System.currentTimeMillis();
    System.out.println("主线程："+Thread.currentThread().getName() + "开始...");
    try {
    getHttpClient().getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY,new HttpHost("128.160.64.5", 1235));
    CharsetHandler handler = new CharsetHandler(acqu.getPageEncoding());
    getAllUrls(acqu,handler);
    CountDownLatch latch = new CountDownLatch(THREAD_NUM);
    ExecutorService exec = Executors.newCachedThreadPool();
    for(int i=0;i<THREAD_NUM;i++){
    Thread thread = new Thread(new Processer(acquisitionSvc,acqu,latch,getHttpClient(),getUrlQueue(),getParseHtmlTool(acqu),handler));
    exec.execute(thread);
    }
    latch.await();
    exec.shutdown();
    } catch (InterruptedException e) {
    e.printStackTrace();
    } catch (ClientProtocolException e) {
    e.printStackTrace();
    } catch (URISyntaxException e) {
    e.printStackTrace();
    } catch (IOException e) {
    e.printStackTrace();
    }finally{
    httpClientThreadLocal.get().getConnectionManager().shutdown();
    cmsAcquisitionMng.end(acqu.getId());
    httpClientThreadLocal.remove();
    parseHtmlToolThreadLocal.remove();
    urlQueueThreadLocal.remove();

    long tEnd = System.currentTimeMillis();
    System.out.println("主线程："+Thread.currentThread().getName() + "结束...");
    System.out.println("主线程："+Thread.currentThread().getName() + "总共用时:" + (tEnd - tStart) + "ms");
    }
    }
    }
    /**
     * 处理类
     * @author javacoo
     * @since 2011-11-02
     */
    private class Processer implements Runnable {
    private AcquisitionSvc acquisitionSvc;
    private CmsAcquisition acqu;
    private CountDownLatch latch;
    private UrlQueue urlQueue;
    private HttpClient httpClient;
    private ParseHtmlTool parseHtmlTool;
    private CharsetHandler handler;
    public Processer(AcquisitionSvc acquisitionSvc,CmsAcquisition acqu,CountDownLatch latch,HttpClient httpClient,UrlQueue urlQueue,ParseHtmlTool parseHtmlTool,CharsetHandler handler) {
    this.acquisitionSvc = acquisitionSvc;
    this.acqu = acqu;
    this.latch = latch;
    this.urlQueue = urlQueue;
    this.httpClient = httpClient;
    this.parseHtmlTool = parseHtmlTool;
    this.handler = handler;
    }
    public void run() {
    System.out.println("======================子线程："+Thread.currentThread().getName() + "开始...");
    try {
    Map<String,String> urlMap = null;
    while(!urlAndTitleMapIsEmpty(urlQueue)) {
    urlMap = getUrlAndTitleMap(urlQueue);
    saveContent(acqu,httpClient,parseHtmlTool,handler,urlMap);
    Thread.sleep(SLEEP_TIME);
    }
    } catch (Exception e) {
    e.printStackTrace();
    log.warn(null, e);
    } finally {
    System.out.println("======================子线程："+Thread.currentThread().getName() + "结束.");
    log.info("Acquisition#{} complete", acqu.getId());
    latch.countDown();
    }
    }
    }


    /**
     * 取得当前主线程的HttpClient对象
     * @return 当前主线程的HttpClient对象
     */
    private static HttpClient getHttpClient(){
    if(httpClientThreadLocal.get() == null){
    HttpClient client = new DefaultHttpClient();
    httpClientThreadLocal.set(client);
    return client;
    }else{
    return httpClientThreadLocal.get();
    }
    }
    /**
     * 取得当前主线程的UrlQueue对象
     * @return 当前主线程的UrlQueue对象
     */
    private static UrlQueue getUrlQueue(){
    if(urlQueueThreadLocal.get() == null){
    UrlQueue urlQueue = new UrlQueue();
    urlQueueThreadLocal.set(urlQueue);
    return urlQueue;
    }else{
    return urlQueueThreadLocal.get();
    }
    }
    /**
     * 取得当前主线程的ParseHtmlTool对象
     * @param acqu 采集参数对象
     * @return 当前主线程的ParseHtmlTool对象
     */
    private static ParseHtmlTool getParseHtmlTool(CmsAcquisition acqu){
    if(parseHtmlToolThreadLocal.get() == null){
    ParseHtmlTool parseHtmlTool = new HtmlParserImpl(acqu);
    parseHtmlToolThreadLocal.set(parseHtmlTool);
    return parseHtmlTool;
    }else{
    return parseHtmlToolThreadLocal.get();
    }
    }
    /**
     * 连接和标题map对象入队列
     * @param map 连接和标题map对象
     */
    private synchronized void addUrlAndTitleMap(Map<String,String> map){
    getUrlQueue().addUnVisitedUrl(map);
    }
    /**
     * 连接和标题map对象出队列
     * @param urlQueue 当前线程的队列
     * @return 连接和标题map对象
     */
    private synchronized Map<String,String> getUrlAndTitleMap(UrlQueue urlQueue){
    return urlQueue.unVisitedUrlDeQueue();
    }
    /**
     *  判断当前对象是否为空
     * @param urlQueue 当前线程的队列
     * @return true/flase
     */
    private synchronized boolean urlAndTitleMapIsEmpty(UrlQueue urlQueue){
    return urlQueue.isEmpty();
    }
    /**
     * 取得当前线程下所有计划的连接，并加入队列
     * @param acqu 采集参数对象
     * @param handler 字符集对象
     * @throws URISyntaxException
     * @throws IOException
     * @throws ClientProtocolException
     */
    private void getAllUrls(CmsAcquisition acqu,CharsetHandler handler) throws URISyntaxException, ClientProtocolException, IOException{
    acqu = cmsAcquisitionMng.start(acqu.getId());
    String[] plans = acqu.getAllPlans();
        String url = null;
    String html = null;
    List<Map<String,String>> urlAndTitleListMap = null;
    HttpGet httpGet = null;
        for (int i = plans.length - acqu.getCurrNum(); i >= 0; i--) {
    url = plans[i];
    httpGet = new HttpGet(new URI(url.trim()));
    html = getHttpClient().execute(httpGet, handler);
    urlAndTitleListMap = getParseHtmlTool(acqu).getUrlAndTitleMap(html);
    for(Map<String,String> map : urlAndTitleListMap){
    addUrlAndTitleMap(map);
    }
    }
        System.out.println("=======当前线程："+Thread.currentThread().getName() + "URL连接数："+getUrlQueue().getUnVisitedUrl().getSize());
    }
    /**
     * 保存内容
     * @param acqu 请求参数对象
     * @param httpClient httpClient对象
     * @param parseHtmlTool parseHtmlTool对象
     * @param handler CharsetHandler对象
     * @param map 连接和标题map对象
     * @return Content
     */
    private synchronized Content saveContent(CmsAcquisition acqu,HttpClient httpClient,ParseHtmlTool parseHtmlTool,CharsetHandler handler,Map<String,String> map) {
    try {
    HttpGet httpGet = null;
    if(map.get(LINK_KEY).contains("http://")){
    httpGet = new HttpGet(new URI(map.get(LINK_KEY).trim()));
    }else{
    httpGet = new HttpGet(new URI("http://localhost/v7/"+map.get(LINK_KEY).trim()));
    }
    String html = httpClient.execute(httpGet, handler);
    System.out.println("=============================子线程："+Thread.currentThread().getName() + "执行");
    String txt = parseHtmlTool.getHtml(html);
    return cmsAcquisitionMng.saveContent(map.get(TITLE_KEY), txt,acqu.getId());
                //return null;
    } catch (Exception e) {
    log.warn(null, e);
    e.printStackTrace();
    return null;
    }
    }
    /**
     * 字符集帮助类
     * @author Administrator
     *
     */
    private class CharsetHandler implements ResponseHandler<String> {
    private String charset;

    public CharsetHandler(String charset) {
    this.charset = charset;
    }

    public String handleResponse(HttpResponse response)
    throws ClientProtocolException, IOException {
    StatusLine statusLine = response.getStatusLine();
    if (statusLine.getStatusCode() >= 300) {
    throw new HttpResponseException(statusLine.getStatusCode(),
    statusLine.getReasonPhrase());
    }
    HttpEntity entity = response.getEntity();
    if (entity != null) {
    if (!StringUtils.isBlank(charset)) {
    return EntityUtils.toString(entity, charset);
    } else {
    return EntityUtils.toString(entity);
    }
    } else {
    return null;
    }
    }
    }


    }

    相关辅助类

    HTML解析工具类接口：ParseHtmlTool.java

    package com.jeecms.cms.service;

    import java.util.List;
    import java.util.Map;

    /**
     * HTML解析工具类接口
     * @author javacoo
     * @since 2011-10-31
     */
    public interface ParseHtmlTool {
    /**
     * 取得连接集合
     * @param orginHtml 原始HTML
     * @return 连接集合
     */
    List<String> getUrlList( String orginHtml);
    /**
     * 取得标题集合
     * @param orginHtml 原始HTML
     * @return 标题集合
     */
    List<String> getTitleList(String orginHtml);
        /**
         * 取得指定区域的HTML内容
         * @return 指定区域的HTML内容
         */
    String getHtml(String orginHtml);
    /**
     * 取得连接标题Map集合
     * @param orginHtml 原始HTML
     * @return 连接标题Map集合
     */
    List<Map<String,String>> getUrlAndTitleMap(String orginHtml);
    }

    HTML解析工具,HtmlParser实现类:HtmlParserImpl.java

    package com.jeecms.cms.service;

    import java.io.BufferedReader;
    import java.io.File;
    import java.io.FileInputStream;
    import java.io.IOException;
    import java.io.InputStreamReader;
    import java.net.URISyntaxException;
    import java.util.ArrayList;
    import java.util.HashMap;
    import java.util.Iterator;
    import java.util.List;
    import java.util.Map;
    import java.util.regex.Matcher;
    import java.util.regex.Pattern;

    import org.apache.commons.lang.StringUtils;
    import org.htmlparser.Node;
    import org.htmlparser.NodeFilter;
    import org.htmlparser.Parser;
    import org.htmlparser.filters.HasAttributeFilter;
    import org.htmlparser.filters.NodeClassFilter;
    import org.htmlparser.filters.TagNameFilter;
    import org.htmlparser.nodes.RemarkNode;
    import org.htmlparser.util.NodeList;
    import org.htmlparser.util.ParserException;

    import com.jeecms.cms.entity.assist.CmsAcquisition;
    /**
     * HTML解析工具,HtmlParser实现类
     * @author javacoo
     * @since 2011-10-31
     */
    public class HtmlParserImpl implements ParseHtmlTool{
    /**连接集合标志*/
        private static String LINK_KEY = "linkKey";
        /**标题集合标志*/
    private static String TITLE_KEY = "titleKey";
    /**单标签标志*/
    private static String SINGLE_TAG = "singleTag";
    /**连接正则表达式*/
    private static String LINK_REGX = "<a.*href=\"(.*?)\".*>(.*?)</a>";
    /**正则表达式对象*/
    private Pattern pt = Pattern.compile(LINK_REGX);
    /**采集参数bean*/
    private ParamBean paramBean;

    public HtmlParserImpl(CmsAcquisition acqu){
    parseRequestParam(acqu);
    }

    /**
     * 取得标题集合
     * @param orginHtml 原始HTML
     * @return 标题集合
     */
    public List<String> getTitleList(String orginHtml) {
    orginHtml = getHtmlByFilter(paramBean.getLinksetStartMap(), paramBean.getLinksetEndMap(),orginHtml);
    if (StringUtils.isNotEmpty(orginHtml)) {
    return getUrlOrTitleListByType(orginHtml,TITLE_KEY);
    }
    return null;
    }

    /**
     * 取得连接集合
     * @param orginHtml 原始HTML
     * @return 连接集合
     */
    public List<String> getUrlList(String orginHtml) {
    orginHtml = getHtmlByFilter(paramBean.getLinksetStartMap(), paramBean.getLinksetEndMap(),orginHtml);
    if (StringUtils.isNotEmpty(orginHtml)) {
    return getUrlOrTitleListByType(orginHtml,LINK_KEY);
    }
    return null;
    }
    /**
         * 取得指定区域的HTML内容
         * @param orginHtml 原始HTML
         * @return 指定区域的HTML内容
         * @throws ParserException
         */
    public String getHtml(String orginHtml) {
    orginHtml = getHtmlByFilter(paramBean.getContentStartMap(), paramBean.getContentEndMap(),orginHtml);
    return orginHtml;
    }
    /**
     * 取得连接标题Map
     * @param orginHtml 原始HTML
     * @return 连接标题Map
     */
    public List<Map<String,String>> getUrlAndTitleMap(String orginHtml){
    return getUrlAandTitleMap(orginHtml);
    }
    /**
     * 解析采集参数,并封装到ParamBean
     * @param acqu 原始采集参数
     * @return 采集参数封装bean
     */
    private void parseRequestParam(CmsAcquisition acqu){
    paramBean = new ParamBean();
    if(!StringUtils.isEmpty(acqu.getLinksetStart())){
    paramBean.setLinksetStartMap(populateParamMap(acqu.getLinksetStart()));
    }
    if(!StringUtils.isEmpty(acqu.getLinksetEnd())){
    paramBean.setLinksetEndMap(populateParamMap(acqu.getLinksetEnd()));
    }
    if(!StringUtils.isEmpty(acqu.getContentStart())){
    paramBean.setContentStartMap(populateParamMap(acqu.getContentStart()));
    }
    if(!StringUtils.isEmpty(acqu.getContentEnd())){
    paramBean.setContentEndMap(populateParamMap(acqu.getContentEnd()));
    }
    }
    /**
     * 得到连接标题MAP
     * @param html html内容
     * @return 连接或者标题集合
     */
    private List<Map<String,String>> getUrlAandTitleMap(String html) {
    html = getHtmlByFilter(paramBean.getLinksetStartMap(), paramBean.getLinksetEndMap(),html);
    List<Map<String,String>> resultMapList = new ArrayList<Map<String,String>>();
    Map<String,String> resultMap = null;
    Matcher m = pt.matcher(html);
    while (m.find()) {
    if(StringUtils.isNotEmpty(m.group(1)) && StringUtils.isNotEmpty(m.group(2))){
    resultMap = new HashMap<String, String>();
    resultMap.put(LINK_KEY, m.group(1));
    resultMap.put(TITLE_KEY, m.group(2));
    resultMapList.add(resultMap);
    }
    }
    return resultMapList;
    }
    /**
     * 得到地址集
     * @param html html内容
     * @param type 1 ：取得连接集合，2：取得标题集合
     * @return 连接或者标题集合
     */
    private List<String> getUrlOrTitleListByType(String html, String type) {
    List<String> resultList = new ArrayList<String>();
    Matcher m = pt.matcher(html);
    String result = "";
    int pos = 1;
    if(TITLE_KEY.equals(type)){
    pos = 2;
    }
    while (m.find()) {
    result = m.group(pos);
    resultList.add(result);
    }
    return resultList;
    }
    /**
         * 取得指定区域的HTML内容
         * @param tagMap 标签MAP
         * @param removeTagMap 要过滤的标签MAP
         * @param orginHtml 原始HTML
         * @return 指定区域的HTML内容
         * @throws ParserException
         */
    private String getHtmlByFilter(Map<String, String> tagMap,
    Map<String, String> removeTagMap, String orginHtml) {
    try {
    Parser parser = new Parser();
    parser.setInputHTML(orginHtml);
    // 第一步取得指定属性/标签内容
    String tempKey = null;
    String tempValue = null;
    String[] tempValueArr = null;
    StringBuilder sb = new StringBuilder();
    NodeFilter filter = null;
    for(Iterator<String> it = tagMap.keySet().iterator(); it.hasNext();){
    tempKey = it.next();
    tempValue = tagMap.get(tempKey);
    if(tempValue.contains("|")){
    tempValueArr = tempValue.split("\\|");
    }else{
    tempValueArr = new String[]{tempValue};
    }
    for(String value : tempValueArr){
    filter = populateFilter(tempKey,value);
    appendHtmlByFilter(parser, filter, sb);
    }
    }
    // 第二步过滤指定属性/标签内容
    String contentHtml = sb.toString();
    for (Iterator<String> it = removeTagMap.keySet().iterator(); it
    .hasNext();) {
    tempKey = it.next();
    tempValue = removeTagMap.get(tempKey);
    if(tempValue.contains("|")){
    tempValueArr = tempValue.split("\\|");
    }else{
    tempValueArr = new String[]{tempValue};
    }
    for(String value : tempValueArr){
    filter = populateFilter(tempKey,value);
    contentHtml = removeHtmlByFilter(parser, filter, contentHtml);
    }
    }
    //第三步过滤注释
    filter = new NodeClassFilter(RemarkNode.class);
    contentHtml = removeHtmlByFilter(parser, filter, contentHtml);
    //System.out.println("=================================结果=======================================");
    //System.out.println(contentHtml);
    return contentHtml;
    } catch (ParserException e) {
    // TODO Auto-generated catch block
    e.printStackTrace();
    }
    return "";
    }

    /**
     * 解析并组装采集参数，支持标签属性/值形式和标签名称形式，可混合使用
     *

约定采集参数格式如下

     *

1，标签属性/值形式，如：class=articleList|tips,id=fxwb|fxMSN|fxMSN

     *

2，标签名称形式，如：div,p,span

     *

3，混合形式，如：class=articleList|tips,id=fxwb|fxMSN|fxMSN,div,p,span

     * @param paramStr 参数字符串
     */
    private Map<String, String> populateParamMap(String paramStr) {
    Map<String, String> paramMap = new HashMap<String, String>();
    String[] paramStrArr = paramStr.split(",");
    String[] tempStrArr = null;
    StringBuilder sb = new StringBuilder();
    for(String temp : paramStrArr){
    if(temp.contains("=")){
    tempStrArr = temp.split("=");
    paramMap.put(tempStrArr[0], tempStrArr[1]);
    }else{
    if(StringUtils.isNotEmpty(temp)){
    sb.append(temp).append("|");
    }
    }
    }
    if(StringUtils.isNotEmpty(sb.toString())){
    paramMap.put(SINGLE_TAG, sb.substring(0, sb.length() - 1));
    }
    return paramMap;
    }
    /**
     * 组装过滤器
     * @param key 键
     * @param value 值
     * @return 过滤器
     */
    private NodeFilter populateFilter(String key,String value) {
    NodeFilter filter;
    if(SINGLE_TAG.equals(key)){
    filter =  new TagNameFilter(value);
    }else{
    filter = new HasAttributeFilter(key,value);
    }
    return filter;
    }
    /**
         * 过滤指定属性标签HTML
         * @param parser 解析器
         * @param filter 属性过滤器
         * @param orginHtml 原始HTML
         * @return 过滤后HTML
         * @throws ParserException
         */
    private String removeHtmlByFilter(Parser parser, NodeFilter filter,String orginHtml) throws ParserException {
    parser.setInputHTML(orginHtml);
    NodeList nodes = parser.extractAllNodesThatMatch(filter);
    for (int i = 0; i < nodes.size(); i++) {
    Node textnode = (Node) nodes.elementAt(i);
    orginHtml = StringUtils.remove(orginHtml, textnode.toHtml());
    }
    return orginHtml;
    }
    /**
     * 取得所有指定属性/标签的HTML
     * @param parser 解析器
     * @param filter 过滤器
     * @param sb
     * @throws ParserException
     */
    private void appendHtmlByFilter(Parser parser, NodeFilter filter,
    StringBuilder sb) throws ParserException {
    NodeList nodes = parser.extractAllNodesThatMatch(filter);
    for (int i = 0; i < nodes.size(); i++) {
    Node textnode = (Node) nodes.elementAt(i);
    sb.append(textnode.toHtml());
    }
    }

    /**
     * 解析并组装采集参数，支持标签属性/值形式和标签名称形式，可混合使用
     *

约定采集参数格式如下

     *

1，标签属性/值形式，如：class=articleList|tips,id=fxwb|fxMSN|fxMSN

     *

2，标签名称形式，如：div,p,span

     *

3，混合形式，如：class=articleList|tips,id=fxwb|fxMSN|fxMSN,div,p,span

     * @param paramMap 参数map
     * @param str 参数字符串
     */
    private void populateParamMap(Map<String, String> paramMap,String paramStr) {
    String[] paramStrArr = paramStr.split(",");
    String[] tempStrArr = null;
    StringBuilder sb = new StringBuilder();
    for(String temp : paramStrArr){
    if(temp.contains("=")){
    tempStrArr = temp.split("=");
    paramMap.put(tempStrArr[0], tempStrArr[1]);
    }else{
    if(StringUtils.isNotEmpty(temp)){
    sb.append(temp).append("|");
    }
    }
    }
    if(StringUtils.isNotEmpty(sb.toString())){
    paramMap.put(SINGLE_TAG, sb.substring(0, sb.length() - 1));
    }
    }

        /**
         * 测试方法-打开文件并返回内容
         * @param szFileName 文件绝对地址
         * @param charset 字符集
         * @return 内容
         */
    public static String openFile(String szFileName,String charset) {
    try {
    BufferedReader bis = new BufferedReader(new InputStreamReader(
    new FileInputStream(new File(szFileName)), charset));
    StringBuilder szContent = new StringBuilder();
    String szTemp;

    while ((szTemp = bis.readLine()) != null) {
    szContent.append(szTemp).append("\n");
    }
    bis.close();
    return szContent.toString();
    } catch (Exception e) {
    return "";
    }
    }
    /**
     * 测试取得连接地址和标题
     * @throws ParserException
     */
    public void testFetchLinkAndTitle() throws ParserException{
    String html = openFile("F:\\4.htm","UTF-8");
    String result = "";
    Map<String, String> map = new HashMap<String, String>();
    map.put("class", "m_list");
    Map<String, String> notMap = new HashMap<String, String>();
    //notMap.put("class", "atc_ic_f");
    result = getHtmlByFilter(map,notMap,html);
    System.out.println("=============================result============================");
    System.out.println(result);
    System.out.println("==========================================================");
    Pattern pt = Pattern.compile("<a.*href=\"(.*?)\".*>(.*?)</a>");

    Matcher m = pt.matcher(result);
            String link = null;
            String title = null;
    while (m.find()) {
    link = m.group(1);
    title = m.group(2);
    if (StringUtils.isNotEmpty(link)) {
    System.out.println("url : " + link);
    System.out.println("title : " + title);
    }
    }
    }
    /**
     * 测试取得内容
     * @throws ParserException
     */
    public void testFetchContent() throws ParserException{
    String html = openFile("F:\\6.shtml","GB2312");
    Map<String, String> map = new HashMap<String, String>();
    map.put("id", "artibody");
    Map<String, String> notMap = new HashMap<String, String>();
    notMap.put(SINGLE_TAG, "style|script");
    notMap.put("type", "text/javascript");
    notMap.put("class", "icon_fx|blkComment otherContent_01");
    notMap.put("style", "text-align: right;padding-right:10px;|margin-top:6px;|font-size: 12px ! important;|font-size:12px");
    notMap.put("id", "fxwb|fxMSN|fxMSN|comment_t_show_top");
    getHtmlByFilter(map,notMap,html);
    }
    /**
     * 测试解析参数
     */
    public void testParseParam(){
    Map<String, String> map = new HashMap<String, String>();
    populateParamMap(map,"class=articleList|tips,p,div");
    String tempKey = null;
    String tempValue = null;
    String[] tempValueArr = null;
    for (Iterator<String> it = map.keySet().iterator(); it.hasNext();) {
    tempKey = it.next();
    tempValue = map.get(tempKey);
    if(tempValue.contains("|")){
    tempValueArr = tempValue.split("\\|");
    }else{
    tempValueArr = new String[]{tempValue};
    }
    for(String value : tempValueArr){
    System.out.println("tempKey:" + tempKey);
    System.out.println("value:" + value);
    }
    }
    }
    /**
     * 测试过滤标签
     * @throws ParserException
     */
    public void testRemarkFilter() throws ParserException{
    String html = openFile("F:\\6.shtml","GB2312");
    System.out.println("=========================过滤注释前HTML==================================");
    System.out.println(html);
    NodeFilter filter = new NodeClassFilter(RemarkNode.class);
    html = removeHtmlByFilter(new Parser(), filter, html);
    System.out.println("=========================过滤注释后HTML==================================");
    System.out.println(html);
    }
    public static void main(String[] args) throws ParserException,
    URISyntaxException, IOException {
    HtmlParserImpl parseHtmlTool = new HtmlParserImpl(new CmsAcquisition());
    //parseHtmlTool.testParseParam();
    //parseHtmlTool.testFetchLinkAndTitle();
    //parseHtmlTool.testFetchContent();
    //parseHtmlTool.testRemarkFilter();
    }

    }
    采集参数封装bean:ParamBean.java
    package com.jeecms.cms.service;

    import java.util.HashMap;
    import java.util.Map;
    /**
     * 采集参数封装bean
     * @author javacoo
     * @since 2011-10-31
     */
    public class ParamBean {
    /**待采集连接区域属性MAP*/
    private Map<String, String> linksetStartMap = new HashMap<String, String>();
    /**待采集连接区域过滤属性MAP*/
    private Map<String, String> linksetEndMap = new HashMap<String, String>();
    /**待采集内容区域属性MAP*/
    private Map<String, String> contentStartMap = new HashMap<String, String>();
    /**待采集内容区域过滤属性MAP*/
    private Map<String, String> contentEndMap = new HashMap<String, String>();

    public Map<String, String> getLinksetStartMap() {
    return linksetStartMap;
    }
    public void setLinksetStartMap(Map<String, String> linksetStartMap) {
    this.linksetStartMap = linksetStartMap;
    }
    public Map<String, String> getLinksetEndMap() {
    return linksetEndMap;
    }
    public void setLinksetEndMap(Map<String, String> linksetEndMap) {
    this.linksetEndMap = linksetEndMap;
    }
    public Map<String, String> getContentStartMap() {
    return contentStartMap;
    }
    public void setContentStartMap(Map<String, String> contentStartMap) {
    this.contentStartMap = contentStartMap;
    }
    public Map<String, String> getContentEndMap() {
    return contentEndMap;
    }
    public void setContentEndMap(Map<String, String> contentEndMap) {
    this.contentEndMap = contentEndMap;
    }


    }
    队列类：Queue.java
    package com.jeecms.cms.service;

    import java.util.LinkedList;
    /**
     * 队列
     * @author javacoo
     * @since 2011-11-01
     * @param <T>
     */
    public class Queue<T> {
    private LinkedList<T> queue = new LinkedList<T>();
    /**
     * 入队列
     * @param t
     */
    public void enQueue(T t){
    queue.addLast(t);
    }
    /**
     * 出队列
     * @return t
     */
    public T deQueue(){
    return queue.removeFirst();
    }
    /**
     * 判断队列是否为空
     * @return
     */
    public boolean isEmpty(){
    return queue.isEmpty();
    }
    /**
     * 判断队列是否含有t
     * @param t
     * @return
     */
    public boolean contains(T t){
    return queue.contains(t);
    }
    /**
     * 取得队列大小
     * @return
     */
    public int getSize(){
    return queue.size();
    }

    }
    URL队列：UrlQueue.java

    package com.jeecms.cms.service;

    import java.util.Map;

    import org.springframework.util.CollectionUtils;

    /**
     * URL队列
     * @author javacoo
     * @since 2011-11-01
     * @param <T>
     */
    public class UrlQueue {
    /**待访问URL集合*/
    private Queue<Map<String, String>> unVisitedUrl = new Queue<Map<String, String>>();

    /**
     * 获得 URL 队列
     * @return
     */
    public Queue<Map<String, String>> getUnVisitedUrl() {
    return unVisitedUrl;
    }
    /**
     * 未访问的 URL 出队列
     * @return
     */
    public Map<String, String> unVisitedUrlDeQueue() {
    return unVisitedUrl.deQueue();
    }
    /**
     * 保证每个 URL 只被访问一次
     * @param url
     */
    public void addUnVisitedUrl(Map<String, String> urlMap) {
    if (!CollectionUtils.isEmpty(urlMap) && !unVisitedUrl.contains(urlMap)){
    unVisitedUrl.enQueue(urlMap);
    }
    }
    /**
     * 判断是否为空
     * @return
     */
    public boolean isEmpty(){
    return unVisitedUrl.isEmpty();
    }
    }