JSOUP抓取网页正文

package com.zs.action.admin;

import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.htmlparser.Parser;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;



public class SpiderByJsoup {
	 
	public static String marchersTitle(String args) throws InterruptedException {
		String regex = "<h\\d[^>]*>";
		Pattern p = Pattern.compile(regex);
		Matcher match = p.matcher(args);
		if (match.find()) {
			return match.group();
		}
		return null;
	}
	
    public static List<String> getLinks(String url){
    	List<String> linkList = new ArrayList<String>();
    	 try{
  	        Document doc = Jsoup.connect(url).get();
  	        Elements links = doc.select("a[href]");
  	        for (Element link : links) {
  	        	linkList.add(link.attr("abs:href"));
  	        }
  		  }catch(Exception e){
  			  e.printStackTrace();
  		  }
    	 return linkList;
    }
	  public static void main(String[] args)  {
		  String url = "http://www.chongzuo.gov.cn/Index";
		  //List<String> linkList = getLinks(url);
		//Elements el = doc.select("div.articleTxtContent");
			//System.out.println("----------"+el.text());
		  Document doc;
		try {
			 doc = 
			 Jsoup.connect("http://society.people.com.cn/n/2015/0604/c136657-27104506.html").header("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.4; en-US; rv:1.9.2.2) Gecko/20100316 Firefox/3.6.2").get();
			 String title = marchersTitle(doc.html()).substring(1,3);
			 System.out.println(title);
			 Elements hElements = doc.select(title);
			 System.out.println("<div  style='font-family: Microsoft YaHei;color: #000;font-size: 24px;line-height: 44px;text-align: center'>"+hElements.get(0).text()+"</div>"+"<p style='font-size: 16px; line-height: 30px;'>"+"</p>");
			Elements elements =doc.getElementsByTag("div");
			
			int max = elements.get(0).text().length();
			int sear = 0;
			for (int i = 1; i < elements.size(); i++) {
				int len = elements.get(i).text().length();
				if(len>max){
					sear = i;
					max = len;
				}
			}

			Elements segMentElements = elements.get(sear).getElementsByTag("div");
			
			for(Element segMentElement:segMentElements){
				if (segMentElement.select("a").size()>1){//过滤不想要的标签.  
			        continue;  
			    }
				
				Elements contentElements = segMentElement.getElementsByTag("p");
				for(Element contentElement:contentElements){
					if( contentElement.text().length()>0 && contentElement.text().length()<10  ){
						continue;
					}
					
						System.out.println("<p>"+contentElement.html()+"</p>");
				}
			}
			
			
			/*String charset = SpiderByJsoup.getCharset("http://www.chongzuo.gov.cn/front/newOnly?id=13002");
			System.out.println("charset----------------"+charset);
			Parser parser = Parser.createParser(doc.body().text(),charset);*/
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
		
	    }
	  /*
	  public static String getCharset (String siteurl) throws Exception{
			URL url = new URL(siteurl);
			Document doc = Jsoup.parse(url, 6*1000);
			Elements eles = doc.select("meta[http-equiv=Content-Type]");
			Iterator<Element> itor = eles.iterator();
			while (itor.hasNext()) 
				return SpiderByJsoup.matchCharset(itor.next().toString());
			return "gb2312";
		}
	  public static String matchCharset(String content) {
			String chs = "gb2312";
			Pattern p = Pattern.compile("(?<=charset=)(.+)(?=\")");
			Matcher m = p.matcher(content);
			if (m.find())
				return m.group();
			return chs;
		}*/
}