Android 网页抓取（实现新闻客户端）

如何实现从各大网抓取新闻并经过格式处理现实到我们的新闻客户端呢？
Android客户端抓取并解析网页的方法我用到的有两种：
一、用jsoup
没仔细研究，网上有类似的，可以参考这两位兄弟的：
http://decentway.iteye.com/blog/1333127
http://blog.csdn.net/hellohaifei/article/details/9352069

二、用htmlparser
我项目中就用htmlparser，抓紧并解析腾讯新闻，代码如下：

public class NetUtil {
	public static List<NewsBrief> DATALIST = new ArrayList<NewsBrief>();

	public static String[][] CHANNEL_URL = new String[][] {
		new String[]{"http://news.qq.com/world_index.shtml","http://news.qq.com"},
		new String[]{"http://news.qq.com/china_index.shtml","http://news.qq.com"},
		new String[]{"http://news.qq.com/society_index.shtml","http://news.qq.com"},
		new String[]{"http://news.qq.com/china_index.shtml","http://news.qq.com"},
		new String[]{"http://news.qq.com/china_index.shtml","http://news.qq.com"},
		new String[]{"http://news.qq.com/china_index.shtml","http://news.qq.com"},
		new String[]{"http://news.qq.com/china_index.shtml","http://news.qq.com"},
		new String[]{"http://news.qq.com/china_index.shtml","http://news.qq.com"},
		new String[]{"http://news.qq.com/china_index.shtml","http://news.qq.com"},
		new String[]{"http://news.qq.com/china_index.shtml","http://news.qq.com"},
		new String[]{"http://news.qq.com/china_index.shtml","http://news.qq.com"},
	};

	public static int getTechNews(List<NewsBrief> techData, int cId) {
		int result = 0;
		try {
			NodeFilter filter = new AndFilter(new TagNameFilter("div"),
					new HasAttributeFilter("id", "listZone"));
			Parser parser = new Parser();
			parser.setURL(CHANNEL_URL[cId][0]);
			parser.setEncoding(parser.getEncoding());
			
			NodeList list = parser.extractAllNodesThatMatch(filter);
			for (int i = 0; i < list.size(); i++) {
				Tag node = (Tag) list.elementAt(i);
				for (int j = 0; j < node.getChildren().size(); j++) {
					try {
						String textstr = node.getChildren().elementAt(j).toHtml();
						if (textstr.trim().length() > 0) {
							NodeFilter subFilter = new TagNameFilter("p");
							Parser subParser = new Parser();
							subParser.setResource(textstr);
							NodeList subList = subParser.extractAllNodesThatMatch(subFilter);

							NodeFilter titleStrFilter = new AndFilter(new TagNameFilter("a"),
									new HasAttributeFilter("class", "linkto"));
							Parser titleStrParser = new Parser();
							titleStrParser.setResource(textstr);
							NodeList titleStrList = titleStrParser.extractAllNodesThatMatch(titleStrFilter);

							int linkstart = titleStrList.toHtml().indexOf("href=\"");
							int linkend = titleStrList.toHtml().indexOf("\">");
							int titleend = titleStrList.toHtml().indexOf("</a>");
							
							String link = CHANNEL_URL[cId][1]+titleStrList.toHtml().substring(linkstart+6, linkend);
							String title = titleStrList.toHtml().substring(linkend+2, titleend);
							
							NewsBrief newsBrief = new NewsBrief();
							newsBrief.setTitle(title);
							newsBrief.setUrl(link);
							newsBrief.setSummary(subList.asString());
							techData.add(newsBrief);
						}
					} catch (Exception e) {
						e.printStackTrace();
					}
				}
			}
		} catch (Exception e) {
			result = 1;
			e.printStackTrace();
		}
		return result;
	}

	public static int getTechNews2(List<NewsBrief> techData, int cId) {
		int result = 0;
		try {
			// 查询http://tech.qq.com/tech_yejie.htm 页面 滚动新闻的 标签 以及ID
			NodeFilter filter = new AndFilter(new TagNameFilter("div"),
					new HasAttributeFilter("id", "listZone"));
			Parser parser = new Parser();
			parser.setURL(CHANNEL_URL[cId][0]);
			parser.setEncoding(parser.getEncoding());
			
			// 获取匹配的fileter的节点
			NodeList list = parser.extractAllNodesThatMatch(filter);
			StringBuilder NewsStr = new StringBuilder("<table>");// 新闻表格字符串
			for (int i = 0; i < list.size(); i++) {
				Tag node = (Tag) list.elementAt(i);
				for (int j = 0; j < node.getChildren().size(); j++) {
					String textstr = node.getChildren().elementAt(j).toHtml()
							.trim();
					if (textstr.length() > 0) {
						int linkbegin = 0, linkend = 0, titlebegin = 0, titleend = 0;
						while (true) {
							linkbegin = textstr.indexOf("href=", titleend);// 截取链接字符串起始位置

							// 如果不存在 href了 也就结束了
							if (linkbegin < 0)
								break;

							linkend = textstr.indexOf("\">", linkbegin);// 截取链接字符串结束位置
							String sublink = textstr.substring(linkbegin + 6,linkend);
							String link = CHANNEL_URL[cId][1] + sublink;
							
							titlebegin = textstr.indexOf("\">", linkend);
							titleend = textstr.indexOf("</a>", titlebegin);
							String title = textstr.substring(titlebegin + 2,titleend);

							NewsStr.append("\r\n<tr>\r\n\t<td><a target=\"_blank\" href=\""
									+ link + "\">");
							NewsStr.append(title);
							NewsStr.append("</a></td></tr>");

							NewsBrief newsBrief = new NewsBrief();
							newsBrief.setTitle(title);
							newsBrief.setUrl(link);
							techData.add(newsBrief);
						}
					}
				}
			}
		} catch (Exception e) {
			result = 1;
			e.printStackTrace();
		}
		return result;
	}
	
	public static int parserURL(String url,NewsBrief newsBrief) {
		int result = 0;
		try {
			Parser parser = new Parser(url);
			NodeFilter contentFilter = new AndFilter(
					new TagNameFilter("div"),
					new HasAttributeFilter("id","Cnt-Main-Article-QQ"));
			NodeFilter newsdateFilter = new AndFilter(
					new TagNameFilter("span"),
					new HasAttributeFilter("class",
							"article-time"));
			NodeFilter newsauthorFilter = new AndFilter(
					new TagNameFilter("span"),
					new HasAttributeFilter("class",
							"color-a-1"));
			NodeFilter imgUrlFilter = new TagNameFilter("IMG");
			
			newsBrief.setContent(parserContent(contentFilter,parser));
			parser.reset(); // 记得每次用完parser后，要重置一次parser。要不然就得不到我们想要的内容了。
			
			newsBrief.setPubDate(parserDate(newsdateFilter,parser));
			parser.reset();
			
			newsBrief.setSource(parserAuthor(newsauthorFilter, parser));
			parser.reset();
			
			newsBrief.setImgUrl(parserImgUrl(contentFilter,imgUrlFilter, parser));
			
		} catch (Exception e) {
			result=1;
			e.printStackTrace();
		}
		return result;
	}

	private static String parserContent(NodeFilter filter, Parser parser) {
		String reslut = "";
		try {
			NodeList contentList = (NodeList) parser.parse(filter);
			// 将DIV中的标签都 去掉只留正文
			reslut = contentList.asString();
		} catch (Exception e) {
			e.printStackTrace();
		}
		return reslut;
	}

	private static String parserDate(NodeFilter filter, Parser parser) {
		String reslut = "";
		try {
			NodeList datetList = (NodeList) parser.parse(filter);
			// 将DIV中的标签都 去掉只留正文
			reslut = datetList.asString();
		} catch (Exception e) {
			e.printStackTrace();
		}
		return reslut;
	}


	private static String parserAuthor(NodeFilter filter, Parser parser) {
		String reslut = "";
		try {
			NodeList authorList = (NodeList) parser.parse(filter);
			// 将DIV中的标签都 去掉只留正文
			reslut = authorList.asString();
		} catch (Exception e) {
			e.printStackTrace();
		}
		return reslut;
	}

	private static List<String> parserImgUrl(NodeFilter bodyfilter,NodeFilter filter, Parser parser) {
		List<String> reslut = new ArrayList<String>();
		try {
			NodeList bodyList = (NodeList) parser.parse(bodyfilter);
			Parser imgParser = new Parser();
			imgParser.setResource(bodyList.toHtml());
			NodeList imgList = imgParser.extractAllNodesThatMatch(filter);
			String bodyString = imgList.toHtml();

			//正文包含图片
			if (bodyString.contains("<IMG")
					&& bodyString.contains("src=")) {
				if(imgList.size()>0){
					for(int i=0;i<imgList.size();i++){
						String imgString = imgList.elementAt(i).toHtml();
						int imglinkstart = imgString.indexOf("src=\"");
						int imglinkend = imgString.indexOf(">");
						if(imgString.contains("\" alt=")){
							imglinkend = imgString.indexOf("\" alt=");
						}
						if(imgString.contains("_fck")){
							imglinkend = imgString.indexOf("_fck");// 截取链接字符串结束位置
						}
						reslut.add(imgString.substring(imglinkstart + 5, imglinkend));
					}
				}
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
		return reslut;
	}
}

附件为用到的jar包；
关于新闻完整的新闻客户端，有需要的哥们可以留下邮箱，完整实现了新闻的抓取：





工程源码在：https://github.com/gqdy365/onlineNews

评论

54 楼 shujen 2016-01-08  

兄弟大神，求份代码学习。新手。麻烦[e-mail]   敬候，谢谢

53 楼 小嘎轩遥 2016-01-04  

兄弟大神，求份代码学习。新手。麻烦[e-mail]   敬候，谢谢

52 楼 dafengyang 2015-12-27  

哥们，求版源码，邮箱[e-mail]
谢谢分享~~

51 楼 dafengyang 2015-12-27  

哥们，求版源码，邮箱[e-mail]
谢谢分享~~

50 楼 对号入座 2015-12-22  

楼主，能不能发个简单点儿的，只需要能抓取新闻并查看新闻内容及图片就行了。不需要语音，谢谢。[e-mail]

49 楼 huanchu369 2015-12-04  

[e-mail] 求源码，谢谢

48 楼 gqdy365 2015-05-20  

u010860871 写道

博主，能加下你的q吗？我有问题想和你探讨

792108314

47 楼 u010860871 2015-05-13  

博主，能加下你的q吗？我有问题想和你探讨

46 楼 gqdy365 2015-05-08  

u010649761 写道

楼主发我一份完整代码吧,这几天一直在研究这块,快急死了.谢谢了.

工程源码在：https://github.com/gqdy365/onlineNews，如果如法下载，留下邮箱，我发给你；

45 楼 u010649761 2015-05-06  

楼主发我一份完整代码吧,这几天一直在研究这块,快急死了.谢谢了.

44 楼 gqdy365 2015-04-10  

hywgtg 写道

楼主求发一份源码学习下！[e-mail]

工程源码在：https://github.com/gqdy365/onlineNews

43 楼 gqdy365 2015-04-10  

hywgtg 写道

楼主求发一份源码学习下！[e-mail]

工程源码在：https://github.com/gqdy365/onlineNews

42 楼 gqdy365 2015-04-10  

hywgtg 写道

楼主求发一份源码学习下！[e-mail]

工程源码在：https://github.com/gqdy365/onlineNews

41 楼 hywgtg 2015-04-08  

楼主求发一份源码学习下！[e-mail]

40 楼 jscxy0407 2015-03-18  

博主好强大，源码可否发我一份，谢谢啊 [e-mail]

39 楼 zzcaaa 2015-03-16  

求一个完整客户端，急求啊 邮箱[e-mail]

38 楼 hao43284 2015-03-13  

博主好强大，源码可否发我一份，谢谢啊 [e-mail]

37 楼 tianqin2011 2015-03-11  

楼主  求源码啊   [e-mail]
有邮箱或者QQ的话  也想交流交流

36 楼 胡普行 2015-02-10  

楼主，麻烦抽空给我发一份源码，本人最近项目需要这个功能。麻烦了！邮箱[e-mail]   麻烦了

35 楼 tianyaleixiaowu 2015-01-19  

博主，在你万忙的时候，抽出一会时间，送我一份新闻app的源码好吗？我的邮箱：[e-mail], 万分感谢