又一采集器

又一采集器，呵呵

JDBC.java

package com.baoruan;
import java.sql.*;

public class JDBC {
	Connection con;

	public Connection getConnection() {
		try {
			Class.forName("com.mysql.jdbc.Driver");
			System.out.println("数据库驱动加载成功！");
		} catch (ClassNotFoundException e) {
			e.printStackTrace();
		}
		try {
			con = DriverManager.getConnection(
					"jdbc:mysql://localhost/baoruan?useUnicode=true&characterEncoding=UTF-8", "root", "123456");

			System.out.println("数据库连接成功！");
		} catch (SQLException e) {
			e.printStackTrace();
		}
		return con;
	}

	public static void main(String[] args) {
		new JDBC().getConnection();
	}
}

JDBCConnect.java

package com.baoruan;

import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;

public class JDBCConnect {
	static Connection con;
	static PreparedStatement sql;

	public JDBCConnect() {
		JDBC conn = new JDBC();
		con = conn.getConnection();
	}

	public ResultSet read(String sqlworld) {
		ResultSet res = null;
		try {
			sql = con.prepareStatement(sqlworld);
			res = sql.executeQuery();
			// System.out.println("执行增加，修改，删除前后数据！");
			// while(res.next()){
			// HashMap <String,String>brandlist=new HashMap<String,String>();
			// id=res.getString("id");
			// String brand=res.getString("brand");
			// String downloadlist=res.getString("downloadlist");
			/*
			 * String gameName=res.getString("gameName"); String
			 * gameType=res.getString("gameType");
			 * 
			 * System.out.println("id 为："+id);
			 * System.out.println("gameid 为："+gameid);
			 * System.out.println("downloadlist 为："+downloadlist);
			 * System.out.println("gameName 为："+gameName);
			 * System.out.println("gameType 为："+gameType);
			 */
			// }
		} catch (Exception e) {
			e.printStackTrace();
			System.out.println("数据库读取异常");
		}
		return res;
	}

	public void update(String sqlworld) {
		try {
			sql = con.prepareStatement(sqlworld);
			sql.executeUpdate();
			sql.close();
			System.out.println("更新成功！");
		} catch (Exception e) {
			System.out.println("更新异常！");
			e.printStackTrace();
		}
	}
	/*
	 * public static void main(String[] args)throws Exception{ hellosql re=new
	 * hellosql(); // String
	 * sql="select COUNT(brand) AS GameNum from jixing where id =50"; //
	 * ResultSet res=re.read(sql); // res.last(); // int count=res.getRow(); //
	 * if(count==0){ // System.out.println("没有该值 "); // }else{ //
	 * System.out.println("有该值 "); // } // 获取行数 // // while(res.next()){ //
	 * String id=res.getString("id"); // String brand=res.getString("brand"); //
	 * String mobileType=res.getString("mobileType"); // String
	 * url=res.getString("url"); //
	 * System.out.println(id+"       "+brand+"        "
	 * +mobileType+"          "+url); // } //
	 * System.out.println(res.getInt("GameNum")); // if(k==0){ //
	 * System.out.println(k); // System.out.println("该值已存在！"); // }else{ //
	 * System.out.println(k); // System.out.println("该值可以保存！"); // } String
	 * sql2="UPDATE sr_soft SET adaptedMobileTypeIds = '88' WHERE softId =61";
	 * re.upadte(sql2); String sql1 = "select * from sr_soft where softId=61";
	 * ResultSet gameIsExist = re.read(sql1); String
	 * adaptedMobileTypeIds_constant=""; while(gameIsExist.next()){
	 * adaptedMobileTypeIds_constant
	 * =gameIsExist.getString("adaptedMobileTypeIds");
	 * System.out.println("读一下机型表"
	 * +adaptedMobileTypeIds_constant);//取adaptedMobileTypeIds字段的值 }
	 * 
	 * }
	 */
}

SavePackageThread.java

package com.baoruan;

import java.io.BufferedInputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;

public class SavePackageThread implements Runnable{
	private String url;
	private String savePath;
	
	public SavePackageThread(){
		
	}
	public SavePackageThread(String url, String savePath){
		this.url = url;
		this.savePath = savePath;
	}
	
	public void checkFile() throws Exception{
		File file = new File(savePath);
		File parentFile = file.getParentFile();
		if(!parentFile.isDirectory())parentFile.mkdirs();
		if(!file.exists())file.createNewFile();
	}
	
	 public InputStream getInputStream(){  
        InputStream is = null;  
        try{  
            if(this.url != null && !"".equals(this.url)){  
                URL url = new URL(this.url);  
                HttpURLConnection httpConn = (HttpURLConnection)url.openConnection();  
                httpConn.setRequestProperty("Connection", "Keep-Alive");  //保持一直连接  
                httpConn.setConnectTimeout(60 * 1000 * 5);                //连接超时5分钟  
                httpConn.setRequestMethod("POST");                         //以GET方式连接  
                httpConn.setAllowUserInteraction(true);  
                return new BufferedInputStream(httpConn.getInputStream(),1024*8);  
            }  
        }catch(Exception ex){  
            ex.printStackTrace();  
        }  
        return is;  
    }  
	
	@Override
	public void run() {
		try{
			checkFile();
			// 此方法只能用于HTTP协议
			File path = new File(this.savePath).getParentFile();
			if (!path.isDirectory()) {
				path.mkdirs();
			}
			try {
				URL url = new URL(this.url);
				HttpURLConnection connection = (HttpURLConnection) url
						.openConnection();
				DataInputStream in = new DataInputStream(connection
						.getInputStream());
				DataOutputStream out = new DataOutputStream(new FileOutputStream(
						this.savePath));
				byte[] buffer = new byte[4096];
				int count = 0;
				while ((count = in.read(buffer)) > 0) {
					out.write(buffer, 0, count);
				}
				out.close();
				in.close();
			} catch (Exception e) {
				path.delete();
			}
			
		}catch(Exception ex){
			ex.printStackTrace();
		}
	}
}

Client.java

package com.baoruan;

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.sql.ResultSet;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;


public class Client {
	private List<String> imageUrlList = new ArrayList<String>();
	
	private static final int BUFFER_SIZE = 1024*80;
	
	public List<String> getImageUrlList() {
		return imageUrlList;
	}
	public void setImageUrlList(List<String> imageUrlList) {
		this.imageUrlList = imageUrlList;
	}
	public static void main(String[] args) throws Exception {
		JDBCConnect connect = new JDBCConnect();
		String baseLink_left = "http://baoruan.com/download/downpage/freegamedown/gid/";//1100279~~1109419
		String baseLink_right = "/nopage/1/mid/12418";
		//1105572
		for(int i=1105572; i<1109420; i++){
			Client client = new Client();
			String urlAddress = baseLink_left + i + baseLink_right;
			System.out.println("urlAddress---------------->" + urlAddress);
			String web_Content = client.httpclient(urlAddress);
//			String web_Content = client.getStringFromReader(client.getReader(urlAddress));
			if(web_Content.length()<1000)continue;
			String gameName = ""; 					//游戏名称
			String version = ""; 					//版本号
			String description = ""; 				//游戏说明
			String category = "";					//游戏分类
			String imageurl = "";					//图片地址
			String apkurl = "";						//游戏包地址	
			String remoteApkurl = "";				//远程游戏包地址
			String model = "";
			String gameNameSub = client.indexOfContent(web_Content, "<card id=\"main\" title=\"", "-宝软网");
			if(gameNameSub == null || "".equals(gameNameSub))continue;
			if(gameNameSub.split("v\\d").length >1){
				String[] st = gameNameSub.split("v");
				gameName = st[0];
				version = st[1];
			}else{
				gameName = gameNameSub;
			}
			
			String updatesql = "select * from baoruan where gameName = '" + gameName + "'"; 
			ResultSet gameIsExist = connect.read(updatesql);
			String id="";
			while(gameIsExist.next()){
				id = gameIsExist.getString("id");
			}
			gameIsExist.last();
			if (gameIsExist.getRow() != 0) { // 如果游戏资料存在，直接跳到下载游戏包
				gameIsExist.close();
				String apkurlCopy = client.indexOfContent(web_Content, "<img src=\"http://baoruan.com/images/down.gif\" alt='' />			<a href=\"", "\">下载安装");
				if(apkurlCopy != null){
					System.out.println("apkurlCopy------------->" + apkurlCopy);
					apkurlCopy = "http://baoruan.com" + apkurlCopy;
					System.out.println("apkurlCopy---------------->" + apkurlCopy);
					String gamePackagePage = client.httpclient(apkurlCopy);
					System.out.println("gamePackagePage----------------->" + gamePackagePage);
					if(gamePackagePage != null){
						remoteApkurl = client.indexOfContent(gamePackagePage, "ontimer=\"", "\"");
					}
				}
				
				
				apkurl = "C:/upload/" + id + "/" + id + ".apk";
				new Thread(new SavePackageThread(remoteApkurl, apkurl)).start();
				if(version != "" && "".equals(version)){
					String sql = "update baoruan set version = '" + version +"' where id = " + id;
					connect.update(sql);
				}
				
				continue;
			}
			description = client.indexOfContent(web_Content, "介绍:", "<img src=\"http://baoruan.com/images/down.gif\" alt='' />");
			model = client.indexOfContent(web_Content, "当前适配机型:", "<br />");
			String categoryCopy = client.indexOfContent(web_Content, "分类:", "<a href=\"/download/cutpic/show");
			String apkurlCopy = client.indexOfContent(web_Content, "<img src=\"http://baoruan.com/images/down.gif\" alt='' />			<a href=\"", "\">下载安装");
			if(apkurlCopy != null){
				System.out.println("apkurlCopy------------->" + apkurlCopy);
				apkurlCopy = "http://baoruan.com" + apkurlCopy;
				System.out.println("apkurlCopy---------------->" + apkurlCopy);
				String gamePackagePage = client.httpclient(apkurlCopy);
				System.out.println("gamePackagePage----------------->" + gamePackagePage);
				if(gamePackagePage != null){
					remoteApkurl = client.indexOfContent(gamePackagePage, "ontimer=\"", "\"");
				}
			}
			
			
			apkurl = "C:/upload/" + i + "/" + i + ".apk";
			new Thread(new SavePackageThread(remoteApkurl, apkurl)).start();
			category = client.gameTypeindexof(categoryCopy);
			
			//处理截图
			imageurl = client.indexOfContent(web_Content, "分类:", "截图</a>");
			imageurl = "http://baoruan.com" + client.indexOfContent(imageurl, "</a><a href=\"", "\">");
			System.out.println("imageurl---------->" + imageurl);
			client.getImageUrlFromURLFirst(imageurl, "C:" + "/" +"upload" + "/" + i + "/");
			
			
			imageurl = client.getStringFromList(",");
			imageurl = imageurl.replaceAll("C:/", "");
			apkurl = apkurl.replaceAll("C:/", "");
			description = description.replaceAll("'", "‘");
			System.out.println("gameName------------->" + gameName);
			System.out.println("version------------->" + version);
			System.out.println("description------------->" + description);
			System.out.println("category------------->" + category);
			System.out.println("apkurl------------->" + apkurl);
			System.out.println("imageurl------------->" + client.getStringFromList(",")); 
			



			
			String sql = "insert into baoruan(gameName, category, imageurl, description, version, apkurl, model)values('" + gameName +"','" + category +"','" + imageurl +"','" + description +"','" + version +"','" + apkurl +"','" + model +"')";
			System.out.println("sql_----------------->" +sql);
			connect.update(sql);
		}
	}
	//去除html代码
	public String gameTypeindexof(String content) {
		if(content == "" || "".equals(content))return content;
		Pattern patt = Pattern.compile("<[^>]+>([^<]*)</[^>]+>");
		Matcher m = patt.matcher(content);
		while (m.find()) {
			content = content.replaceFirst("<[^>]+>([^<]*)</[^>]+>", m.group(1)
					.toString());
		}
		return content;
	}
	//去除html代码
	
	//拿到APK游戏包
	/*public String getGamePackageUrl(String url, String path){
		String wap_content = null;
		try {
			HttpClient httpclient = new HttpClient();
			httpclient.getParams().setParameter(HttpMethodParams.USER_AGENT,"Mozilla/5.0 (X11; U; Linux i686; zh-CN; rv:1.9.1.2) Gecko/20090803 Fedora/3.5.2-2.fc11 Firefox/3.5.2");
			GetMethod getmethod = new GetMethod(url);
//			使用系统提供的默认的恢复策略  默认连接失败后重复连接3次
			getmethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler());
			getmethod.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "UTF-8");
			int code = httpclient.executeMethod(getmethod);
			if (code != HttpStatus.SC_OK) {
				throw new Exception("网页读取失败！");
//				 continue;
			}
			BufferedInputStream bis = getInputStream(url);
			FileOutputStream fos = new FileOutputStream(path);
			byte[] by = new byte[4096];
			int length = 0;
			while(-1 != (length = bis.read(by))){
				fos.write(by, 0, length);
			}
			fos.close();
			bis.close();
			
			getmethod.releaseConnection();//关闭getmethod连接
			return wap_content;
		} catch (Exception e) {
			e.printStackTrace();
			return wap_content;
		}
	}
	//拿到APK游戏包
*/	
	//正则分析
	public String indexOfContent(String content, String head, String footer) {
		int index;
		int index1;
		String newio = null;
		index = content.indexOf(head);
		if (index >= 0) {
			String inputreader1 = content.substring(index + head.length());
			index1 = inputreader1.indexOf(footer);
			if (index1 >= 0) {
				newio = inputreader1.substring(0, index1);
			}
		}
		return newio;
	}
	//正则分析
	
	/*---httpclient---*/
	public String httpclient(String url) {
		String wap_content = null;
		try {
			HttpClient httpclient = new HttpClient();
			httpclient.getParams().setParameter(HttpMethodParams.USER_AGENT,"Mozilla/5.0 (X11; U; Linux i686; zh-CN; rv:1.9.1.2) Gecko/20090803 Fedora/3.5.2-2.fc11 Firefox/3.5.2");
			GetMethod getmethod = new GetMethod(url);
//			使用系统提供的默认的恢复策略  默认连接失败后重复连接3次
			getmethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
			    new DefaultHttpMethodRetryHandler());
			getmethod.getParams().setParameter(
					HttpMethodParams.HTTP_CONTENT_CHARSET, "UTF-8");
			int code = httpclient.executeMethod(getmethod);
			if (code != HttpStatus.SC_OK) {
				throw new Exception("网页读取失败！");
//				 continue;
			}
			BufferedReader buffer = new BufferedReader(new InputStreamReader(
					getmethod.getResponseBodyAsStream(), "UTF-8"));
			StringBuffer string_buffer = new StringBuffer();
			String snap = "";
			while ((snap = buffer.readLine()) != null) {
				string_buffer.append(snap);
			}
			wap_content = string_buffer.toString();
			getmethod.releaseConnection();//关闭getmethod连接
			return wap_content;
		} catch (Exception e) {
			e.printStackTrace();
			return wap_content;
		}

	}

	/*---httpclient---*/
	
	
	public BufferedInputStream getInputStream(String urlAddress){
		BufferedInputStream is = null;
		try{
			URL url = new URL(urlAddress);
			HttpURLConnection httpConn = (HttpURLConnection)url.openConnection();
			   httpConn.setRequestProperty("Connection", "Keep-Alive");  //保持一直连接  
               httpConn.setConnectTimeout(60 * 1000 * 5);                //连接超时5分钟  
               httpConn.setRequestMethod("GET");                         //以GET方式连接  
               httpConn.setAllowUserInteraction(true);  
           is = new BufferedInputStream(httpConn.getInputStream(), Client.BUFFER_SIZE);
		}catch(Exception ex){
			return null;
		}
		return is;
	}
	
	public String getStringFromReader(Reader reader){
		if(reader == null)return null;
		StringBuffer sb = new StringBuffer();
		try{
			BufferedReader br = (BufferedReader)reader;
			String str = null;
			while(null != (str = br.readLine())){
				sb.append(str);
			}
		}catch(Exception ex){
			ex.printStackTrace();
		}
		
		return sb.toString();
	}
	
	public void getImageUrlFromURLFirst(String urlAddress,String savePath){
		if(urlAddress == null || "".equals(urlAddress))return;
		String content = httpclient(urlAddress);
		if(content == null || "".equals(content))return;
		System.out.println("content------------>" + content);
		String urlSpan = indexOfContent(content,".jpg\" /><br />","图片经过压缩");
		if(urlSpan == null || "".equals(urlSpan))return;
		if(urlSpan.split("<a href = \"").length >1){
			String[] urlList = urlSpan.split("<a href = \"");
			for(String url : urlList){
				if(url.length()<10)continue;
				System.out.println(url.split("\">")[0]);
				getImageUrlFromURL("http://baoruan.com" + url.split("\">")[0], savePath);
			}
		}
	}
	
	public void getImageUrlFromURL(String urlAddress,String savePath){
		if(urlAddress == null || "".equals(urlAddress))return;
		String content = httpclient(urlAddress);
		if(content == null || "".equals(content))return;
		System.out.println("content------------>" + content);
		String hotaddress = indexOfContent(content,"-=截图欣赏=-<br /><img src=\"","\" />");
		if(hotaddress == null || "".equals(hotaddress))return;
		
		if(hotaddress.indexOf(".jpg")>0){
			String imageUrl = getImageName(savePath);
			new Thread(new SavePackageThread(hotaddress, imageUrl)).start();
			this.imageUrlList.add(imageUrl);
		}
	}
	public String getImageName(String savePath){
		int i = 1;
		while(new File(savePath + i + ".jpg").exists()){
			i++;
		}
		String imageName = savePath + i + ".jpg";
		return imageName;
	}
	public String getStringFromList(String joinSign){
		StringBuffer sb = new StringBuffer();
		for(String str : this.imageUrlList){
			sb.append(str + joinSign);
		}
		return sb.toString();
	}
}