下载上传wget，附带java代码一份

场景1：
项目放到国外服务器，配置ftp，上传老掉线，网速实在不如人意

解决：
wget方式下载，-c就ok，把你的东西放到一个url可以直接下载的地方，俺测试下了360云盘，报错不支持，可以找速度快并且稳定的地方上次，不行那就自己动手，放nginx里面，远程下载，不用操心
nuhop xxx & ，到时候看nohup.out就ok

场景2
收集网页分析数据，wget方式可以下载全站的，如果觉得速度太慢，不给力，自己动手
附带简单的下载代码

public class SimpleDownLoadHtmlUtil  {

	public Logger log = Logger.getLogger(getClass());

	public static class DownLoadParams {
		// 下载链接
		private String uri;
		// 开始Id
		private int startId;
		// 结束id
		private int endId;
		// 下载存放目录
		private String downLoadDir;

		public String getUri() {
			return uri;
		}

		public void setUri(String uri) {
			this.uri = uri;
		}

		public int getStartId() {
			return startId;
		}

		public void setStartId(int startId) {
			this.startId = startId;
		}

		public int getEndId() {
			return endId;
		}

		public void setEndId(int endId) {
			this.endId = endId;
		}

		public String getDownLoadDir() {
			return downLoadDir;
		}

		public void setDownLoadDir(String downLoadDir) {
			this.downLoadDir = downLoadDir;
		}
		
	}

	public void downLoadHtmls(DownLoadParams downLoadParams) {

		HttpClient httpClient = new HttpClient();
		long start = System.currentTimeMillis();
		String url =downLoadParams.getUri();
		for (int i = downLoadParams.getStartId(); i <= downLoadParams
				.getEndId(); i++) {
			try {
				long startP = System.currentTimeMillis();
				String currentUrl = url + i;
				GetMethod getMethod = new GetMethod(currentUrl);
				httpClient.executeMethod(getMethod);
				saveFile(getMethod.getResponseBodyAsStream(),
							downLoadParams.getDownLoadDir() + i + ".html");
				long endP = System.currentTimeMillis();
					log.info("waste" + (endP - startP) + "----rid" + i);
			} catch (Exception e) {
				log.error("exception" + e + "-->>" + i);
			}
		}
		long end = System.currentTimeMillis();
		log.info("waste over" + (end - start));

	}

	public static void saveFile(InputStream in, String filePath)
			throws IOException {
		File file = new File(filePath);
		file.createNewFile();
		FileOutputStream fileOutputStream = new FileOutputStream(file);
		IOUtils.copy(in, fileOutputStream);
		fileOutputStream.close();
		in.close();
	}
	/**
	 * 数据量大使用，线程数根据cpu个数与网络决定，常规为cpu个数的倍数，可以测试使用，最少为4，
	 * @param downLoadParams
	 * @param threadCount
	 */
	public  static void SpeedUpDownd(DownLoadParams downLoadParams,int threadCount){
		 class  MultiProcess implements Runnable {
			 private DownLoadParams downLoadParams;
			
				public void setDownLoadParams(DownLoadParams downLoadParams) {
					this.downLoadParams = downLoadParams;
				}
			@Override
			public void run() {
				new SimpleDownLoadHtmlUtil().downLoadHtmls(downLoadParams);
			}
		}
		
		int total = downLoadParams.endId-downLoadParams.startId+1;
		ArrayBlockingQueue<Runnable> arrayQue = new ArrayBlockingQueue<Runnable>(
				threadCount - 4);
		ExecutorService service = new ThreadPoolExecutor(50, 50, 2,
						TimeUnit.DAYS, arrayQue);
		for (int i = 0; i <threadCount; i++) {
			MultiProcess multiProcess=new MultiProcess();
			DownLoadParams currentParams=new DownLoadParams(); 
			currentParams.setStartId((total/threadCount)*i+1);
			currentParams.setEndId((total/threadCount)*(i+1));
			currentParams.setDownLoadDir(downLoadParams.getDownLoadDir());
			currentParams.setUri(downLoadParams.getUri());
			multiProcess.setDownLoadParams(currentParams);
			service.execute(multiProcess);
		}
		service.shutdown();
		
	}

	
	

}