浏览 10106 次
锁定老帖子 主题:多线程断点续传
精华帖 (0) :: 良好帖 (0) :: 新手帖 (1) :: 隐藏帖 (9)
|
|
---|---|
作者 | 正文 |
发表时间:2011-01-12
以前看过一个前辈写的断点续传的文章,记得当时没看懂,就扔那了。昨天翻了出来,自己也仿照写了一个,不过感觉没人家写的好,有点乱,希望大家能来批评、指正,给点意见!
功能很简单,就是启动多个线程分别从给定的地址下载数据,用RandomAccessFile写到目标文件。实现思路是: 1、获得连接的长度(即要下载的文件大小),除以设定的线程数,即得到每个线程要下载的大小。 2、记录临时文件,文件中记录每个线程的编号(id),该线程要下载的起始位置、终止位置和当前位置(当前位置在首次下载时与起始位置相同)。 3、启动具体执行下载任务的线程,并等待其结束。 4、下载完成,删除临时文件。
代码如下: 主线程及测试的main方法 package com.why.download.test; import java.io.DataOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.math.BigDecimal; import java.net.HttpURLConnection; import java.net.URL; import java.net.URLConnection; import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; /** * * @author why * */ public class DownLoad { //文件目录、文件名 public String fileDir = "E:/MyDownLoad"; public String fileName; //超时重连时间 public long reconnectTime = 5; //线程数 private int poolSize = 5; //每个线程的缓冲区大小 public int bufferSize = 1024; //url地址 private String urlLocation = null; public DownLoad(){} public DownLoad(String url){ this.urlLocation = url; } public void downLoad(){ if(this.urlLocation == null || "".equals(this.urlLocation))return; downLoad(this.urlLocation); } public void downLoad(String urlLocation){ File file = null; File tempFile = null; CountDownLatch latch; URL url = null; ExecutorService pool = Executors.newCachedThreadPool(); long contentLength = 0; long threadLength = 0; try { //如果未指定名称,则从url中获得下载的文件格式与名字 if(fileName == null || "".equals(fileName)){ this.fileName = urlLocation.substring(urlLocation.lastIndexOf("/") + 1, urlLocation.lastIndexOf("?") > 0 ? urlLocation.lastIndexOf("?") : urlLocation.length()); if ("".equalsIgnoreCase(this.fileName)) { this.fileName = UUID.randomUUID().toString(); } } new File(fileDir).mkdirs(); file = new File(fileDir + File.separator + fileName); tempFile = new File(fileDir + File.separator + fileName + "_temp"); url = new URL(urlLocation); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); setHeader(conn); //得到content的长度 contentLength = conn.getContentLength(); System.out.println("total length=" + contentLength); //把context分为poolSize段,计算每段的长度。 // threadLength = contentLength / this.poolSize; BigDecimal b1 = new BigDecimal(Double.toString(contentLength)); BigDecimal b2 = new BigDecimal(Double.toString(this.poolSize)); threadLength = b1.divide(b2, 0, BigDecimal.ROUND_HALF_UP).longValue(); if(file.exists() && tempFile.exists()){ //如果文件已存在,根据临时文件中记载的线程数量,继续上次的任务 latch = new CountDownLatch((int)tempFile.length()/28); for(int i=0;i<tempFile.length()/28;i++){ pool.submit(new DownLoadTask(file, tempFile, url, i+1,latch,reconnectTime,bufferSize)); } }else{ //如果下载的目标文件不存在,则创建新文件 latch = new CountDownLatch(poolSize); file.createNewFile(); tempFile.createNewFile(); DataOutputStream os = new DataOutputStream(new FileOutputStream(tempFile)); for(int i=0;i<this.poolSize;i++){ os.writeInt(i+1); os.writeLong(i*threadLength); if(i==this.poolSize-1){//最后一个线程的结束位置应为文件末端 os.writeLong(contentLength); }else{ os.writeLong((i+1)*threadLength); } os.writeLong(i*threadLength); pool.submit(new DownLoadTask(file, tempFile, url, i+1,latch,reconnectTime,bufferSize)); } os.close(); } //等待下载任务完成 latch.await(); //删除临时文件 tempFile.delete(); } catch (IOException e) { e.printStackTrace(); } catch (InterruptedException e) { e.printStackTrace(); } finally{ pool.shutdown(); } } private void setHeader(URLConnection conn) { conn.setRequestProperty("User-Agent", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.3) Gecko/2008092510 Ubuntu/8.04 (hardy) Firefox/3.0.3"); conn.setRequestProperty("Accept-Language", "en-us,en;q=0.7,zh-cn;q=0.3"); conn.setRequestProperty("Accept-Encoding", "aa"); conn.setRequestProperty("Accept-Charset","ISO-8859-1,utf-8;q=0.7,*;q=0.7"); conn.setRequestProperty("Keep-Alive", "300"); conn.setRequestProperty("Connection", "keep-alive"); conn.setRequestProperty("If-Modified-Since", "Fri, 02 Jan 2009 17:00:05 GMT"); conn.setRequestProperty("If-None-Match", "\"1261d8-4290-df64d224\""); conn.setRequestProperty("Cache-Control", "max-age=0"); conn.setRequestProperty("Referer","http://www.skycn.com/soft/14857.html"); } public String getFileDir() { return fileDir; } public void setFileDir(String fileDir) { this.fileDir = fileDir; } public String getFileName() { return fileName; } public void setFileName(String fileName) { this.fileName = fileName; } public long getReconnectTime() { return reconnectTime; } public void setReconnectTime(long reconnectTime) { this.reconnectTime = reconnectTime; } public int getPoolSize() { return poolSize; } public void setPoolSize(int poolSize) { this.poolSize = poolSize; } public int getBufferSize() { return bufferSize; } public void setBufferSize(int bufferSize) { this.bufferSize = bufferSize; } /** * @param args */ public static void main(String[] args) { DownLoad dl = new DownLoad(); dl.setFileDir("E:/MyDownLoad/music/"); dl.setFileName("大笑江湖.mp3"); dl.setPoolSize(20); long beginTime = System.currentTimeMillis(); dl.downLoad("http://mh.163k.com/UploadFile/video/2010/12-13/201012131213448942190.mp3"); long endTime = System.currentTimeMillis(); BigDecimal b1 = new BigDecimal(endTime - beginTime); BigDecimal b2 = new BigDecimal(1000); double cost = b1.divide(b2, 2, BigDecimal.ROUND_HALF_UP).doubleValue(); System.out.println("Time cost:" + cost + "s"); } }
执行下载任务的线程: package com.why.download.test; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.RandomAccessFile; import java.net.HttpURLConnection; import java.net.URL; import java.net.URLConnection; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; /** * * @author why * */ public class DownLoadTask implements Callable<String>{ //超时重连时间 private long reconnectTime = 5; //缓冲区大小 private int bufferSize = 1024; private CountDownLatch latch; private RandomAccessFile file = null; private RandomAccessFile tempFile = null; private URL url = null; private int id; private long startPosition; private long endPosition; private long currentPosition ; public DownLoadTask(File file,File tempFile,URL url,int id,CountDownLatch latch,long reconnectTime,int bufferSize){ try { this.file = new RandomAccessFile(file, "rw"); this.tempFile = new RandomAccessFile(tempFile, "rw"); } catch (FileNotFoundException e) { e.printStackTrace(); } this.url = url; this.id = id; this.latch = latch; } public String call(){ try { tempFile.seek((id-1)*28); tempFile.readInt(); this.startPosition = tempFile.readLong(); this.endPosition = tempFile.readLong(); this.currentPosition = tempFile.readLong(); } catch (IOException e) { e.printStackTrace(); } System.out.println("Thread " + id + " begin!"); HttpURLConnection conn = null; InputStream inputStream = null; while(true){ try { tempFile.seek(id*28 - 8); // 打开URLConnection conn = (HttpURLConnection) this.url.openConnection(); setHeader(conn); // 设置连接超时时间为10000ms conn.setConnectTimeout(10000); // 设置读取数据超时时间为10000ms conn.setReadTimeout(10000); if (currentPosition < endPosition) { // 设置下载数据的起止区间 conn.setRequestProperty("Range", "bytes=" + currentPosition + "-" + endPosition); System.out.println("Thread " + id + " startPosition=" + startPosition + ",endPosition=" + endPosition + ",currentPosition=" + currentPosition); file.seek(currentPosition); // 判断http status是否为HTTP/1.1 206 Partial Content或者200 OK // 如果不是以上两种状态,把status改为STATUS_HTTPSTATUS_ERROR if (conn.getResponseCode() != HttpURLConnection.HTTP_OK && conn.getResponseCode() != HttpURLConnection.HTTP_PARTIAL) { System.out.println("Thread " + id + ": code = " + conn.getResponseCode() + ", status = " + conn.getResponseMessage()); file.close(); conn.disconnect(); System.out.println("Thread " + id + " finished."); break; } inputStream = conn.getInputStream(); int len = 0; byte[] b = new byte[bufferSize]; while ((len = inputStream.read(b)) != -1) { file.write(b, 0, len); currentPosition += len; // set tempFile now position tempFile.seek(id*28 - 8); tempFile.writeLong(currentPosition); } file.close(); tempFile.close(); inputStream.close(); conn.disconnect(); } System.out.println("Thread " + id + " finished."); break; } catch (IOException e) { try { TimeUnit.SECONDS.sleep(getReconnectTime()); } catch (InterruptedException e1) { e1.printStackTrace(); } continue; } } latch.countDown(); return "finish"; } private void setHeader(URLConnection conn) { conn.setRequestProperty("User-Agent", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.3) Gecko/2008092510 Ubuntu/8.04 (hardy) Firefox/3.0.3"); conn.setRequestProperty("Accept-Language", "en-us,en;q=0.7,zh-cn;q=0.3"); conn.setRequestProperty("Accept-Encoding", "aa"); conn.setRequestProperty("Accept-Charset","ISO-8859-1,utf-8;q=0.7,*;q=0.7"); conn.setRequestProperty("Keep-Alive", "300"); conn.setRequestProperty("Connection", "keep-alive"); conn.setRequestProperty("If-Modified-Since", "Fri, 02 Jan 2009 17:00:05 GMT"); conn.setRequestProperty("If-None-Match", "\"1261d8-4290-df64d224\""); conn.setRequestProperty("Cache-Control", "max-age=0"); conn.setRequestProperty("Referer","http://www.skycn.com/soft/14857.html"); } public long getReconnectTime() { return reconnectTime; } public void setReconnectTime(long reconnectTime) { this.reconnectTime = reconnectTime; } public int getBufferSize() { return bufferSize; } public void setBufferSize(int bufferSize) { this.bufferSize = bufferSize; } } 最近一直在测试,写文档,好久没敲代码了。手有点痒,写着玩的,代码写的不是很工整,纯属娱乐!
声明:ITeye文章版权属于作者,受法律保护。没有作者书面许可不得转载。
推荐链接
|
|
返回顶楼 | |
发表时间:2011-01-14
没做过类似的功能,学习下
|
|
返回顶楼 | |
发表时间:2011-01-14
以前用java写过,曾经想用as也写一个,发现as实现比较难。。。
|
|
返回顶楼 | |
发表时间:2011-01-14
好东西,我找了很久了,能用的没几个,这个很好
|
|
返回顶楼 | |
发表时间:2011-04-27
有些地方不太明白,请教:
1:tempFile.seek(id*28 -; 2://如果文件已存在,根据临时文件中记载的线程数量,继续上次的任务 latch = new CountDownLatch((int)tempFile.length()/28); 这里的28是什么?是怎么计算的? |
|
返回顶楼 | |
发表时间:2011-05-09
找了好几例子都不能用,试试这个。。。。
|
|
返回顶楼 | |
发表时间:2011-05-09
ltx 写道 有些地方不太明白,请教: 1:tempFile.seek(id*28 -; 2://如果文件已存在,根据临时文件中记载的线程数量,继续上次的任务 latch = new CountDownLatch((int)tempFile.length()/28); 这里的28是什么?是怎么计算的? 最近一直没上javaeye,今天才看到,不好意思,呵呵,道歉先! 我在临时文件中记录了每个线程的编号、起始位置、终止位置和当前位置,分别为一个int型(4个字节)和3个long型(8个字节),加起来就是28了。 |
|
返回顶楼 | |