Java多线程断点下载

mousefat

贡献于2012-04-01

字数:14558 关键词: Java开发 Java

Java 多线程断点下载文件 基本原理:利用URLConnection获取要下载文件的长度、头部等相关信息,并设置响应的头部信息。并且通过URLConnection获取输入流,将文件分成指定的块,每一块单独开辟一个线程完成数据的读取、写入。通过输入流读取下载文件的信息,然后将读取的信息用RandomAccessFile随机写入到本地文件中。同时,每个线程写入的数据都文件指针也就是写入数据的长度,需要保存在一个临时文件中。这样当本次下载没有完成的时候,下次下载的时候就从这个文件中读取上一次下载的文件长度,然后继续接着上一次的位置开始下载。并且将本次下载的长度写入到这个文件中。 个人博客: http://hoojo.cnblogs.com http://blog.csdn.net/IBM_hoojo email: hoojo_@126.com 一、下载文件信息类、实体 封装即将下载资源的信息 package com.hoo.entity; /** * function: 下载文件信息类 * @author hoojo * @createDate 2011-9-21 下午05:14:58 * @file DownloadInfo.java * @package com.hoo.entity * @project MultiThreadDownLoad * @blog http://blog.csdn.net/IBM_hoojo * @email hoojo_@126.com * @version 1.0 */ public class DownloadInfo { //下载文件url private String url; //下载文件名称 private String fileName; //下载文件路径 private String filePath; //分成多少段下载, 每一段用一个线程完成下载 private int splitter; //下载文件默认保存路径 private final static String FILE_PATH = "C:/temp"; //默认分块数、线程数 private final static int SPLITTER_NUM = 5; public DownloadInfo() { super(); } /** * @param url 下载地址 */ public DownloadInfo(String url) { this(url, null, null, SPLITTER_NUM); } /** * @param url 下载地址url * @param splitter 分成多少段或是多少个线程下载 */ public DownloadInfo(String url, int splitter) { this(url, null, null, splitter); } /*** * @param url 下载地址 * @param fileName 文件名称 * @param filePath 文件保存路径 * @param splitter 分成多少段或是多少个线程下载 */ public DownloadInfo(String url, String fileName, String filePath, int splitter) { super(); if (url == null || "".equals(url)) { throw new RuntimeException("url is not null!"); } this.url = url; this.fileName = (fileName == null || "".equals(fileName)) ? getFileName(url) : fileName; this.filePath = (filePath == null || "".equals(filePath)) ? FILE_PATH : filePath; this.splitter = (splitter < 1) ? SPLITTER_NUM : splitter; } /** * function: 通过url获得文件名称 * @author hoojo * @createDate 2011-9-30 下午05:00:00 * @param url * @return */ private String getFileName(String url) { return url.substring(url.lastIndexOf("/") + 1, url.length()); } public String getUrl() { return url; } public void setUrl(String url) { if (url == null || "".equals(url)) { throw new RuntimeException("url is not null!"); } this.url = url; } public String getFileName() { return fileName; } public void setFileName(String fileName) { this.fileName = (fileName == null || "".equals(fileName)) ? getFileName(url) : fileName; } public String getFilePath() { return filePath; } public void setFilePath(String filePath) { this.filePath = (filePath == null || "".equals(filePath)) ? FILE_PATH : filePath; } public int getSplitter() { return splitter; } public void setSplitter(int splitter) { this.splitter = (splitter < 1) ? SPLITTER_NUM : splitter; } @Override public String toString() { return this.url + "#" + this.fileName + "#" + this.filePath + "#" + this.splitter; } } 二、随机写入一段文件 package com.hoo.download; import java.io.IOException; import java.io.RandomAccessFile; /** * function: 写入文件、保存文件 * @author hoojo * @createDate 2011-9-21 下午05:44:02 * @file SaveItemFile.java * @package com.hoo.download * @project MultiThreadDownLoad * @blog http://blog.csdn.net/IBM_hoojo * @email hoojo_@126.com * @version 1.0 */ public class SaveItemFile { //存储文件 private RandomAccessFile itemFile; public SaveItemFile() throws IOException { this("", 0); } /** * @param name 文件路径、名称 * @param pos 写入点位置 position * @throws IOException */ public SaveItemFile(String name, long pos) throws IOException { itemFile = new RandomAccessFile(name, "rw"); //在指定的pos位置开始写入数据 itemFile.seek(pos); } /** * function: 同步方法写入文件 * @author hoojo * @createDate 2011-9-26 下午12:21:22 * @param buff 缓冲数组 * @param start 起始位置 * @param length 长度 * @return */ public synchronized int write(byte[] buff, int start, int length) { int i = -1; try { itemFile.write(buff, start, length); i = length; } catch (IOException e) { e.printStackTrace(); } return i; } public void close() throws IOException { if (itemFile != null) { itemFile.close(); } } } 这个类主要是完成向本地的指定文件指针出开始写入文件,并返回当前写入文件的长度(文件指针)。这个类将被线程调用,文件被分成对应的块后,将被线程调用。每个线程都将会调用这个类完成文件的随机写入。 三、单个线程下载文件 package com.hoo.download; import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; import java.net.URLConnection; import com.hoo.util.LogUtils; /** * function: 单线程下载文件 * @author hoojo * @createDate 2011-9-22 下午02:55:10 * @file DownloadFile.java * @package com.hoo.download * @project MultiThreadDownLoad * @blog http://blog.csdn.net/IBM_hoojo * @email hoojo_@126.com * @version 1.0 */ public class DownloadFile extends Thread { //下载文件url private String url; //下载文件起始位置 private long startPos; //下载文件结束位置 private long endPos; //线程id private int threadId; //下载是否完成 private boolean isDownloadOver = false; private SaveItemFile itemFile; private static final int BUFF_LENGTH = 1024 * 8; /** * @param url 下载文件url * @param name 文件名称 * @param startPos 下载文件起点 * @param endPos 下载文件结束点 * @param threadId 线程id * @throws IOException */ public DownloadFile(String url, String name, long startPos, long endPos, int threadId) throws IOException { super(); this.url = url; this.startPos = startPos; this.endPos = endPos; this.threadId = threadId; //分块下载写入文件内容 this.itemFile = new SaveItemFile(name, startPos); } @Override public void run() { while (endPos > startPos && !isDownloadOver) { try { URL url = new URL(this.url); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); // 设置连接超时时间为10000ms conn.setConnectTimeout(10000); // 设置读取数据超时时间为10000ms conn.setReadTimeout(10000); setHeader(conn); String property = "bytes=" + startPos + "-"; conn.setRequestProperty("RANGE", property); //输出log信息 LogUtils.log("开始 " + threadId + ":" + property + endPos); //printHeader(conn); //获取文件输入流,读取文件内容 InputStream is = conn.getInputStream(); byte[] buff = new byte[BUFF_LENGTH]; int length = -1; LogUtils.log("#start#Thread: " + threadId + ", startPos: " + startPos + ", endPos: " + endPos); while ((length = is.read(buff)) > 0 && startPos < endPos && !isDownloadOver) { //写入文件内容,返回最后写入的长度 startPos += itemFile.write(buff, 0, length); } LogUtils.log("#over#Thread: " + threadId + ", startPos: " + startPos + ", endPos: " + endPos); LogUtils.log("Thread " + threadId + " is execute over!"); this.isDownloadOver = true; } catch (MalformedURLException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } finally { try { if (itemFile != null) { itemFile.close(); } } catch (IOException e) { e.printStackTrace(); } } } if (endPos < startPos && !isDownloadOver) { LogUtils.log("Thread " + threadId + " startPos > endPos, not need download file !"); this.isDownloadOver = true; } if (endPos == startPos && !isDownloadOver) { LogUtils.log("Thread " + threadId + " startPos = endPos, not need download file !"); this.isDownloadOver = true; } } /** * function: 打印下载文件头部信息 * @author hoojo * @createDate 2011-9-22 下午05:44:35 * @param conn HttpURLConnection */ public static void printHeader(URLConnection conn) { int i = 1; while (true) { String header = conn.getHeaderFieldKey(i); i++; if (header != null) { LogUtils.info(header + ":" + conn.getHeaderField(i)); } else { break; } } } /** * function: 设置URLConnection的头部信息,伪装请求信息 * @author hoojo * @createDate 2011-9-28 下午05:29:43 * @param con */ public static void setHeader(URLConnection conn) { conn.setRequestProperty("User-Agent", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.3) Gecko/2008092510 Ubuntu/8.04 (hardy) Firefox/3.0.3"); conn.setRequestProperty("Accept-Language", "en-us,en;q=0.7,zh-cn;q=0.3"); conn.setRequestProperty("Accept-Encoding", "utf-8"); conn.setRequestProperty("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7"); conn.setRequestProperty("Keep-Alive", "300"); conn.setRequestProperty("connnection", "keep-alive"); conn.setRequestProperty("If-Modified-Since", "Fri, 02 Jan 2009 17:00:05 GMT"); conn.setRequestProperty("If-None-Match", "\"1261d8-4290-df64d224\""); conn.setRequestProperty("Cache-conntrol", "max-age=0"); conn.setRequestProperty("Referer", "http://www.baidu.com"); } public boolean isDownloadOver() { return isDownloadOver; } public long getStartPos() { return startPos; } public long getEndPos() { return endPos; } } 这个类主要是完成单个线程的文件下载,将通过URLConnection读取指定url的资源信息。然后用InputStream读取文件内容,然后调用调用SaveItemFile类,向本地写入当前要读取 的块的内容。 四、分段多线程写入文件内容 package com.hoo.download; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; import com.hoo.entity.DownloadInfo; import com.hoo.util.LogUtils; /** * function: 分批量下载文件 * @author hoojo * @createDate 2011-9-22 下午05:51:54 * @file BatchDownloadFile.java * @package com.hoo.download * @project MultiThreadDownLoad * @blog http://blog.csdn.net/IBM_hoojo * @email hoojo_@126.com * @version 1.0 */ public class BatchDownloadFile implements Runnable { //下载文件信息 private DownloadInfo downloadInfo; //一组开始下载位置 private long[] startPos; //一组结束下载位置 private long[] endPos; //休眠时间 private static final int SLEEP_SECONDS = 500; //子线程下载 private DownloadFile[] fileItem; //文件长度 private int length; //是否第一个文件 private boolean first = true; //是否停止下载 private boolean stop = false; //临时文件信息 private File tempFile; public BatchDownloadFile(DownloadInfo downloadInfo) { this.downloadInfo = downloadInfo; String tempPath = this.downloadInfo.getFilePath() + File.separator + downloadInfo.getFileName() + ".position"; tempFile = new File(tempPath); //如果存在读入点位置的文件 if (tempFile.exists()) { first = false; //就直接读取内容 try { readPosInfo(); } catch (IOException e) { e.printStackTrace(); } } else { //数组的长度就要分成多少段的数量 startPos = new long[downloadInfo.getSplitter()]; endPos = new long[downloadInfo.getSplitter()]; } } @Override public void run() { //首次下载,获取下载文件长度 if (first) { length = this.getFileSize();//获取文件长度 if (length == -1) { LogUtils.log("file length is know!"); stop = true; } else if (length == -2) { LogUtils.log("read file length is error!"); stop = true; } else if (length > 0) { /** * eg * start: 1, 3, 5, 7, 9 * end: 3, 5, 7, 9, length */ for (int i = 0, len = startPos.length; i < len; i++) { int size = i * (length / len); startPos[i] = size; //设置最后一个结束点的位置 if (i == len - 1) { endPos[i] = length; } else { size = (i + 1) * (length / len); endPos[i] = size; } LogUtils.log("start-end Position[" + i + "]: " + startPos[i] + "-" + endPos[i]); } } else { LogUtils.log("get file length is error, download is stop!"); stop = true; } } //子线程开始下载 if (!stop) { //创建单线程下载对象数组 fileItem = new DownloadFile[startPos.length];//startPos.length = downloadInfo.getSplitter() for (int i = 0; i < startPos.length; i++) { try { //创建指定个数单线程下载对象,每个线程独立完成指定块内容的下载 fileItem[i] = new DownloadFile( downloadInfo.getUrl(), this.downloadInfo.getFilePath() + File.separator + downloadInfo.getFileName(), startPos[i], endPos[i], i ); fileItem[i].start();//启动线程,开始下载 LogUtils.log("Thread: " + i + ", startPos: " + startPos[i] + ", endPos: " + endPos[i]); } catch (IOException e) { e.printStackTrace(); } } //循环写入下载文件长度信息 while (!stop) { try { writePosInfo(); LogUtils.log("downloading……"); Thread.sleep(SLEEP_SECONDS); stop = true; } catch (IOException e) { e.printStackTrace(); } catch (InterruptedException e) { e.printStackTrace(); } for (int i = 0; i < startPos.length; i++) { if (!fileItem[i].isDownloadOver()) { stop = false; break; } } } LogUtils.info("Download task is finished!"); } } /** * 将写入点数据保存在临时文件中 * @author hoojo * @createDate 2011-9-23 下午05:25:37 * @throws IOException */ private void writePosInfo() throws IOException { DataOutputStream dos = new DataOutputStream(new FileOutputStream(tempFile)); dos.writeInt(startPos.length); for (int i = 0; i < startPos.length; i++) { dos.writeLong(fileItem[i].getStartPos()); dos.writeLong(fileItem[i].getEndPos()); //LogUtils.info("[" + fileItem[i].getStartPos() + "#" + fileItem[i].getEndPos() + "]"); } dos.close(); } /** * function:读取写入点的位置信息 * @author hoojo * @createDate 2011-9-23 下午05:30:29 * @throws IOException */ private void readPosInfo() throws IOException { DataInputStream dis = new DataInputStream(new FileInputStream(tempFile)); int startPosLength = dis.readInt(); startPos = new long[startPosLength]; endPos = new long[startPosLength]; for (int i = 0; i < startPosLength; i++) { startPos[i] = dis.readLong(); endPos[i] = dis.readLong(); } dis.close(); } /** * function: 获取下载文件的长度 * @author hoojo * @createDate 2011-9-26 下午12:15:08 * @return */ private int getFileSize() { int fileLength = -1; try { URL url = new URL(this.downloadInfo.getUrl()); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); DownloadFile.setHeader(conn); int stateCode = conn.getResponseCode(); //判断http status是否为HTTP/1.1 206 Partial Content或者200 OK if (stateCode != HttpURLConnection.HTTP_OK && stateCode != HttpURLConnection.HTTP_PARTIAL) { LogUtils.log("Error Code: " + stateCode); return -2; } else if (stateCode >= 400) { LogUtils.log("Error Code: " + stateCode); return -2; } else { //获取长度 fileLength = conn.getContentLength(); LogUtils.log("FileLength: " + fileLength); } //读取文件长度 /*for (int i = 1; ; i++) { String header = conn.getHeaderFieldKey(i); if (header != null) { if ("Content-Length".equals(header)) { fileLength = Integer.parseInt(conn.getHeaderField(i)); break; } } else { break; } } */ DownloadFile.printHeader(conn); } catch (MalformedURLException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } return fileLength; } } 这个类主要是完成读取指定url资源的内容,获取该资源的长度。然后将该资源分成指定的块数,将每块的起始下载位置、结束下载位置,分别保存在一个数组中。每块都单独开辟一个独立线程开始下载。在开始下载之前,需要创建一个临时文件,写入当前下载线程的开始下载指针位置和结束下载指针位置。 五、工具类、测试类 日志工具类 package com.hoo.util; /** * function: 日志工具类 * @author hoojo * @createDate 2011-9-21 下午05:21:27 * @file LogUtils.java * @package com.hoo.util * @project MultiThreadDownLoad * @blog http://blog.csdn.net/IBM_hoojo * @email hoojo_@126.com * @version 1.0 */ public abstract class LogUtils { public static void log(Object message) { System.err.println(message); } public static void log(String message) { System.err.println(message); } public static void log(int message) { System.err.println(message); } public static void info(Object message) { System.out.println(message); } public static void info(String message) { System.out.println(message); } public static void info(int message) { System.out.println(message); } } 下载工具类 package com.hoo.util; import com.hoo.download.BatchDownloadFile; import com.hoo.entity.DownloadInfo; /** * function: 分块多线程下载工具类 * @author hoojo * @createDate 2011-9-28 下午05:22:18 * @file DownloadUtils.java * @package com.hoo.util * @project MultiThreadDownLoad * @blog http://blog.csdn.net/IBM_hoojo * @email hoojo_@126.com * @version 1.0 */ public abstract class DownloadUtils { public static void download(String url) { DownloadInfo bean = new DownloadInfo(url); LogUtils.info(bean); BatchDownloadFile down = new BatchDownloadFile(bean); new Thread(down).start(); } public static void download(String url, int threadNum) { DownloadInfo bean = new DownloadInfo(url, threadNum); LogUtils.info(bean); BatchDownloadFile down = new BatchDownloadFile(bean); new Thread(down).start(); } public static void download(String url, String fileName, String filePath, int threadNum) { DownloadInfo bean = new DownloadInfo(url, fileName, filePath, threadNum); LogUtils.info(bean); BatchDownloadFile down = new BatchDownloadFile(bean); new Thread(down).start(); } } 下载测试类 package com.hoo.test; import com.hoo.util.DownloadUtils; /** * function: 下载测试 * @author hoojo * @createDate 2011-9-23 下午05:49:46 * @file TestDownloadMain.java * @package com.hoo.download * @project MultiThreadDownLoad * @blog http://blog.csdn.net/IBM_hoojo * @email hoojo_@126.com * @version 1.0 */ public class TestDownloadMain { public static void main(String[] args) { /*DownloadInfo bean = new DownloadInfo("http://i7.meishichina.com/Health/UploadFiles/201109/2011092116224363.jpg"); System.out.println(bean); BatchDownloadFile down = new BatchDownloadFile(bean); new Thread(down).start();*/ //DownloadUtils.download("http://i7.meishichina.com/Health/UploadFiles/201109/2011092116224363.jpg"); DownloadUtils.download("http://mp3.baidu.com/j?j=2&url=http%3A%2F%2Fzhangmenshiting2.baidu.com%2Fdata%2Fmusic%2F1669425%2F%25E9%2599%25B7%25E5%2585%25A5%25E7%2588%25B1%25E9%2587%258C%25E9%259D%25A2.mp3%3Fxcode%3D2ff36fb70737c816553396c56deab3f1", "aa.mp3", "c:/temp", 5); } } 多线程下载主要在第三部和第四部,其他的地方还是很好理解。源码中提供相应的注释了,便于理解。

下载文档,方便阅读与编辑

文档的实际排版效果,会与网站的显示效果略有不同!!

需要 5 金币 [ 分享文档获得金币 ]
3 人已下载

下载文档

相关文档