java實現大文件下載(http方式)


java實現大文件下載,基於http方式,控件神馬的就不說了。
思路:下載文件無非要讀取文件然后寫文件,主要這兩個步驟,主要難點:
1.讀文件,就是硬盤到內存的過程,由於jdk內存限制,不能讀的太大。
2.寫文件,就是響應到瀏覽器端的過程,http協議是短鏈接,如果寫文件太慢,時間過久,會造成瀏覽器死掉。

知識點:

1.org.apache.http.impl.client.CloseableHttpClient  模擬httpClient客戶端發送http請求,可以控制到請求文件的字節位置。
2.BufferedInputStream都熟悉,用它接受請求來的流信息緩存。
3.RandomAccessFile文件隨機類,可以向文件寫入指定位置的流信息。
  • 1
  • 2
  • 3

基於以上信息,我的實現思路就是首先判斷下載文件大小,配合多線程分割定制http請求數量和請求內容,響應到寫入到RandomAccessFile指定位置中。在俗點就是大的http分割成一個個小的http請求,相當於每次請求一個網頁。

廢話不說,上代碼。

DownLoadManagerTest類:

package xxxx; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.net.HttpURLConnection; import java.net.URL; import java.util.concurrent.CountDownLatch; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.task.TaskExecutor; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; /** * * 文件下載管理類 */ @RunWith(SpringJUnit4ClassRunner.class) @ActiveProfiles("test") @ContextConfiguration(locations={"classpath:test/applicationContext.xml"}) public class DownLoadManagerTest extends AbstractTransactionalJUnit4SpringContextTests{ private static final Logger LOGGER = LoggerFactory.getLogger(DownLoadManagerTest.class); /** * * 每個線程下載的字節數 */ private long unitSize = 1000 * 1024; @Autowired private TaskExecutor taskExecutor; private CloseableHttpClient httpClient; private Long starttimes; private Long endtimes; @Before public void setUp() throws Exception{ starttimes = System.currentTimeMillis(); System.out.println("測試開始...."); } @After public void tearDown() throws Exception{ endtimes = System.currentTimeMillis(); System.out.println("測試結束!!"); System.out.println("********************"); System.out.println("下載總耗時:"+(endtimes-starttimes)/1000+"s"); System.out.println("********************"); } public DownLoadManagerTest() { System.out.println("初始化測試類...."); PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); cm.setMaxTotal(100); httpClient = HttpClients.custom().setConnectionManager(cm).build(); } /** * * 啟動多個線程下載文件 */ @Test public void doDownload() throws IOException { String remoteFileUrl="http://{host}:{port}/{project}/xx.xml"; String localPath="E://test//"; String fileName = new URL(remoteFileUrl).getFile(); System.out.println("遠程文件名稱:"+fileName); fileName = fileName.substring(fileName.lastIndexOf("/") + 1, fileName.length()).replace("%20", " "); System.out.println("本地文件名稱:"+fileName); long fileSize = this.getRemoteFileSize(remoteFileUrl); this.createFile(localPath+System.currentTimeMillis()+fileName, fileSize); Long threadCount = (fileSize/unitSize)+(fileSize % unitSize!=0?1:0); long offset = 0; CountDownLatch end = new CountDownLatch(threadCount.intValue()); // 如果遠程文件尺寸小於等於unitSize if (fileSize <= unitSize) { DownloadThreadTest downloadThread = new DownloadThreadTest(remoteFileUrl, localPath+fileName, offset, fileSize,end,httpClient); taskExecutor.execute(downloadThread); } else { // 如果遠程文件尺寸大於unitSize for (int i = 1; i < threadCount; i++) { DownloadThreadTest downloadThread = new DownloadThreadTest( remoteFileUrl, localPath+fileName, offset, unitSize,end,httpClient); taskExecutor.execute(downloadThread); offset = offset + unitSize; } // 如果不能整除,則需要再創建一個線程下載剩余字節 if (fileSize % unitSize != 0) { DownloadThreadTest downloadThread = new DownloadThreadTest(remoteFileUrl, localPath+fileName, offset, fileSize - unitSize * (threadCount-1),end,httpClient); taskExecutor.execute(downloadThread); } } try { end.await(); } catch (InterruptedException e) { LOGGER.error("DownLoadManager exception msg:{}",ExceptionUtils.getFullStackTrace(e)); e.printStackTrace(); } // System.out.println("111111"); LOGGER.debug("下載完成!{} ",localPath+fileName); //return localPath+fileName; } /** * * 獲取遠程文件尺寸 */ private long getRemoteFileSize(String remoteFileUrl) throws IOException { long fileSize = 0; HttpURLConnection httpConnection = (HttpURLConnection) new URL( remoteFileUrl).openConnection(); httpConnection.setRequestMethod("HEAD"); int responseCode = httpConnection.getResponseCode(); if (responseCode >= 400) { LOGGER.debug("Web服務器響應錯誤!"); return 0; } String sHeader; for (int i = 1;; i++) { sHeader = httpConnection.getHeaderFieldKey(i); if (sHeader != null && sHeader.equals("Content-Length")) { System.out.println("文件大小ContentLength:" + httpConnection.getContentLength()); fileSize = Long.parseLong(httpConnection .getHeaderField(sHeader)); break; } } return fileSize; } /** * * 創建指定大小的文件 */ private void createFile(String fileName, long fileSize) throws IOException { File newFile = new File(fileName); RandomAccessFile raf = new RandomAccessFile(newFile, "rw"); raf.setLength(fileSize); raf.close(); } public TaskExecutor getTaskExecutor() { return taskExecutor; } public void setTaskExecutor(TaskExecutor taskExecutor) { this.taskExecutor = taskExecutor; } } 
  • 1
  • 2
  • 3
  • 4
  • 5
  • 6
  • 7
  • 8
  • 9
  • 10
  • 11
  • 12
  • 13
  • 14
  • 15
  • 16
  • 17
  • 18
  • 19
  • 20
  • 21
  • 22
  • 23
  • 24
  • 25
  • 26
  • 27
  • 28
  • 29
  • 30
  • 31
  • 32
  • 33
  • 34
  • 35
  • 36
  • 37
  • 38
  • 39
  • 40
  • 41
  • 42
  • 43
  • 44
  • 45
  • 46
  • 47
  • 48
  • 49
  • 50
  • 51
  • 52
  • 53
  • 54
  • 55
  • 56
  • 57
  • 58
  • 59
  • 60
  • 61
  • 62
  • 63
  • 64
  • 65
  • 66
  • 67
  • 68
  • 69
  • 70
  • 71
  • 72
  • 73
  • 74
  • 75
  • 76
  • 77
  • 78
  • 79
  • 80
  • 81
  • 82
  • 83
  • 84
  • 85
  • 86
  • 87
  • 88
  • 89
  • 90
  • 91
  • 92
  • 93
  • 94
  • 95
  • 96
  • 97
  • 98
  • 99
  • 100
  • 101
  • 102
  • 103
  • 104
  • 105
  • 106
  • 107
  • 108
  • 109
  • 110
  • 111
  • 112
  • 113
  • 114
  • 115
  • 116
  • 117
  • 118
  • 119
  • 120
  • 121
  • 122
  • 123
  • 124
  • 125
  • 126
  • 127
  • 128
  • 129
  • 130
  • 131
  • 132
  • 133
  • 134
  • 135
  • 136
  • 137
  • 138
  • 139
  • 140
  • 141
  • 142
  • 143
  • 144
  • 145
  • 146
  • 147
  • 148
  • 149
  • 150
  • 151
  • 152
  • 153
  • 154
  • 155
  • 156
  • 157
  • 158
  • 159
  • 160
  • 161
  • 162
  • 163
  • 164
  • 165
  • 166
  • 167
  • 168
  • 169
  • 170
  • 171
  • 172
  • 173
  • 174
  • 175
  • 176
  • 177
  • 178
  • 179
  • 180

DownloadThreadTest類:

package xxxx; import java.io.BufferedInputStream; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.util.concurrent.CountDownLatch; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.protocol.BasicHttpContext; import org.apache.http.protocol.HttpContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * * 負責文件下載的類 */ public class DownloadThreadTest extends Thread { private static final Logger LOGGER = LoggerFactory .getLogger(DownloadThreadTest.class); /** * * 待下載的文件 */ private String url = null; /** * * 本地文件名 */ private String fileName = null; /** * * 偏移量 */ private long offset = 0; /** * * 分配給本線程的下載字節數 */ private long length = 0; private CountDownLatch end; private CloseableHttpClient httpClient; private HttpContext context; /** * * @param url 下載文件地址 * * @param fileName 另存文件名 * * @param offset 本線程下載偏移量 * * @param length 本線程下載長度 * * @author Angus.wang * * */ public DownloadThreadTest(String url, String file, long offset, long length, CountDownLatch end, CloseableHttpClient httpClient) { this.url = url; this.fileName = file; this.offset = offset; this.length = length; this.end = end; this.httpClient = httpClient; this.context = new BasicHttpContext(); LOGGER.debug("偏移量=" + offset + ";字節數=" + length); } public void run() { try { HttpGet httpGet = new HttpGet(this.url); httpGet.addHeader("Range", "bytes=" + this.offset + "-" + (this.offset + this.length - 1)); CloseableHttpResponse response = httpClient.execute(httpGet, context); BufferedInputStream bis = new BufferedInputStream(response .getEntity().getContent()); byte[] buff = new byte[1024]; int bytesRead; File newFile = new File(fileName); RandomAccessFile raf = new RandomAccessFile(newFile, "rw"); while ((bytesRead = bis.read(buff, 0, buff.length)) != -1) { raf.seek(this.offset); raf.write(buff, 0, bytesRead); this.offset = this.offset + bytesRead; } raf.close(); bis.close(); } catch (ClientProtocolException e) { LOGGER.error("DownloadThread exception msg:{}",ExceptionUtils.getFullStackTrace(e)); } catch (IOException e) { LOGGER.error("DownloadThread exception msg:{}",ExceptionUtils.getFullStackTrace(e)); } finally { end.countDown(); LOGGER.info(end.getCount() + " is go on!"); System.out.println(end.getCount() + " is go on!"); } } } 
  • 1
  • 2
  • 3
  • 4
  • 5
  • 6
  • 7
  • 8
  • 9
  • 10
  • 11
  • 12
  • 13
  • 14
  • 15
  • 16
  • 17
  • 18
  • 19
  • 20
  • 21
  • 22
  • 23
  • 24
  • 25
  • 26
  • 27
  • 28
  • 29
  • 30
  • 31
  • 32
  • 33
  • 34
  • 35
  • 36
  • 37
  • 38
  • 39
  • 40
  • 41
  • 42
  • 43
  • 44
  • 45
  • 46
  • 47
  • 48
  • 49
  • 50
  • 51
  • 52
  • 53
  • 54
  • 55
  • 56
  • 57
  • 58
  • 59
  • 60
  • 61
  • 62
  • 63
  • 64
  • 65
  • 66
  • 67
  • 68
  • 69
  • 70
  • 71
  • 72
  • 73
  • 74
  • 75
  • 76
  • 77
  • 78
  • 79
  • 80
  • 81
  • 82
  • 83
  • 84
  • 85
  • 86
  • 87
  • 88
  • 89
  • 90
  • 91
  • 92
  • 93
  • 94
  • 95
  • 96
  • 97
  • 98
  • 99
  • 100
  • 101
  • 102
  • 103
  • 104
  • 105
  • 106
  • 107
  • 108
  • 109

application.xml

<bean id="taskExecutor" class="org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor"> <!-- 線程池活躍的線程數 --> <property name="corePoolSize" value="5" /> <!-- 線程池最大活躍的線程數 --> <property name="maxPoolSize" value="10" /> <!-- 隊列的最大容量 --> <property name="queueCapacity" value="600" /> </bean> <bean id="downLoadManager" class="xx.DownLoadManagerTest"> <property name="taskExecutor" ref="taskExecutor" /> </bean> 
  • 1
  • 2
  • 3
  • 4
  • 5
  • 6
  • 7
  • 8
  • 9
  • 10
  • 11
  • 12

測試運行,500M,我這網速得半個小時左右。要想下載更大的文件,只要jdk內存夠大,就無限更改隊列最大容量吧。

如果不同意見,歡迎各位大神指正。

轉載:https://my.oschina.net/zmf/blog/336961


免責聲明!

本站轉載的文章為個人學習借鑒使用,本站對版權不負任何法律責任。如果侵犯了您的隱私權益,請聯系本站郵箱yoyou2525@163.com刪除。



 
粵ICP備18138465號   © 2018-2025 CODEPRJ.COM