測試代碼
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
|
package
com.boot.demo.test.io;
import
java.io.*;
import
java.lang.reflect.Method;
import
java.nio.MappedByteBuffer;
import
java.nio.channels.FileChannel;
import
java.nio.file.Files;
import
java.nio.file.Paths;
import
java.nio.file.StandardOpenOption;
import
java.security.AccessController;
import
java.security.PrivilegedAction;
/**
* @author braska
* @date 2020/3/19
**/
public
class
FileTest {
public
static
void
fileStream(String sourceFile, String targetFile) {
File file =
new
File(targetFile);
try
(FileInputStream fis =
new
FileInputStream(sourceFile);
FileOutputStream fos =
new
FileOutputStream(file)) {
byte
[] bytes =
new
byte
[
1024
*
1024
];
int
len;
while
((len = fis.read(bytes)) >
0
) {
fos.write(bytes,
0
, len);
}
}
catch
(Exception e) {
}
}
public
static
void
bufferStream(String sourceFile, String targetFile) {
try
(BufferedInputStream bis =
new
BufferedInputStream(Files.newInputStream(Paths.get(sourceFile)));
BufferedOutputStream bos =
new
BufferedOutputStream(Files.newOutputStream(Paths.get(targetFile),
StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.WRITE))) {
byte
[] bytes =
new
byte
[
1024
*
1024
];
int
len;
while
((len = bis.read(bytes)) >
0
) {
bos.write(bytes,
0
, len);
}
}
catch
(Exception e) {
e.printStackTrace();
}
}
public
static
void
randomFile(String sourceFile, String targetFile) {
try
(RandomAccessFile read =
new
RandomAccessFile(sourceFile,
"r"
);
RandomAccessFile write =
new
RandomAccessFile(targetFile,
"rw"
)) {
byte
[] bytes =
new
byte
[
1024
*
1024
];
int
len;
while
((len = read.read(bytes)) >
0
) {
write.write(bytes,
0
, len);
}
}
catch
(Exception e) {
}
}
public
static
void
memoryMap(String sourceFile, String targetFile) {
try
(FileChannel rc = FileChannel.open(Paths.get(sourceFile));
FileChannel wc = FileChannel.open(Paths.get(targetFile),
StandardOpenOption.CREATE,
StandardOpenOption.READ,
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.WRITE)) {
long
copy = 1L <<
30
;
long
cur =
0
;
long
fileLength = rc.size();
while
(cur < fileLength) {
copy = cur + copy > fileLength ? (fileLength - cur) : copy;
MappedByteBuffer rMap = rc.map(FileChannel.MapMode.READ_ONLY, cur, copy);
MappedByteBuffer wMap = wc.map(FileChannel.MapMode.READ_WRITE, cur, copy);
for
(
int
i =
0
; i < copy; i++) {
byte
b = rMap.get(i);
//從源文件讀取字節
wMap.put(i, b);
//把字節寫到目標文件中
}
System.gc();
//手動調用 GC <必須的，否則出現異常>
System.runFinalization();
cur += copy;
}
}
catch
(Exception e) {
e.printStackTrace();
}
}
private
static
String buildFilePath(String path, String fileName, String extension) {
return
String.format(
"%s%s.%s"
, path, fileName, extension);
}
public
static
void
main(String[] args) {
/* String path = "F:\\workspace\\demo\\";
String extension = "hprof";
// 30M文件
String sourceFile = buildFilePath(path, "01", extension);*/
/* String path = "E:\\software\\";
String extension = "exe";
// 460M文件
String sourceFile = buildFilePath(path, "Anaconda3-2019.10-Windows-x86_64", extension);*/
String path =
"E:\\software\\"
;
String extension =
"zip"
;
// 1.47G文件
String sourceFile = buildFilePath(path,
"software"
, extension);
String targetFile;
long
start;
/* targetFile = buildFilePath(path, "target_file_stream", extension);
start = System.currentTimeMillis();
FileTest.fileStream(sourceFile, targetFile);
System.out.println("file stream used time:" + (System.currentTimeMillis() - start));*/
/* targetFile = buildFilePath(path, "target_buffer_stream", extension);
start = System.currentTimeMillis();
FileTest.bufferStream(sourceFile, targetFile);
System.out.println("buffer stream used time:" + (System.currentTimeMillis() - start));*/
/* targetFile = buildFilePath(path, "target_random_file", extension);
start = System.currentTimeMillis();
FileTest.randomFile(sourceFile, targetFile);
System.out.println("random file used time:" + (System.currentTimeMillis() - start));*/
targetFile = buildFilePath(path,
"target_memory_map"
, extension);
start = System.currentTimeMillis();
FileTest.memoryMap(sourceFile, targetFile);
System.out.println(
"memory map used time:"
+ (System.currentTimeMillis() - start));
}
}
|
測試結果
| 文件大小 | 讀寫方式 | 耗時 |
|---|---|---|
| 30M | 普通文件流 | 50-60 ms |
| 緩存流 | 32-35 ms | |
| 隨機文件方式 | 40-50 ms | |
| 內存映射文件 | 50-60 ms | |
| 461M | 普通文件流 | 1300-2300 ms |
| 緩存流 | 1700-2000 ms | |
| 隨機文件方式 | 1300-3000 ms | |
| 內存映射文件 | 890-1000 ms | |
| 1.47G | 普通文件流 | 11s |
| 緩存流 | 9s | |
| 隨機文件方式 | 10s | |
| 內存映射文件 | 3s(首次較慢) |
結尾：測試1.47G大文件時，內存映射文件中copy大小做了調整，當copy為1G時(copy=1L<<30)性能最佳，整過過程1-3秒左右。調至128M、512M。大約耗時都在15秒左右。為了公平起見，把其他方法中的byte緩沖區大小也做了同樣調整，耗時變化不大。有些甚至更慢。
封裝MappedBuffer工具類，代碼如下：
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
|
package
com.boot.demo.test.io;
import
java.io.Closeable;
import
java.io.IOException;
import
java.nio.MappedByteBuffer;
import
java.nio.channels.FileChannel;
import
java.nio.file.Paths;
import
java.nio.file.StandardOpenOption;
import
java.util.concurrent.ArrayBlockingQueue;
import
java.util.concurrent.BlockingQueue;
/**
* @author braska
* @date 2020/3/20
**/
public
class
MappedByteBufferReader
implements
Closeable {
private
static
final
int
BUFFERED_SIZE =
1
<<
27
;
private
static
final
FileChannel.MapMode MAP_MODE = FileChannel.MapMode.READ_ONLY;
private
FileChannel fileChannel;
private
final
long
fileSize;
private
final
int
bufferedSize;
BlockingQueue<MappedByteBuffer> queue;
public
MappedByteBufferReader(String file)
throws
Exception {
this
(file, BUFFERED_SIZE, MAP_MODE);
}
public
MappedByteBufferReader(String file,
int
bufferedSize)
throws
Exception {
this
(file, bufferedSize, MAP_MODE);
}
public
MappedByteBufferReader(String file, FileChannel.MapMode mapMode)
throws
Exception {
this
(file, BUFFERED_SIZE, mapMode);
}
public
MappedByteBufferReader(String file,
int
bufferedSize, FileChannel.MapMode mapMode)
throws
Exception {
this
.fileChannel = FileChannel.open(Paths.get(file));
this
.fileSize = fileChannel.size();
this
.bufferedSize = bufferedSize;
int
capacity = (
int
) Math.ceil((
double
) fileSize / (
double
) bufferedSize);
this
.queue =
new
ArrayBlockingQueue(capacity);
long
readSize = bufferedSize;
long
cursor = 0l;
while
(cursor < fileSize) {
readSize = cursor + readSize > fileSize ? fileSize - cursor : readSize;
queue.add(
fileChannel.map(mapMode, cursor, readSize)
);
cursor += readSize;
}
}
public
byte
[] read() {
byte
[] bytes;
MappedByteBuffer byteBuffer = queue.poll();
if
(byteBuffer !=
null
) {
int
limit = byteBuffer.limit();
int
position = byteBuffer.position();
int
realSize =
this
.bufferedSize;
if
(limit - position <
this
.bufferedSize) {
realSize = limit - position;
}
bytes =
new
byte
[realSize];
byteBuffer.get(bytes);
byteBuffer.clear();
return
bytes;
}
return
null
;
}
public
long
size() {
return
this
.fileSize;
}
@Override
public
void
close()
throws
IOException {
if
(
this
.fileChannel !=
null
) {
this
.fileChannel.close();
}
}
public
static
void
main(String[] args) {
long
start = System.currentTimeMillis();
try
(MappedByteBufferReader reader =
new
MappedByteBufferReader(
"E:\\software\\software.zip"
,
1
<<
30
);
FileChannel wc = FileChannel.open(Paths.get(
"E:\\software\\software_reader.zip"
),
StandardOpenOption.CREATE,
StandardOpenOption.READ,
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.WRITE)) {
byte
[] data;
MappedByteBuffer writer = wc.map(FileChannel.MapMode.READ_WRITE,
0
, reader.size());
while
((data = reader.read()) !=
null
) {
writer.put(data);
}
writer.clear();
System.out.println(
"used times: "
+ (System.currentTimeMillis() - start));
}
catch
(Exception e) {
e.printStackTrace();
}
}
}
|