测试代码
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
|
package
com.boot.demo.test.io;
import
java.io.*;
import
java.lang.reflect.Method;
import
java.nio.MappedByteBuffer;
import
java.nio.channels.FileChannel;
import
java.nio.file.Files;
import
java.nio.file.Paths;
import
java.nio.file.StandardOpenOption;
import
java.security.AccessController;
import
java.security.PrivilegedAction;
/**
* @author braska
* @date 2020/3/19
**/
public
class
FileTest {
public
static
void
fileStream(String sourceFile, String targetFile) {
File file =
new
File(targetFile);
try
(FileInputStream fis =
new
FileInputStream(sourceFile);
FileOutputStream fos =
new
FileOutputStream(file)) {
byte
[] bytes =
new
byte
[
1024
*
1024
];
int
len;
while
((len = fis.read(bytes)) >
0
) {
fos.write(bytes,
0
, len);
}
}
catch
(Exception e) {
}
}
public
static
void
bufferStream(String sourceFile, String targetFile) {
try
(BufferedInputStream bis =
new
BufferedInputStream(Files.newInputStream(Paths.get(sourceFile)));
BufferedOutputStream bos =
new
BufferedOutputStream(Files.newOutputStream(Paths.get(targetFile),
StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.WRITE))) {
byte
[] bytes =
new
byte
[
1024
*
1024
];
int
len;
while
((len = bis.read(bytes)) >
0
) {
bos.write(bytes,
0
, len);
}
}
catch
(Exception e) {
e.printStackTrace();
}
}
public
static
void
randomFile(String sourceFile, String targetFile) {
try
(RandomAccessFile read =
new
RandomAccessFile(sourceFile,
"r"
);
RandomAccessFile write =
new
RandomAccessFile(targetFile,
"rw"
)) {
byte
[] bytes =
new
byte
[
1024
*
1024
];
int
len;
while
((len = read.read(bytes)) >
0
) {
write.write(bytes,
0
, len);
}
}
catch
(Exception e) {
}
}
public
static
void
memoryMap(String sourceFile, String targetFile) {
try
(FileChannel rc = FileChannel.open(Paths.get(sourceFile));
FileChannel wc = FileChannel.open(Paths.get(targetFile),
StandardOpenOption.CREATE,
StandardOpenOption.READ,
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.WRITE)) {
long
copy = 1L <<
30
;
long
cur =
0
;
long
fileLength = rc.size();
while
(cur < fileLength) {
copy = cur + copy > fileLength ? (fileLength - cur) : copy;
MappedByteBuffer rMap = rc.map(FileChannel.MapMode.READ_ONLY, cur, copy);
MappedByteBuffer wMap = wc.map(FileChannel.MapMode.READ_WRITE, cur, copy);
for
(
int
i =
0
; i < copy; i++) {
byte
b = rMap.get(i);
//从源文件读取字节
wMap.put(i, b);
//把字节写到目标文件中
}
System.gc();
//手动调用 GC <必须的，否则出现异常>
System.runFinalization();
cur += copy;
}
}
catch
(Exception e) {
e.printStackTrace();
}
}
private
static
String buildFilePath(String path, String fileName, String extension) {
return
String.format(
"%s%s.%s"
, path, fileName, extension);
}
public
static
void
main(String[] args) {
/* String path = "F:\\workspace\\demo\\";
String extension = "hprof";
// 30M文件
String sourceFile = buildFilePath(path, "01", extension);*/
/* String path = "E:\\software\\";
String extension = "exe";
// 460M文件
String sourceFile = buildFilePath(path, "Anaconda3-2019.10-Windows-x86_64", extension);*/
String path =
"E:\\software\\"
;
String extension =
"zip"
;
// 1.47G文件
String sourceFile = buildFilePath(path,
"software"
, extension);
String targetFile;
long
start;
/* targetFile = buildFilePath(path, "target_file_stream", extension);
start = System.currentTimeMillis();
FileTest.fileStream(sourceFile, targetFile);
System.out.println("file stream used time:" + (System.currentTimeMillis() - start));*/
/* targetFile = buildFilePath(path, "target_buffer_stream", extension);
start = System.currentTimeMillis();
FileTest.bufferStream(sourceFile, targetFile);
System.out.println("buffer stream used time:" + (System.currentTimeMillis() - start));*/
/* targetFile = buildFilePath(path, "target_random_file", extension);
start = System.currentTimeMillis();
FileTest.randomFile(sourceFile, targetFile);
System.out.println("random file used time:" + (System.currentTimeMillis() - start));*/
targetFile = buildFilePath(path,
"target_memory_map"
, extension);
start = System.currentTimeMillis();
FileTest.memoryMap(sourceFile, targetFile);
System.out.println(
"memory map used time:"
+ (System.currentTimeMillis() - start));
}
}
|
测试结果
| 文件大小 | 读写方式 | 耗时 |
|---|---|---|
| 30M | 普通文件流 | 50-60 ms |
| 缓存流 | 32-35 ms | |
| 随机文件方式 | 40-50 ms | |
| 内存映射文件 | 50-60 ms | |
| 461M | 普通文件流 | 1300-2300 ms |
| 缓存流 | 1700-2000 ms | |
| 随机文件方式 | 1300-3000 ms | |
| 内存映射文件 | 890-1000 ms | |
| 1.47G | 普通文件流 | 11s |
| 缓存流 | 9s | |
| 随机文件方式 | 10s | |
| 内存映射文件 | 3s(首次较慢) |
结尾：测试1.47G大文件时，内存映射文件中copy大小做了调整，当copy为1G时(copy=1L<<30)性能最佳，整过过程1-3秒左右。调至128M、512M。大约耗时都在15秒左右。为了公平起见，把其他方法中的byte缓冲区大小也做了同样调整，耗时变化不大。有些甚至更慢。
封装MappedBuffer工具类，代码如下：
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
|
package
com.boot.demo.test.io;
import
java.io.Closeable;
import
java.io.IOException;
import
java.nio.MappedByteBuffer;
import
java.nio.channels.FileChannel;
import
java.nio.file.Paths;
import
java.nio.file.StandardOpenOption;
import
java.util.concurrent.ArrayBlockingQueue;
import
java.util.concurrent.BlockingQueue;
/**
* @author braska
* @date 2020/3/20
**/
public
class
MappedByteBufferReader
implements
Closeable {
private
static
final
int
BUFFERED_SIZE =
1
<<
27
;
private
static
final
FileChannel.MapMode MAP_MODE = FileChannel.MapMode.READ_ONLY;
private
FileChannel fileChannel;
private
final
long
fileSize;
private
final
int
bufferedSize;
BlockingQueue<MappedByteBuffer> queue;
public
MappedByteBufferReader(String file)
throws
Exception {
this
(file, BUFFERED_SIZE, MAP_MODE);
}
public
MappedByteBufferReader(String file,
int
bufferedSize)
throws
Exception {
this
(file, bufferedSize, MAP_MODE);
}
public
MappedByteBufferReader(String file, FileChannel.MapMode mapMode)
throws
Exception {
this
(file, BUFFERED_SIZE, mapMode);
}
public
MappedByteBufferReader(String file,
int
bufferedSize, FileChannel.MapMode mapMode)
throws
Exception {
this
.fileChannel = FileChannel.open(Paths.get(file));
this
.fileSize = fileChannel.size();
this
.bufferedSize = bufferedSize;
int
capacity = (
int
) Math.ceil((
double
) fileSize / (
double
) bufferedSize);
this
.queue =
new
ArrayBlockingQueue(capacity);
long
readSize = bufferedSize;
long
cursor = 0l;
while
(cursor < fileSize) {
readSize = cursor + readSize > fileSize ? fileSize - cursor : readSize;
queue.add(
fileChannel.map(mapMode, cursor, readSize)
);
cursor += readSize;
}
}
public
byte
[] read() {
byte
[] bytes;
MappedByteBuffer byteBuffer = queue.poll();
if
(byteBuffer !=
null
) {
int
limit = byteBuffer.limit();
int
position = byteBuffer.position();
int
realSize =
this
.bufferedSize;
if
(limit - position <
this
.bufferedSize) {
realSize = limit - position;
}
bytes =
new
byte
[realSize];
byteBuffer.get(bytes);
byteBuffer.clear();
return
bytes;
}
return
null
;
}
public
long
size() {
return
this
.fileSize;
}
@Override
public
void
close()
throws
IOException {
if
(
this
.fileChannel !=
null
) {
this
.fileChannel.close();
}
}
public
static
void
main(String[] args) {
long
start = System.currentTimeMillis();
try
(MappedByteBufferReader reader =
new
MappedByteBufferReader(
"E:\\software\\software.zip"
,
1
<<
30
);
FileChannel wc = FileChannel.open(Paths.get(
"E:\\software\\software_reader.zip"
),
StandardOpenOption.CREATE,
StandardOpenOption.READ,
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.WRITE)) {
byte
[] data;
MappedByteBuffer writer = wc.map(FileChannel.MapMode.READ_WRITE,
0
, reader.size());
while
((data = reader.read()) !=
null
) {
writer.put(data);
}
writer.clear();
System.out.println(
"used times: "
+ (System.currentTimeMillis() - start));
}
catch
(Exception e) {
e.printStackTrace();
}
}
}
|