在TCP下進行大文件傳輸不象小文件那樣直接打包個BUFFER發送出去,因為文件比較大所以不可能把文件讀到一個BUFFER發送出去.主要有些文件的大小可能是1G,2G或更大,分配這么大的BUFFER對內存來說顯然是不現實的事情;針對服務端的設計來說就更需要嚴緊些,BUFFER大小的限制也是變得很重要.下面介紹使用Beetle簡單地實現大文件在TCP的傳輸應用.
協議制定
既然需要把文件分塊來處理,那在TCP傳輸的過程需要制定一些協議來規范數據有效性,數據協議主要有三個:告訴服務器需要上傳文件,文件塊上傳和返回每個環節處理的結果.
1)上傳文件指令
public class Upload:ObjectMessage
{
public string FileMD5
{
get;
set;
}
public string FileName
{
get;
set;
}
public long FileSize
{
get;
set;
}
public override void FromProtocolData(HttpData httpbase)
{
FileName = httpbase[CONSTVALUE.HEADER_NAME];
FileMD5 = httpbase[CONSTVALUE.HEADER_MD5];
FileSize = long.Parse(httpbase[CONSTVALUE.HEADER_FILESIZE]);
}
protected override void OnDisposed()
{
}
protected override void OnToProtocolData(HttpData httpbase)
{
httpbase.Command = CONSTVALUE.COMMAND_UPLOAD;
httpbase[CONSTVALUE.HEADER_MD5] = FileMD5;
httpbase[CONSTVALUE.HEADER_NAME] = FileName;
httpbase[CONSTVALUE.HEADER_FILESIZE] = FileSize.ToString();
}
}
2)上傳文件塊指令
public class UploadData:ObjectMessage
{
public string FileMD5
{
get;
set;
}
public Beetle.ByteArraySegment Data
{
get;
set;
}
public override void FromProtocolData(HttpData httpbase)
{
FileMD5 = httpbase[CONSTVALUE.HEADER_MD5];
Data = httpbase.Content;
}
protected override void OnDisposed()
{
if (Data != null)
{
FileTransferPackage.BufferPool.Push(Data);
Data = null;
}
}
protected override void OnToProtocolData(HttpData httpbase)
{
httpbase.Command = CONSTVALUE.COMMAND_UPLOAD_DATA;
httpbase[CONSTVALUE.HEADER_MD5] = FileMD5;
httpbase.Content = Data;
}
}
3)返回值指令
public class Result :ObjectMessage
{
public string FileMD5
{
get;
set;
}
public bool Error
{
get;
set;
}
public string ErrorDetail
{
get;
set;
}
public override void FromProtocolData(HttpData httpbase)
{
ErrorDetail = httpbase[CONSTVALUE.HEADER_STATUS_DETAIL];
Error = httpbase[CONSTVALUE.HEADER_STATUS] == CONSTVALUE.VALUE_SUCCESS;
FileMD5 = httpbase[CONSTVALUE.HEADER_MD5];
}
protected override void OnDisposed()
{
}
protected override void OnToProtocolData(HttpData httpbase)
{
httpbase.Command = CONSTVALUE.COMMAND_RESULT;
if (Error)
{
httpbase[CONSTVALUE.HEADER_STATUS] = CONSTVALUE.VALUE_SUCCESS;
}
else
{
httpbase[CONSTVALUE.HEADER_STATUS] = CONSTVALUE.VALUE_ERROR;
}
httpbase[CONSTVALUE.HEADER_STATUS_DETAIL] = ErrorDetail;
httpbase[CONSTVALUE.HEADER_MD5] = FileMD5;
}
}
ObjectMessage是Beetle一個簡化HTTP協議的擴展對象,它提供自定義Header和Body等功能.
文件讀寫器
既然需要處理文件塊,那提供一些簡單的文件塊讀取和寫入方法是比較重要的.它不僅從設計解決功能的偶合度,還可以方便今后的利用.
1)UploadReader
public class UploadReader : IDisposable
{
public UploadReader(string file)
{
mStream = System.IO.File.OpenRead(file);
StringBuilder sb = new StringBuilder();
MD5 md5Hasher = MD5.Create();
foreach (Byte b in md5Hasher.ComputeHash(mStream))
sb.Append(b.ToString("x2").ToLower());
FileMD5 = sb.ToString();
mStream.Position = 0;
FileSize = mStream.Length;
FileName = System.IO.Path.GetFileName(file);
}
private System.IO.FileStream mStream = null;
public string FileName
{
get;
set;
}
public long LastReadLength
{
get;
set;
}
public long ReadLength
{
get;
set;
}
public long FileSize
{
get;
set;
}
public string FileMD5
{
get;
set;
}
public bool Completed
{
get
{
return mStream != null && ReadLength == mStream.Length;
}
}
public void Close()
{
if (mStream != null)
{
mStream.Close();
mStream.Dispose();
}
}
public void Reset()
{
mStream.Position = 0;
LastReadLength = 0;
ReadLength = 0;
}
public void Read(ByteArraySegment segment)
{
int loads = mStream.Read(segment.Array, 0, FileTransferPackage.BUFFER_SIZE);
segment.SetInfo(0, loads);
ReadLength += loads;
}
public void Dispose()
{
mStream.Dispose();
}
public override string ToString()
{
string value= string.Format("{0}(MD5:{4})\r\n\r\n[{1}/{2}({3}/秒)]",FileName,ReadLength,FileSize,ReadLength-LastReadLength,FileMD5);
if (!Completed)
{
LastReadLength = ReadLength;
}
return value;
}
}
UploadReader的功能主要是把文件流讀取到指定大小的Buffer中,並提供方法獲取當前的讀取情況
2)UploadWriter
public class UploadWriter
{
public UploadWriter(string rootPath, string filename,string fileMD5,long size)
{
mFullName = rootPath + filename;
FileName = filename;
FileMD5 = fileMD5;
Size = size;
}
private string mFullName;
private System.IO.FileStream mStream;
public System.IO.FileStream Stream
{
get
{
if (mStream == null)
{
mStream = System.IO.File.Create(mFullName+".up");
}
return mStream;
}
}
public long WriteLength
{
get;
set;
}
public long LastWriteLength
{
get;
set;
}
public long Size
{
get;
set;
}
public string FileName
{
get;
set;
}
public string FileMD5
{
get;
set;
}
public bool Write(ByteArraySegment segment)
{
Stream.Write(segment.Array, 0, segment.Count);
WriteLength += segment.Count;
Stream.Flush();
if (WriteLength == Size)
{
Stream.Close();
if (System.IO.File.Exists(mFullName))
System.IO.File.Delete(mFullName);
System.IO.File.Move(mFullName + ".up", mFullName);
return true;
}
return false;
}
}
UploadWriter的功能主要是把文件寫入到臨時文件中,寫入完成后再更改相應的名稱,為了方便查詢同樣也提供了一些寫入情況信息.
服務端代碼
如果有了解過Beetle的服務端制定的話,那服務端的實現是非常簡單的,只需要寫一個對象承繼ServerBase並實現數據接收方法處理即可以,接收的數據會會自動轉換成之前定義的消息對象,而服務端內部處理的細節是完全不用關心.
protected override void OnMessageReceive(Beetle.PacketRecieveMessagerArgs e)
{
base.OnMessageReceive(e);
if (e.Message is Protocol.Upload)
{
OnUpload(e.Channel, e.Message as Protocol.Upload);
}
else if (e.Message is Protocol.UploadData)
{
OnUploadData(e.Channel, e.Message as Protocol.UploadData);
}
}
private Protocol.Result GetErrorResult(string detail)
{
Protocol.Result result = new Protocol.Result();
result.Error = true;
result.ErrorDetail = detail;
return result;
}
private void OnUpload(Beetle.TcpChannel channel, Protocol.Upload e)
{
Protocol.Result result;
if (mTask[e.FileMD5] != null)
{
result = GetErrorResult( "該文件正在上傳任務中!");
channel.Send(result);
return;
}
UploadWriter writer = new UploadWriter(mRootPath, e.FileName, e.FileMD5, e.FileSize);
lock (mTask)
{
mTask[e.FileMD5] = writer;
}
result = new Protocol.Result();
channel.Send(result);
}
private void OnUploadData(Beetle.TcpChannel channel, Protocol.UploadData e)
{
using (e)
{
Protocol.Result result;
UploadWriter writer = (UploadWriter)mTask[e.FileMD5];
if (writer == null)
{
result = GetErrorResult("上傳任務不存在!");
channel.Send(result);
return;
}
if (writer.Write(e.Data))
{
lock (mTask)
{
mTask.Remove(e.FileMD5);
}
}
result = new Protocol.Result();
result.FileMD5 = writer.FileMD5;
channel.Send(result);
}
}
當接收到客戶求上傳請求后會建立對應MD5的文件寫入器,后面文件塊的上傳寫入相關對象即可.
客戶端代碼
Beetle對於Client的支持也是非常簡單方便,只需要定義一個TcpChannel直接發送定義的對象消息並獲取服務器端返回的消息即可.
1 private void OnUpload(object state)
2 {
3 Lib.UploadReader reader = (Lib.UploadReader)state;
4 try
5 {
6 IsUpload = true;
7 Lib.Protocol.Upload upload = new Lib.Protocol.Upload();
8 upload.FileMD5 = reader.FileMD5;
9 upload.FileName = reader.FileName;
10 upload.FileSize = reader.FileSize;
11 Lib.Protocol.Result result = mClient.Send<Lib.Protocol.Result>(upload);
12 if (result.Error)
13 {
14 mLastError = result.ErrorDetail;
15 return;
16 }
17 while (!reader.Completed)
18 {
19 mLastError = "文件上傳中...";
20 Lib.Protocol.UploadData data = new Lib.Protocol.UploadData();
21 data.Data = Lib.FileTransferPackage.BufferPool.Pop();
22 data.FileMD5 = reader.FileMD5;
23 reader.Read(data.Data);
24 result = mClient.Send<Lib.Protocol.Result>(data);
25 if (result.Error)
26 {
27 mLastError = result.ErrorDetail;
28 return;
29 }
30 }
31 mLastError = "文件上傳完成!";
32
33 }
34 catch (Exception e_)
35 {
36 mLastError = e_.Message;
37 }
38 mReader.Reset();
39 IsUpload = false;
40
41 }
整個過程只需要一個方法卻可完成,首先把需要上傳的文件信息發送到服務器,當服務器確認后不停地把文件塊信息輸送到服務端即可.
使用測試
下載代碼
