Minio 使用.NET + Vue 實現斷點續傳、秒傳

來源：https://blog.csdn.net/qq_33474360/article/details/110238308

Minio是什么
官方解釋：
MinIO 是一個基於Apache License v2.0開源協議的對象存儲服務。它兼容亞馬遜S3雲存儲服務接口，非常適合於存儲大容量非結構化的數據，例如圖片、視頻、日志文件、備份數據和容器/虛擬機鏡像等，而一個對象文件可以是任意大小，從幾kb到最大5T不等。

說白了Minio就是一個文件管理服務工具，最大支持5個T的文件上傳，具體存儲機制與細節自行查看官網去研究。

安裝
Minio服務安裝與部署
雖然官方提供了SDK，但是並不能滿足我們的需要，所以我們需要去下載源碼，但盡量不去修改源碼，以免日后不好升級。

從此處進入官網中文網站
從此處進入官網英文網站

Minio dotnet
從此處下載
將該項目中的Minio文件夾及解決方案目錄下的Minio.snk、mono_install.sh、netfx.props文件拷貝至你的項目中，並將Minio項目添加到你的項目.
前端參考
主要參考該博主的分片上傳流程，不用minio是一樣的,代碼幾乎都是用的該博主的，具體的我就不貼了
點擊跳轉

后端流程及代碼
該流程也是根據前端來配合的，主要后端代碼是從一個java朋友那里修改過來的,歡迎討論

前端發送第一次請求
校驗文件是否傳輸，以及minio分配uploadId及part[]
uploadid是minio進行上傳分片和合並操作的關鍵
part[]是根據你傳入的文件totalsize以5M為分割點進行的文件分片生成part[],在上傳時part[]中的每一個part只有自己的序號(從1開始),沒有任何文件信息,當進行分片上傳時,上傳對應的分片及序號即可

邏輯代碼
文件已上傳
如果已經上傳，返回skipUpload為true，以此實現秒傳。

/// <summary>
        /// 獲取文件信息
        /// </summary>
        /// <param name="identifier"></param>
        /// <param name="totalSize"></param>
        public FileInfoModel GetFileInfo(string identifier, string fileName, long totalSize)
        {       
            // 根據MD5查詢數據庫此文件是否已上傳成功
            // 成功則直接秒傳
            using (MEDbContext db = new MEDbContext())
            {
                // 驗證該MD5是否存在
                var entites = db.SystemFileBusinessEntities.Where(x => x.ObjectName == identifier && x.BucketName == bucketName);
                if (!entites.Any())
                    return null;
                // 驗證文件名是否發生過變更
                return FileUpdate(identifier, fileName, totalSize, db);
            }
        }

        /// <summary>
        /// 文件是否需要更新
        /// </summary>
        /// <param name="identifier"></param>
        /// <param name="fileName"></param>
        /// <param name="totalSize"></param>
        /// <param name="db"></param>
        /// <returns></returns>
        private static FileInfoModel FileUpdate(string identifier, string fileName, long totalSize, MEDbContext db)
        {
            string title = Path.GetFileNameWithoutExtension(fileName);
            string extension = Path.GetExtension(fileName);
            var entity = db.Set<SystemFileBusinessEntity>().FirstOrDefault(x => x.BucketName == bucketName && x.ObjectName == identifier && x.Title == title && x.Extension == extension);
            if (entity != null)
            {
                return new FileInfoModel
                {
                    Id = entity.Id,
                    NeedMerge = false,
                    SkipUpload = true,
                    FileName = fileName,
                };
            }
            else
            {
                entity = db.Set<SystemFileBusinessEntity>().Add(new SystemFileBusinessEntity
                {
                    BucketName = bucketName,
                    CreateTime = DateTime.Now,
                    Extension = extension,
                    ObjectName = identifier,
                    Title = title,
                    UpdateTime = DateTime.Now,
                    Size = totalSize
                }).Entity;
                db.SaveChanges();
                return new FileInfoModel
                {
                    Id = entity.Id,
                    NeedMerge = false,
                    SkipUpload = true,
                    FileName = fileName,
                };
            }
        }

文件未上傳

如果沒有上傳,返回分片信息uploaded，實現斷點續傳，如果uploaded為空代表還沒傳輸過該文件

/// <summary>
        /// 查詢分片信息
        /// </summary>
        /// <param name="identifier"></param>
        /// <param name="totalSize"></param>
        /// <param name="fileName"></param>
        /// <returns></returns>
        public async Task<int[]> GePartInfoAsync(string identifier, long totalSize, string fileName)
        {
            // Redis查詢已上傳分片,返回未上傳分片
            var basicInfoIsExist = await _redisHelper.HashExistsAsync("file_" + identifier, "basic_info");
            if (basicInfoIsExist == false)
            {
                var partDic = await Global.MINIOAPI.MultUploadByStreamAsync(bucketName, identifier, totalSize, null, "application/octet-stream", null);
                var fileModel = new FileModel();
                fileModel.UploadId = partDic["uploadId"].ToString();
                fileModel.Parts = (Part[])partDic["parts"];
                fileModel.Md5 = identifier;
                fileModel.Size = totalSize;
                fileModel.FileName = fileName;
                await _redisHelper.HashSetAsync("file_" + identifier, "basic_info", fileModel);
                return new int[] { };
            }
            List<string> keys = await _redisHelper.HashKeysAsync<string>("file_" + identifier + "_part");
            return keys.Select(s => Convert.ToInt32(s)).ToArray();
    }

minio生成uploadId及parts[]關鍵代碼

/// <summary>
        /// 在進行多文件上傳之前，先初始化，獲取uploaderId等信息
        /// </summary>
        /// <param name="bucketName"></param>
        /// <param name="objectName"></param>
        /// <param name="headerMap"></param>
        /// <param name="contentType"></param>
        /// <returns></returns>
        public async Task<string> InitMultUploadAsync(string bucketName, string objectName,
                                     Dictionary<string, string> headerMap, string contentType)
        {

            if (headerMap == null)
            {
                headerMap = new Dictionary<string, string>();
            }

            if (contentType == null)
            {
                if (!headerMap.ContainsKey("Content-Type"))
                {
                    headerMap.Add("Content-Type", "application/octet-stream");
                }
            }
            else
            {
                headerMap.Add("Content-Type", contentType);
            }


            string uploadId = await this.NewMultipartUploadAsync(bucketName, objectName, new Dictionary<string, string>(), headerMap, default).ConfigureAwait(false);
            return uploadId;
        }

        public Part[] MakeMultUpload(long size, Part[] parts)
        {
            /* Multipart upload */
            Part[] totalParts = parts;
            if (totalParts == null)
            {
                dynamic multiPartInfo = utils.CalculateMultiPartSize(size);
                double partSize = multiPartInfo.partSize;
                double partCount = multiPartInfo.partCount;
                double lastPartSize = multiPartInfo.lastPartSize;
                totalParts = new Part[(int)partCount];
                for (int i = 0; i < totalParts.Length; i++)
                {
                    totalParts[i] = new Part() { PartNumber = i + 1 };
                    if (i != totalParts.Length - 1)
                    {
                        totalParts[i].Size = ((long)partSize);
                    }
                    else
                    {
                        totalParts[i].Size = ((long)lastPartSize);
                    }
                }
            }
            return totalParts;
        }

分片上傳

邏輯代碼

等校驗完畢后就會根據以5M一片的次數發起post請求上傳一個個的分片
上傳成功后將分片信息存入，主要是分片ChunkNumber及Etag

/// <summary>
        /// 上載分片
        /// </summary>
        /// <returns>是否需要合並</returns>
        public async Task UploadPartAsync(MinioFilePartUpload partData)
        {
            string md5 = partData.Identifier;
            int partNumber = partData.ChunkNumber;
            FileModel fileModel = await _redisHelper.HashGeAsync<FileModel>("file_" + md5, "basic_info");
            if (fileModel == null) // 為空異常（可能傳完了）
                throw new Exception("上傳出現異常");
            // 加鎖上傳（）
            string key = "file_" + md5 + "_part";
            string strPartNumber = partNumber.ToString();

            string lockKey = "lock_" + key + "_" + strPartNumber;
            string locVlue = strPartNumber;
            try
            {
                if (_redisHelper.LockTake(lockKey, locVlue, 20))
                {
                    Console.WriteLine(locVlue + ":上傳開始");
                    var isExist = await _redisHelper.HashExistsAsync(key, strPartNumber);
                    if (isExist == false)
                    {
                        long size = partData.UpFile.Length;
                        Part[] parts = await Global.MINIOAPI.MultUploadByStreamAsync(fileModel.UploadId, bucketName, md5, partData.UpFile.OpenReadStream(), size, fileModel.Parts, partNumber);
                        string etag = parts[partNumber - 1].ETag;
                        if (string.IsNullOrEmpty(etag))
                            throw new Exception($"{strPartNumber}:{partData.ChunkNumber}獲取文件etag失敗");
                        _ = await _redisHelper.HashSetAsync(key, strPartNumber, etag);
                        await _redisHelper.HashIncrementAsync("file_" + md5, "part_count", 1);
                    }
                }
            }
            finally
            {
                _redisHelper.LockRelease(lockKey, locVlue);
            }
        }

minio上傳分片關鍵代碼

此處主要是根據上傳分片流獲取etag，合並時要根據etag作合並操作

public async Task<Part[]> MakeMultUploadAsync(string uploadId, string bucketName, string objectName, long expectedReadSize, Object data, Part[] parts, int partNumber)
        {
            /* Multipart upload */
            Part[] totalParts = parts;
            try
            {
                Stream stream = (Stream)data;
                byte[] bytes = new byte[stream.Length];
                await stream.ReadAsync(bytes, 0, bytes.Length);
               
                Dictionary<string, string> metaData = new Dictionary<string, string>();
                metaData["Content-Type"] = "application/octet-stream";
                var sseHeaders = new Dictionary<string, string>();
                ServerSideEncryption serverSideEncryption = new  SSES3();
                serverSideEncryption.Marshal(sseHeaders);
                string etag = await PutObjectAsync(bucketName, objectName, uploadId, partNumber, bytes, metaData, sseHeaders, default);
               
                totalParts[partNumber - 1].ETag = (etag);
            }
            catch (Exception e)
            {
                //出錯后打出異常，繼續執行，這樣就可以得到出錯的卷
                //totalParts[partNumber - 1].setState(-1);
                Console.WriteLine(e.StackTrace);
            }

            return totalParts;
        }

合並

邏輯代碼

/// <summary>
        /// 分片合並
        /// </summary>
        /// <param name="md5"></param>
        /// <param name="fileName"></param>
        /// <returns></returns>
        public async Task Compose(FileCompose fileCompose)
        {
            string identifier = fileCompose.Identifier;
            string key = "file_" + identifier;
            var fileModel = await _redisHelper.HashGeAsync<FileModel>("file_" + identifier, "basic_info");
            if (fileModel == null) // 為空代表異常（可能傳完了）
                throw new Exception("上傳出現異常");
            try
            {
                if (!_redisHelper.LockTake("lock" + key, key, 20))
                {
                    Console.WriteLine("該鎖已被使用");
                    return;
                }
                string cutStr = await _redisHelper.HashGeAsync(key, "part_count");
                string fCoutStr = fileModel.Parts.Length.ToString();
                while (cutStr != fCoutStr)
                {
                    await Task.Delay(200);
                    cutStr = await _redisHelper.HashGeAsync(key, "part_count");
                }

                using (MEDbContext db = new MEDbContext())
                {
                    // 查詢數據庫中是否已存入信息
                    var entity = db.Set<SystemFileBusinessEntity>().FirstOrDefault(x => x.BucketName == bucketName && x.ObjectName == identifier);
                    if (entity == null)
                    {
                        Part[] parts = new Part[fileModel.Parts.Length];
                        for (int i = 0; i < fileModel.Parts.Length; i++)
                        {
                            parts[i] = fileModel.Parts[i];
                            parts[i].ETag = await _redisHelper.HashGeAsync("file_" + identifier + "_part", fileModel.Parts[i].PartNumber.ToString());
                        }
                        await Global.MINIOAPI.CommitMultUploadAsync(fileModel.UploadId, bucketName, identifier, parts);
                        _redisHelper.KeyDelete(new string[] { "file_" + identifier, "file_" + identifier + "_part" }.ToList());
                    }
                    FileUpdate(identifier, fileModel.FileName, fileModel.Size, db);
                }
            }
            finally
            {
                _redisHelper.LockRelease("lock" + key, key);
            }
        }

minio

public Task CommitMultUploadAsync(string uploadId, string bucketName, string objectName, Part[] parts)
        {
            Dictionary<int, string> etags = new Dictionary<int, string>();
            for (int partNumber = 1; partNumber <= parts.Length; partNumber++)
            {
                etags[partNumber] = parts[partNumber - 1].ETag;
            }
            //this.Secure = true;
            Dictionary<string, string> metaData = new Dictionary<string, string>();
            metaData["Content-Type"] = "application/octet-stream";
            var sseHeaders = new Dictionary<string, string>();
            ServerSideEncryption serverSideEncryption = new SSES3();
            serverSideEncryption.Marshal(sseHeaders);
            return this.CompleteMultipartUploadAsync(bucketName, objectName, uploadId, etags, sseHeaders, default);
        }

        // 如果需要調用MD5SUM進行文件校驗的話就加上 Content-MD5 Header,否則調用原來Minio的方法即可,不用新加重寫
        private async Task CompleteMultipartUploadAsync(string bucketName, string objectName, string uploadId, Dictionary<int, string> etags, Dictionary<string, string> meta, CancellationToken cancellationToken)
        {
            //this.Secure = true;
            var request = await this.CreateRequest(Method.POST, bucketName,
                                                     objectName: objectName,
                                                     headerMap: meta)
                                    .ConfigureAwait(false);
            request.AddQueryParameter("uploadId", $"{uploadId}");

            List<XElement> parts = new List<XElement>();

            for (int i = 1; i <= etags.Count; i++)
            {
                parts.Add(new XElement("Part",
                                       new XElement("PartNumber", i),
                                       new XElement("ETag", etags[i])));
            }

            var completeMultipartUploadXml = new XElement("CompleteMultipartUpload", parts);
            var bodyString = completeMultipartUploadXml.ToString();
            var body = System.Text.Encoding.UTF8.GetBytes(bodyString);

            request.AddParameter("application/xml", body, ParameterType.RequestBody);
            
            //var md5 = MD5.Create();
            //byte[] hash = md5.ComputeHash(body);
            //string base64 = Convert.ToBase64String(hash);
            //request.AddOrUpdateParameter("Content-MD5", base64, ParameterType.HttpHeader);
            var response = await this.ExecuteTaskAsync(this.NoErrorHandlers, request, cancellationToken).ConfigureAwait(false);
        }

前端注意：

加上此屬性 forceChunkSize: true, // 強制每片都小於分片大小
在進行校驗MD5的時候將暫停按鈕隱藏，否則會出現上傳時傳遞的不是MD5的情況
參考資料:
https://blog.csdn.net/lmlm21/article/details/107768581
https://blog.csdn.net/anxyh_name/article/details/108397774
https://www.cnblogs.com/xiahj/p/vue-simple-uploader.html
————————————————
版權聲明：本文為CSDN博主「洋洋灑灑魏先生」的原創文章，遵循CC 4.0 BY-SA版權協議，轉載請附上原文出處鏈接及本聲明。
原文鏈接：https://blog.csdn.net/qq_33474360/article/details/110238308