大多數RTSP的播放都面向直播場景，除了H.264，還需要支持H.265，目前市面上的RTSP H.265攝像頭越來越多，支持H.265的RTSP播放器迫在眉睫，此外，單純的播放H.265還不夠，還需要可以能把H.265的數據能錄制下來。
青犀團隊接觸到好多開發者，希望能在播放的同時，獲取到YUV或RGB數據，進行人臉匹配等算法分析，所以音視頻回調可選。
EasyPlayer-RTSP實現支持H265編碼格式
EasyPlayer-RTSP-Win已全面支持H265的RTSP流的解碼播放，這里就支持H265過程做簡要介紹；
一、libEasyRTSPClient庫已支持H265視頻源的RTSP流的拉取和解析
二、H265頭解析
H265和H264類似，不過其NAL type格式更多樣化，除了SPS，PPS之外，還增加了VPS，下面就針對H265幀nal 頭做簡單分析；
首先，看X265源碼中的H265nal頭格式定義：
//H265 NAL type
//this enum have been defined in x265.h
typedef enum tagH265NalUnitType
{
NAL_UNIT_CODED_SLICE_TRAIL_N = 0, // 0
NAL_UNIT_CODED_SLICE_TRAIL_R, // 1
NAL_UNIT_CODED_SLICE_TSA_N, // 2
NAL_UNIT_CODED_SLICE_TLA, // 3 // Current name in the spec: TSA_R
NAL_UNIT_CODED_SLICE_STSA_N, // 4
NAL_UNIT_CODED_SLICE_STSA_R, // 5
NAL_UNIT_CODED_SLICE_RADL_N, // 6
NAL_UNIT_CODED_SLICE_DLP, // 7 // Current name in the spec: RADL_R
NAL_UNIT_CODED_SLICE_RASL_N, // 8
NAL_UNIT_CODED_SLICE_TFD, // 9 // Current name in the spec: RASL_R
NAL_UNIT_RESERVED_10,
NAL_UNIT_RESERVED_11,
NAL_UNIT_RESERVED_12,
NAL_UNIT_RESERVED_13,
NAL_UNIT_RESERVED_14,
NAL_UNIT_RESERVED_15,
NAL_UNIT_CODED_SLICE_BLA, // 16 // Current name in the spec: BLA_W_LP
NAL_UNIT_CODED_SLICE_BLANT, // 17 // Current name in the spec: BLA_W_DLP
NAL_UNIT_CODED_SLICE_BLA_N_LP, // 18
NAL_UNIT_CODED_SLICE_IDR, // 19 // Current name in the spec: IDR_W_DLP
NAL_UNIT_CODED_SLICE_IDR_N_LP, // 20
NAL_UNIT_CODED_SLICE_CRA, // 21
NAL_UNIT_RESERVED_22,
NAL_UNIT_RESERVED_23,
NAL_UNIT_RESERVED_24,
NAL_UNIT_RESERVED_25,
NAL_UNIT_RESERVED_26,
NAL_UNIT_RESERVED_27,
NAL_UNIT_RESERVED_28,
NAL_UNIT_RESERVED_29,
NAL_UNIT_RESERVED_30,
NAL_UNIT_RESERVED_31,
NAL_UNIT_VPS, // 32
NAL_UNIT_SPS, // 33
NAL_UNIT_PPS, // 34
NAL_UNIT_ACCESS_UNIT_DELIMITER, // 35
NAL_UNIT_EOS, // 36
NAL_UNIT_EOB, // 37
NAL_UNIT_FILLER_DATA, // 38
NAL_UNIT_SEI, // 39 Prefix SEI
NAL_UNIT_SEI_SUFFIX, // 40 Suffix SEI
NAL_UNIT_RESERVED_41,
NAL_UNIT_RESERVED_42,
NAL_UNIT_RESERVED_43,
NAL_UNIT_RESERVED_44,
NAL_UNIT_RESERVED_45,
NAL_UNIT_RESERVED_46,
NAL_UNIT_RESERVED_47,
NAL_UNIT_UNSPECIFIED_48,
NAL_UNIT_UNSPECIFIED_49,
NAL_UNIT_UNSPECIFIED_50,
NAL_UNIT_UNSPECIFIED_51,
NAL_UNIT_UNSPECIFIED_52,
NAL_UNIT_UNSPECIFIED_53,
NAL_UNIT_UNSPECIFIED_54,
NAL_UNIT_UNSPECIFIED_55,
NAL_UNIT_UNSPECIFIED_56,
NAL_UNIT_UNSPECIFIED_57,
NAL_UNIT_UNSPECIFIED_58,
NAL_UNIT_UNSPECIFIED_59,
NAL_UNIT_UNSPECIFIED_60,
NAL_UNIT_UNSPECIFIED_61,
NAL_UNIT_UNSPECIFIED_62,
NAL_UNIT_UNSPECIFIED_63,
NAL_UNIT_INVALID,
}H265NalUnitType;
#endif
我們可以看到其中VPS, SPS和PPS的定義：
NAL_UNIT_VPS, // 32
NAL_UNIT_SPS, // 33
NAL_UNIT_PPS, // 34
同樣，我們也很容易知道P幀NAL type定義是0-9， I幀定義是16-21；可見H265的NAL type定義比H264要多樣化，判斷也不限制於一種類型；
同時，測試發現，實際H265幀數據中的VPS=0x40 , SPS=0x42, PPS=0x44, 通過換算，我們不難得出：NALtype*2 = 實際的流中的NaLType;
具體解析過程如下：
//輸入的pbuf必須包含start code(00 00 00 01)
int GetH265VPSandSPSandPPS(char *pbuf, int bufsize, char *_vps, int *_vpslen, char *_sps, int *_spslen, char *_pps, int *_ppslen)
{
char vps[512]={0}, sps[512] = {0}, pps[128] = {0};
int vpslen=0, spslen=0, ppslen=0, i=0, iStartPos=0, ret=-1;
int iFoundVPS=0, iFoundSPS=0, iFoundPPS=0, iFoundSEI=0;
if (NULL == pbuf || bufsize<4) return -1;
for (i=0; i<bufsize; i++)
{
if ( (unsigned char)pbuf[i] == 0x00 && (unsigned char)pbuf[i+1] == 0x00 &&
(unsigned char)pbuf[i+2] == 0x00 && (unsigned char)pbuf[i+3] == 0x01 )
{
printf("0x%X\n", (unsigned char)pbuf[i+4]);
switch ((unsigned char)pbuf[i+4])
{
case 0x40: //VPS
{
iFoundVPS = 1;
iStartPos = i+4;
}
break;
case 0x42: //SPS
{
if (iFoundVPS == 0x01 && i>4)
{
vpslen = i-iStartPos;
if (vpslen>256) return -1; //vps長度超出范圍
memset(vps, 0x00, sizeof(vps));
memcpy(vps, pbuf+iStartPos, vpslen);
}
iStartPos = i+4;
iFoundSPS = 1;
}
break;
case 0x44: //PPS
{
if (iFoundSPS == 0x01 && i>4)
{
spslen = i-iStartPos;
if (spslen>256) return -1;
memset(sps, 0x0, sizeof(sps));
memcpy(sps, pbuf+iStartPos, spslen);
}
iStartPos = i+4;
iFoundPPS = 1;
}
break;
case 0x4E: //Prefix SEI
case 0x50: //Suffix SEI
case 0x20: //I frame 16
case 0x22: //I frame 17
case 0x24: //I frame 18
case 0x26: //I frame 19
case 0x28: //I frame 20
case 0x2A: //I frame 21(acturally we should find naltype 16-21)
{
if (iFoundPPS == 0x01 && i>4)
{
ppslen = i-iStartPos;
if (ppslen>256) return -1;
memset(pps, 0x0, sizeof(pps));
memcpy(pps, pbuf+iStartPos, ppslen);
}
iStartPos = i+4;
iFoundSEI = 1;
}
break;
default:
break;
}
}
if (iFoundSEI == 0x01) break;
}
if (iFoundVPS == 0x01)
{
if (vpslen < 1)
{
if (bufsize < sizeof(vps))
{
vpslen = bufsize-4;
memset(vps, 0x00, sizeof(vps));
memcpy(vps, pbuf+4, vpslen);
}
}
if (vpslen > 0)
{
if (NULL != _vps) memcpy(_vps, vps, vpslen);
if (NULL != _vpslen) *_vpslen = vpslen;
}
ret = 0;
}
if (iFoundSPS == 0x01)
{
if (spslen < 1)
{
if (bufsize < sizeof(sps))
{
spslen = bufsize-4;
memset(sps, 0x00, sizeof(sps));
memcpy(sps, pbuf+4, spslen);
}
}
if (spslen > 0)
{
if (NULL != _sps) memcpy(_sps, sps, spslen);
if (NULL != _spslen) *_spslen = spslen;
}
ret = 0;
}
if (iFoundPPS == 0x01)
{
if (ppslen < 1)
{
if (bufsize < sizeof(pps))
{
ppslen = bufsize-4;
memset(pps, 0x00, sizeof(pps));
memcpy(pps, pbuf+4, ppslen); //pps
}
}
if (ppslen > 0)
{
if (NULL != _pps) memcpy(_pps, pps, ppslen);
if (NULL != _ppslen) *_ppslen = ppslen;
}
ret = 0;
}
return ret;
}
三、 解碼器需支持H265
解碼器直接使用最新的FFMPEG庫即支持H265解碼，且軟解效率還可以，大家如果不知道怎么用，可以去看看ffplay的源碼，這里不做過多贅述；這里就EasyPlayer調用遇到的問題做簡單說明：
- 舊版的ffmpeg以及live555等對H265的定義是對“H265”子串做的字串格式組合，而新版的FFMPEG使用的自定義的順序定義的枚舉類型，所以在使用過程中可能出現對應不上的情況，比如，在libEasyRTSPClient庫中對H265的定義為：#define EASY_SDK_VIDEO_CODEC_H265 0x48323635 /* 1211250229 */
而FFMPEG中定義H265（HEVC）格式為174
EasyPlayer中進行格式統一代碼如下：
//H265 codecID改成FFMPEG新版的
int nCodec = (_frameinfo->codec == EASY_SDK_VIDEO_CODEC_H265) ? 174 : _frameinfo->codec;
- EasyPlayer中之前對關鍵幀幀解碼失敗的處理是將以該I幀為關鍵幀為依托的所有P幀丟棄，當然這從某種程度上是可以避免花屏的，但是測試解碼H265時發現，H265的第一個I幀會經常解碼失敗，經調試發現其實是FFNPEG的解碼函數返回沒有解碼完成的結果被程序判斷為解碼失敗，而這個時候應該不做任何處理等下一次返回的時候就能獲取到正確的返回結果了，EasyPlayer處理如下：
nRet = FFD_DecodeVideo3(pDecoderObj->ffDecoder, pbuf, frameinfo.length, pThread->yuvFrame[pThread->decodeYuvIdx].pYuvBuf, frameinfo.width, frameinfo.height, lTimestamp, lTimestamp);
if (0 != nRet)
{
if(nRet == -4)//-4表示為當前幀尚未解碼完成，不作為錯誤判斷
{
_TRACE("視頻幀解碼尚未完成[%d]... framesize:%d %02X %02X %02X %02X %02X %02X %02X %02X %02X %02X\n", nRet, frameinfo.length,
(unsigned char)pbuf[0], (unsigned char)pbuf[1], (unsigned char)pbuf[2], (unsigned char)pbuf[3], (unsigned char)pbuf[4],
(unsigned char)pbuf[5], (unsigned char)pbuf[6], (unsigned char)pbuf[7], (unsigned char)pbuf[8], (unsigned char)pbuf[9]);
}
else
{
_TRACE("視頻幀解解碼失敗[%d]... framesize:%d %02X %02X %02X %02X %02X %02X %02X %02X %02X %02X\n", nRet, frameinfo.length,
(unsigned char)pbuf[0], (unsigned char)pbuf[1], (unsigned char)pbuf[2], (unsigned char)pbuf[3], (unsigned char)pbuf[4],
(unsigned char)pbuf[5], (unsigned char)pbuf[6], (unsigned char)pbuf[7], (unsigned char)pbuf[8], (unsigned char)pbuf[9]);
if (frameinfo.type == EASY_SDK_VIDEO_FRAME_I) //關鍵幀
{
_TRACE("[ch%d]當前關鍵幀解碼失敗...\n", pThread->channelId);
#ifdef _DEBUG
FILE *f = fopen("keyframe.txt", "wb");
if (NULL != f)
{
fwrite(pbuf, 1, frameinfo.length, f);
fclose(f);
}
#endif
}
else
{
#ifdef _DEBUG
FILE *f = fopen("pframe.txt", "wb");
if (NULL != f)
{
fwrite(pbuf, 1, frameinfo.length, f);
fclose(f);
}
#endif
}
pThread->findKeyframe = 0x01;
}
經測試，經過如上代碼 修改就能有效的避免的解碼H265視頻的時候開頭幀總是會局部卡幀或者花屏的情況出現。
四、H265格式視頻寫MP4
這里接着之前EasyPlayer系列的寫MP4篇講，將H265封裝MP4；
-
解析H265的頭，或者VPS，SPS和PPS
從H265幀中取出NAL頭在上文已經作過講解這里就不做過多贅述； -
寫入VPS, SPS和PPS 等關鍵解碼信息
bool EasyMP4Writer::WriteH265VPSSPSandPPS(unsigned char*vps, int vpslen, unsigned char*sps, int spslen,
unsigned char*pps, int ppslen, int width, int height)
{
if (m_nCreateFileFlag&ZOUTFILE_FLAG_VIDEO)
{
m_videtrackid = gf_isom_new_track(p_file, 0, GF_ISOM_MEDIA_VISUAL, 1000);
gf_isom_set_track_enabled(p_file, m_videtrackid, 1);
}
else
{
return false;
}
p_videosample = gf_isom_sample_new();
p_videosample->data = (char*)malloc(1024 * 1024);
p_hevc_config = gf_odf_hevc_cfg_new();
p_hevc_config->nal_unit_size = 32 / 8;
//gf_isom_avc_config_new(p_file, m_videtrackid, p_config, NULL, NULL, &i_videodescidx);
//gf_isom_set_visual_info(p_file, m_videtrackid, i_videodescidx, width, height);
//初始化配置
gf_isom_hevc_config_new(p_file, m_videtrackid, p_hevc_config, NULL, NULL, &i_videodescidx);
gf_isom_set_nalu_extract_mode(p_file, m_videtrackid, GF_ISOM_NALU_EXTRACT_INSPECT);
gf_isom_set_cts_packing(p_file, m_videtrackid, GF_TRUE);
HEVCState hevc = { 0 };
m_slotsps = { 0 };
m_slotpps = { 0 };
m_slotvps = { 0 };
m_spss = { 0 };
m_ppss = { 0 };
m_vpss = { 0 };
p_hevc_config->configurationVersion = 1;
//Config vps
int idx = gf_media_hevc_read_vps((char*)vps, vpslen, &hevc);
hevc.vps[idx].crc = gf_crc_32((char*)vps, vpslen);
p_hevc_config->avgFrameRate = hevc.vps[idx].rates[0].avg_pic_rate;
p_hevc_config->constantFrameRate = hevc.vps[idx].rates[0].constand_pic_rate_idc;
p_hevc_config->numTemporalLayers = hevc.vps[idx].max_sub_layers;
p_hevc_config->temporalIdNested = hevc.vps[idx].temporal_id_nesting;
m_vpss.nalus = gf_list_new();
gf_list_add(p_hevc_config->param_array, &m_vpss);
m_vpss.array_completeness = 1;
m_vpss.type = GF_HEVC_NALU_VID_PARAM;// naltype = VPS
m_slotvps.id = idx;
m_slotvps.size = vpslen;
m_slotvps.data = (char*)malloc(vpslen);
memcpy(m_slotvps.data, vps, vpslen);
gf_list_add(m_vpss.nalus, &m_slotvps);
//Config sps
idx = gf_media_hevc_read_sps((char*)sps, spslen, &hevc);
hevc.sps[idx].crc = gf_crc_32((char*)sps, spslen);
p_hevc_config->profile_space = hevc.sps[idx].ptl.profile_space;
p_hevc_config->tier_flag = hevc.sps[idx].ptl.tier_flag;
p_hevc_config->profile_idc = hevc.sps[idx].ptl.profile_idc;
m_spss.nalus = gf_list_new();
gf_list_add(p_hevc_config->param_array, &m_spss);
m_spss.array_completeness = 1;
m_spss.type = GF_HEVC_NALU_SEQ_PARAM;// naltype = SPS
m_slotsps.id = idx;
m_slotsps.size = spslen;
m_slotsps.data = (char*)malloc(spslen);
memcpy(m_slotsps.data, sps, spslen);
gf_list_add(m_spss.nalus, &m_slotsps);
int act_width = hevc.sps[idx].width;
int act_height = hevc.sps[idx].height;
//Config pps
idx = gf_media_hevc_read_pps((char*)pps, ppslen, &hevc);
hevc.pps[idx].crc = gf_crc_32((char*)pps, ppslen);
m_ppss.nalus = gf_list_new();
gf_list_add(p_hevc_config->param_array, &m_ppss);
m_ppss.array_completeness = 1;
m_ppss.type = GF_HEVC_NALU_PIC_PARAM;// naltype = PPS
m_slotpps.id = idx;
m_slotpps.size = ppslen;
m_slotpps.data = (char*)malloc(ppslen);
memcpy(m_slotpps.data, pps, ppslen);
gf_list_add(m_ppss.nalus, &m_slotpps);
gf_isom_set_visual_info(p_file, m_videtrackid, i_videodescidx, act_width, act_height);
gf_isom_hevc_config_update(p_file, m_videtrackid, 1, p_hevc_config);
//銷毀申請的內存資源
gf_list_del(m_vpss.nalus);
gf_list_del(m_spss.nalus);
gf_list_del(m_ppss.nalus);
free(m_slotvps.data);
free(m_slotsps.data);
free(m_slotpps.data);
p_hevc_config->param_array = NULL;
return true;
}
結合寫MP4篇我們不難看出，MP4BOX對H265專門封裝了個結構函數gf_isom_hevc_config_new（）用以對H265參數的設置，設置方法和H264相似，不過對H265處理更加細致，MP4BOX將VPS,SPS,PPS的各個參數拆分出來進行賦值，通過gf_isom_hevc_config_update寫入解碼參數信息。