做语音识别这块的呢，国内领先的有科大讯飞，BAT这几家公司，鉴于使用科大讯飞的接口需要付费，腾讯云的语音识别申请了几天也没给通过，比较了一下阿里和百度的，个人觉得阿里云的好用一些，这篇博客来讲讲怎么讲阿里云的语音识别应用到项目中。
首先是一些链接
阿里云语音识别官网：https://help.aliyun.com/document_detail/30416.html
项目中的应用
要实现的功能：前端浏览器录取用户说话的录音，然后转换为文字显示在浏览器。例如回答问题后，回答的答案显示在下方
流程分析：实现这个功能流程很简单就是浏览器收集到用户的语音输入流后，发送给后台，后台继续将数据发送到阿里云的服务器端，进行语音转文字，完成后将返回的结果进行处理，再返回到前台。
本例中主要分为两个模块，（1）前台获得麦克风的权限进行录音；（2）录制完成将数据发送到后台进行语音转文字的处理。
（1）前台录音
前台的功能是，点击录音，获取浏览器麦克风权限后，开始录音。点击转换按钮，停止录音，将数据发送到后台进行转换，转换后的结果显示在下方的文本域中，同时出现audio元素标签和文件下载链接，可回放录音和保存文件到本地。
录音相关的js文件来源其他大神，本人将其代码进行部分修改以满足需求。
JSP页面的代码
<%@ page language="java" contentType="text/html; charset=utf-8" pageEncoding="utf-8"%>
<% String path = request.getContextPath(); String basePath = request.getScheme() + "://" + request.getServerName() + ":" + request.getServerPort() + path + "/"; %>
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<title>Insert title here</title>
</head>
<body>
<form id="questions">
<div><h1>回答问题</h1></div>
<input type="hidden" name="records[0].question" value="AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA">
<div><h3>问题一：AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA</h3></div>
<div>
<button onclick="startRecording(this)" >录音</button>
<button onclick="uploadAudio(this,1)" disabled>转换</button>
<div id="recordingslist1"></div>
</div>
<textarea id="audioText1" name="records[0].answer" rows="3" cols="50" style="font-size:18px"></textarea>
<input type="hidden" name="records[1].question" value="BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB">
<div><h3>问题二：BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB</h3></div>
<div>
<button onclick="startRecording(this)" >录音</button>
<button onclick="uploadAudio(this,2)" disabled>转换</button>
<div id="recordingslist2"></div>
</div>
<textarea id="audioText2" name="records[1].answer" rows="3" cols="50" style="font-size:18px"></textarea>
<input type="hidden" name="records[2].question" value="CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC">
<div><h3>问题三：CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC</h3></div>
<div>
<button onclick="startRecording(this)" >录音</button>
<button onclick="uploadAudio(this,3)" disabled>转换</button>
<div id="recordingslist3"></div>
</div>
<textarea id="audioText3" name="records[2].answer" rows="3" cols="50" style="font-size:18px"></textarea>
<br>
<input type="button" onclick="save()" value="保存录音"/>
</form>
<a href="<%=path %>/audio/getAllRecord">查看记录详情</a>
<form action="<%=path %>/audio/getaudio" method="post" enctype="multipart/form-data">
<h2>文件上传</h2> 文件:<input type="file" name="audioData"/><br/><br/>
<input type="submit" value="上传"/>
</form>
<script type="text/javascript" src="resources/js/HZRecorder.js"></script>
<script type="text/javascript" src="resources/js/jquery-1.11.1.js"></script>
<script> function save() { $.ajax({ type: "POST", dataType: "json", url: "<%=path %>/audio/saveRecord", data: $('#questions').serialize(), success: function (result) { if (result) { alert("添加成功"); }else { alert("添加失败"); } }, error : function() { alert("异常！"); } }); } var recorder; var audio = document.querySelector('audio'); // 开始录音 function startRecording(button) { button.disabled = true; button.nextElementSibling.disabled = false; HZRecorder.get(function (rec) { recorder = rec; recorder.start(); }); } // 播放录音 function playRecording() { recorder.play(audio); } // 转换录音 function uploadAudio(button,num) { button.disabled = true; button.previousElementSibling.disabled = false; recorder.stop(); recorder.upload("<%=path %>/audio/getaudio", num); createDownloadLink(num); } // 创建下载链接 function createDownloadLink(num) { var blob = recorder.getBlob(); var url = URL.createObjectURL(blob); var div = document.createElement('div'); var au = document.createElement('audio'); var hf = document.createElement('a'); var record = "recordingslist"+num; au.controls = true; au.src = url; hf.href = url; hf.download = new Date().toISOString() + '.wav'; hf.innerHTML = hf.download; div.appendChild(au); div.appendChild(hf); document.getElementById(record).appendChild(div); } </script>
</body>
</html>
引用的js文件 HZRecorder.js
(function (window) { //兼容
window.URL = window.URL || window.webkitURL; navigator.getUserMedia = navigator.getUserMedia || navigator.webkitGetUserMedia || navigator.mozGetUserMedia || navigator.msGetUserMedia; var HZRecorder = function (stream, config) { config = config || {}; config.sampleBits = config.sampleBits || 16; //采样数位 8, 16
config.sampleRate = config.sampleRate || (16000); //采样率(1/6 44100)
var context = new (window.webkitAudioContext || window.AudioContext)(); var audioInput = context.createMediaStreamSource(stream); var createScript = context.createScriptProcessor || context.createJavaScriptNode; var recorder = createScript.apply(context, [4096, 1, 1]); var audioData = { size: 0 //录音文件长度
, buffer: [] //录音缓存
, inputSampleRate: context.sampleRate //输入采样率
, inputSampleBits: 16 //输入采样数位 8, 16
, outputSampleRate: config.sampleRate //输出采样率
, oututSampleBits: config.sampleBits //输出采样数位 8, 16
, input: function (data) { this.buffer.push(new Float32Array(data)); this.size += data.length; } , compress: function () { //合并压缩
//合并
var data = new Float32Array(this.size); var offset = 0; for (var i = 0; i < this.buffer.length; i++) { data.set(this.buffer[i], offset); offset += this.buffer[i].length; } //压缩
var compression = parseInt(this.inputSampleRate / this.outputSampleRate); var length = data.length / compression; var result = new Float32Array(length); var index = 0, j = 0; while (index < length) { result[index] = data[j]; j += compression; index++; } return result; } , encodeWAV: function () { var sampleRate = Math.min(this.inputSampleRate, this.outputSampleRate); var sampleBits = Math.min(this.inputSampleBits, this.oututSampleBits); var bytes = this.compress(); var dataLength = bytes.length * (sampleBits / 8); var buffer = new ArrayBuffer(44 + dataLength); var data = new DataView(buffer); var channelCount = 1;//单声道
var offset = 0; var writeString = function (str) { for (var i = 0; i < str.length; i++) { data.setUint8(offset + i, str.charCodeAt(i)); } } // 资源交换文件标识符
writeString('RIFF'); offset += 4; // 下个地址开始到文件尾总字节数,即文件大小-8
data.setUint32(offset, 36 + dataLength, true); offset += 4; // WAV文件标志
writeString('WAVE'); offset += 4; // 波形格式标志
writeString('fmt '); offset += 4; // 过滤字节,一般为 0x10 = 16
data.setUint32(offset, 16, true); offset += 4; // 格式类别 (PCM形式采样数据)
data.setUint16(offset, 1, true); offset += 2; // 通道数
data.setUint16(offset, channelCount, true); offset += 2; // 采样率,每秒样本数,表示每个通道的播放速度
data.setUint32(offset, sampleRate, true); offset += 4; // 波形数据传输率 (每秒平均字节数) 单声道×每秒数据位数×每样本数据位/8
data.setUint32(offset, channelCount * sampleRate * (sampleBits / 8), true); offset += 4; // 快数据调整数 采样一次占用字节数 单声道×每样本的数据位数/8
data.setUint16(offset, channelCount * (sampleBits / 8), true); offset += 2; // 每样本数据位数
data.setUint16(offset, sampleBits, true); offset += 2; // 数据标识符
writeString('data'); offset += 4; // 采样数据总数,即数据总大小-44
data.setUint32(offset, dataLength, true); offset += 4; // 写入采样数据
if (sampleBits === 8) { for (var i = 0; i < bytes.length; i++, offset++) { var s = Math.max(-1, Math.min(1, bytes[i])); var val = s < 0 ? s * 0x8000 : s * 0x7FFF; val = parseInt(255 / (65535 / (val + 32768))); data.setInt8(offset, val, true); } } else { for (var i = 0; i < bytes.length; i++, offset += 2) { var s = Math.max(-1, Math.min(1, bytes[i])); data.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true); } } return new Blob([data], { type: 'audio/wav' }); } }; //开始录音
this.start = function () { audioInput.connect(recorder); recorder.connect(context.destination); } //停止ֹ
this.stop = function () { recorder.disconnect(); } //获取音频文件
this.getBlob = function () { this.stop(); return audioData.encodeWAV(); } //回放
this.play = function (audio) { audio.src = window.URL.createObjectURL(this.getBlob()); } //转换
this.upload = function (url, num) { var id = "audioText"+num; var fd = new FormData(); fd.append("audioData", this.getBlob()); var xhr = new XMLHttpRequest(); xhr.open("POST", url); xhr.send(fd); xhr.onreadystatechange = function () { if (xhr.readyState == 4 && xhr.status == 200) { document.getElementById(id).value += xhr.responseText; } }; } //音频采集
recorder.onaudioprocess = function (e) { audioData.input(e.inputBuffer.getChannelData(0)); //record(e.inputBuffer.getChannelData(0));
} }; //抛出异常
HZRecorder.throwError = function (message) { alert(message); throw new function () { this.toString = function () { return message; } } } //是否支持录音
HZRecorder.canRecording = (navigator.getUserMedia != null); //获取录音机
HZRecorder.get = function (callback, config) { if (callback) { if (navigator.getUserMedia) { navigator.getUserMedia( { audio: true } //只启用音频
, function (stream) { var rec = new HZRecorder(stream, config); callback(rec); } , function (error) { switch (error.code || error.name) { case 'PERMISSION_DENIED': case 'PermissionDeniedError': HZRecorder.throwError('用户拒绝提供信息。'); break; case 'NOT_SUPPORTED_ERROR': case 'NotSupportedError': HZRecorder.throwError('浏览器不支持硬件设备。'); break; case 'MANDATORY_UNSATISFIED_ERROR': case 'MandatoryUnsatisfiedError': HZRecorder.throwError('无法发现指定的硬件设备。'); break; default: HZRecorder.throwError('无法打开麦克风。异常信息:' + (error.name)); break; } }); } else { HZRecorder.throwErr('当前浏览器不支持录音功能。'); return; } } } window.HZRecorder = HZRecorder; })(window);
页面中也没有多少要注意的问题。注意的是每一个问题上方都有一个隐藏域，里面的值是问题的内容，这样做是为了将问题和答案一起存放在数据库中，因为form只能提交input中的内容，所以想出了这个办法，不知道还有没有其他方式。
（2）后台转换（SSM框架）
录音文件流以文件上传的方式传到后台（这里不必将文件流转换成音频文件，因为阿里云的实时语音识别Demo中是将文件转化为InputStream，再进行转文字，可直接获得MultipartFile的InputStream传给语音转换）
录音文件时长超过13分钟左右，在转换的过程中，通信会被关闭（即录音20分钟，只会转换10分钟的内容，本人目前不清楚具体的原因）暂时的解决办法是将上传的录音文件分割成两部分，分别执行转换的方法。
RecordController.java
package cn.com.sysystem.controller; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.util.List; import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.multipart.MultipartFile; import cn.com.sysystem.base.util.RealtimeAsr; import cn.com.sysystem.entity.RecordEntity; import cn.com.sysystem.model.RecordModel; import cn.com.sysystem.service.RecordService; @Controller @RequestMapping("/audio") public class RecordController{ @Resource RecordService recordService; @ResponseBody @RequestMapping(value = "/getaudio" ,produces = "application/json; charset=utf-8") public String getaudio(MultipartFile audioData,HttpServletRequest request) throws Exception { StringBuffer sb = new StringBuffer(2000); if (audioData != null) {
byte[] bytes = audioData.getBytes(); // 当录音文件过大时，将文件分割成两段
if (bytes.length < 20000000) { InputStream inputStream = audioData.getInputStream(); sb.append(getText(inputStream)); } else { byte[] tmp1 = new byte[bytes.length/2]; byte[] tmp2 = new byte[bytes.length-tmp1.length]; System.arraycopy(bytes, 0, tmp1, 0, tmp1.length); System.arraycopy(bytes, tmp1.length, tmp2, 0, tmp2.length); InputStream input1 = new ByteArrayInputStream(tmp1); InputStream input2 = new ByteArrayInputStream(tmp2); sb.append(getText(input1)); sb.append(getText(input2)); }
}else { return "文件上传失败"; } return sb.toString(); } @ResponseBody @RequestMapping(value = "/saveRecord") public boolean saveRecord(RecordModel recordlist) throws Exception { boolean flag = true; List<RecordEntity> records = recordlist.getRecords(); for (RecordEntity recordEntity : records) { int row = recordService.saveRecord(recordEntity); if (row < 0) { flag = false; } } return flag; } @RequestMapping(value = "/getAllRecord") public String getAllRecord(HttpServletRequest request) throws Exception { List<RecordEntity> allRecord = recordService.getAllRecord(); request.setAttribute("recordList", allRecord); return "showrecord"; } /** * 将语音输入流转换为文字 * @param input * @return
*/
private synchronized String getText(InputStream input) { StringBuilder finaltext = new StringBuilder(2000); List<String> results = null; RealtimeAsr realtimeAsr = new RealtimeAsr(); results = realtimeAsr.AliAudio2Text(input); // 去除集合中含有status_code = 0的元素
results.removeIf(p -> p.indexOf("\"status_code\":0") == -1); for (String str : results) { String text = ""; String[] split = str.split(","); text = split[split.length-1]; text = text.substring(8, text.length()-2); finaltext.append(text); } // 清空集合
results.clear(); return finaltext.toString(); } }
Controller中getText方法，创建RealtimeAsr类的对象，调用AliAudio2Text方法获得转换结果，RealtimeAsr类如下：
RealtimeAsr.java
package cn.com.sysystem.base.util; import java.io.File; import java.io.FileInputStream; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONPath; import com.alibaba.idst.nls.realtime.NlsClient; import com.alibaba.idst.nls.realtime.NlsFuture; import com.alibaba.idst.nls.realtime.event.NlsEvent; import com.alibaba.idst.nls.realtime.event.NlsListener; import com.alibaba.idst.nls.realtime.protocol.NlsRequest; import com.alibaba.idst.nls.realtime.protocol.NlsResponse; /** * asr 示例 * */
public class RealtimeAsr implements NlsListener { protected NlsClient client = new NlsClient(); protected static final String asrSC = "pcm"; static Logger logger = LoggerFactory.getLogger(RealtimeAsr.class); public String filePath = ""; //public String appKey = "nls-service-shurufa16khz"; //社交聊天领域
public String appKey = "nls-service-multi-domain"; //短视频，视频直播领域，教育，娱乐，文学，法律，财经等 //public String appKey = "nls-service-en"; //英语
protected String ak_id = ""; //阿里云的AccessKeyID 和 AccessKeySecret 自行去注册账户，这里就不提供了
protected String ak_secret = ""; protected String url = "https://nlsapi.aliyun.com/asr/custom/vocabs"; public static List<String> results = new ArrayList<String>(5000); public RealtimeAsr() { } public void shutDown() { logger.debug("close NLS client manually!"); client.close(); logger.debug("demo done"); } public void start() { logger.debug("init Nls client..."); client.init(); } public void process() { logger.debug("open audio file..."); FileInputStream fis = null; try { File file = new File(filePath); fis = new FileInputStream(file); } catch (Exception e) { logger.error("fail to open file", e); } if (fis != null) { logger.debug("create NLS future"); process(fis); logger.debug("calling NLS service end"); } } public void process(InputStream ins) { try { NlsRequest req = buildRequest(); NlsFuture future = client.createNlsFuture(req, this); logger.debug("call NLS service"); byte[] b = new byte[5000]; int len = 0; while ((len = ins.read(b)) > 0) { future.sendVoice(b, 0, len); //Thread.sleep(200);
} logger.debug("send finish signal!"); future.sendFinishSignal(); logger.debug("main thread enter waiting ."); future.await(100000); } catch (Exception e) { e.printStackTrace(); } } protected NlsRequest buildRequest() { NlsRequest req = new NlsRequest(); req.setAppkey(appKey); req.setFormat(asrSC); req.setResponseMode("streaming"); req.setSampleRate(16000); String body="{\n"
+ " \"global_weight\": 1,\n"
+ " \"words\": [\n"
+ " \"SpringMVC\",\n"
+ " \"Mybatis\",\n"
+ " \"Hibernate\"\n"
+ " ],\n"
+ " \"word_weights\": {\n"
+ " \"spring\": 2\n"
+ " }\n"
+ " }"; //create
String result=HttpUtil.sendPost(url,body,ak_id,ak_secret); String vocabId=(String)JSONPath.read(result,"vocabulary_id"); //update
result=HttpUtil.sendPut(url+"/"+vocabId,body,ak_id,ak_secret); req.setVocabularyId(vocabId); // 设置关键词库ID 使用时请修改为自定义的词库ID // req.setKeyWordListId("c1391f1c1f1b4002936893c6d97592f3");
req.authorize(ak_id, ak_secret); return req; } @Override public void onMessageReceived(NlsEvent e) { NlsResponse response = e.getResponse(); response.getFinish(); if (response.result != null) { String tmptext = response.getResult().toString(); results.add(tmptext); //logger.debug("status code = {},get finish is {},get recognize result: {}", response.getStatusCode(), // response.getFinish(), response.getResult());
if (response.getQuality() != null) { logger.info("Sentence {} is over. Get ended sentence recognize result: {}, voice quality is {}", response.result.getSentence_id(), response.getResult(), JSON.toJSONString(response.getQuality())); } } else { logger.info(JSON.toJSONString(response)); } } @Override public void onOperationFailed(NlsEvent e) { logger.error("status code is {}, on operation failed: {}", e.getResponse().getStatusCode(), e.getErrorMessage()); } @Override public void onChannelClosed(NlsEvent e) { logger.debug("on websocket closed."); } /** * @param inputStream */
public List<String> AliAudio2Text(InputStream inputStream) { RealtimeAsr lun = new RealtimeAsr(); lun.start(); lun.process(inputStream); lun.shutDown(); return results; } }
注意的地方
1、@RequestMapping(value = "/getaudio" ,produces = "application/json; charset=utf-8")
produces = "application/json; charset=utf-8" 保证Controller在return中文时乱码的问题。
2、StringBuffer sb = new StringBuffer(2000);
因为要经常拼接字符串，所以StringBuffer的效率会比String高些，另外还有一个小窍门，就是在new StringBuffer时指定大小，若不指定且内容较长时，会频繁的扩容，影响性能（具体也不知道能提高多少，提高一点是一点吧，同时集合中的list和map也是一样的道理）
3、synchronized
转换的方法中加入synchronized关键字保证线程安全的目的是，当上一段录音时长较长时，转换需要一定的时间（20分钟的音频，转换过程3分钟左右），若立即开始第二段录音，且时间较短，若不加锁，第二段转换的文本中显示的是第一段的内容。
4、results.removeIf(p -> p.indexOf("\"status_code\":0") == -1);
这里用到了Java8的Lambda表达式，不明白的同学可以自行了解一下，很好用。
5、
语音转换收集到的信息如下：（例如说ABCD）
{"sentence_id":1,"begin_time":280,"current_time":1670,"end_time":-1,"status_code":1,"text":"A"}
{"sentence_id":1,"begin_time":280,"current_time":1670,"end_time":1793,"status_code":0,"text":"A B C D"}
其中status_code = 1 表示的是转换的中间状态，status_code = 0表示语音转换完成。所以我们要从集合中筛选出status_code = 0的所有字符串，并截取text的值。