Android 音视频采集那些事

{"type":"doc","content":[{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"音视频采集","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在整个音视频处理的过程中，位于发送端的音视频采集工作无疑是整个音视频链路的开始。在Android或者IOS上都有相关的硬件设备——Camera和麦克风作为输入源。本章我们来分析如何在Android上通过Camera以及录音设备采集数据。本章可结合之前发布的文章","attrs":{}},{"type":"link","attrs":{"href":"https://xie.infoq.cn/article/4a1a0966b08fea3b51808f2e8","title":"","type":null},"content":[{"type":"text","text":"Android 音视频 - MediaCodec 编解码音视频","attrs":{}}]},{"type":"text","text":"做一个完整的Demo","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"Camera","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在Android上的图片/视频采集设备无疑就是Camera了，在Android SDK API21之前的版本只能使用Camera1 ，在 API 21之后Camera1已经被标记为","attrs":{}},{"type":"text","marks":[{"type":"del","attrs":{}}],"text":"Deprecated","attrs":{}},{"type":"text","text":" ,Google 推荐使用Camera2,下面我们来分别看一下","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"Camera1","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"我们先来看一下Camera1体系的部分类图","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/cb/cb3061c2d0e79288d203eb7ab18dc1e7.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Camera类是Camera1体系的核心类，该类还有好多内部类,如上图，","attrs":{}}]},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Camera.CameraInfo 类表达Camera的前后(facing)和旋转(orientation)等Camera相关的信息。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Camera.Parameters 类是Camera相关的参数设置比如设置预览Size以及设置旋转角度等。","attrs":{}}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Camera类拥有打开Camera、设置参数、设置预览等API,下面我们来看使用Camera API打开系统照相机的流程。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/94/945c30a38795c3a37a869b55132f54f7.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"numberedlist","attrs":{"start":1,"normalizeStart":1},"content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":1,"align":null,"origin":null},"content":[{"type":"text","text":"在开启Camera之前先释放Camera,这一步的目的是重置Camera的状态","attrs":{}}]}]}]},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"重置Camera的previewCallback为null","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"调用Camera的release释放","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"把Camera对象设置为null","attrs":{}}]}],"attrs":{}},{"type":"codeblock","attrs":{"lang":"kotlin"},"content":[{"type":"text","text":"/**\n*释放Camera\n*/\n private fun releaseCamera() {\n \t//重置previewCallback为空\n cameraInstance!!.setPreviewCallback(null)\n cameraInstance!!.release()\n cameraInstance = null\n }\n","attrs":{}}]},{"type":"numberedlist","attrs":{"start":2,"normalizeStart":2},"content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":2,"align":null,"origin":null},"content":[{"type":"text","text":"获取Camera的Id","attrs":{}}]}]}]},{"type":"codeblock","attrs":{"lang":"kotlin"},"content":[{"type":"text","text":"/**\n*获取Camera Id\n*/\n private fun getCurrentCameraId(): Int {\n val cameraInfo = Camera.CameraInfo()\n //遍历所有的Camera id,比较CameraInfo facing \n for (id in 0 until Camera.getNumberOfCameras()) {\n Camera.getCameraInfo(id, cameraInfo)\n if (cameraInfo.facing == cameraFacing) {\n return id\n }\n }\n return 0\n }\n","attrs":{}}]},{"type":"numberedlist","attrs":{"start":3,"normalizeStart":3},"content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":3,"align":null,"origin":null},"content":[{"type":"text","text":"打开Camera获取Camera对象","attrs":{}}]}]}]},{"type":"codeblock","attrs":{"lang":"kotlin"},"content":[{"type":"text","text":"/**\n*获取Camera 实例\n*/\n private fun getCameraInstance(id: Int): Camera {\n return try {\n //调用Camera的open函数获取Camera的实例\n Camera.open(id)\n } catch (e: Exception) {\n throw IllegalAccessError(\"Camera not found\")\n }\n }","attrs":{}}]},{"type":"numberedlist","attrs":{"start":4,"normalizeStart":4},"content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":4,"align":null,"origin":null},"content":[{"type":"text","text":"设置Camera的相关参数","attrs":{}}]}]}]},{"type":"codeblock","attrs":{"lang":"kotlin"},"content":[{"type":"text","text":"//[3]设置参数\nval parameters = cameraInstance!!.parameters\n\n if (parameters.supportedFocusModes.contains(Camera.Parameters.FOCUS_MODE_CONTINUOUS_PICTURE)) {\n parameters.focusMode = Camera.Parameters.FOCUS_MODE_CONTINUOUS_PICTURE\n }\n cameraInstance!!.parameters = parameters","attrs":{}}]},{"type":"numberedlist","attrs":{"start":5,"normalizeStart":5},"content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":5,"align":null,"origin":null},"content":[{"type":"text","text":"设置previewDisplay","attrs":{}}]}]}]},{"type":"codeblock","attrs":{"lang":"kotlin"},"content":[{"type":"text","text":"//【4】 调用Camera API 设置预览Surface\n surfaceHolder?.let { cameraInstance!!.setPreviewDisplay(it) }","attrs":{}}]},{"type":"numberedlist","attrs":{"start":6,"normalizeStart":6},"content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":6,"align":null,"origin":null},"content":[{"type":"text","text":"设置预览回调","attrs":{}}]}]}]},{"type":"codeblock","attrs":{"lang":"kotlin"},"content":[{"type":"text","text":"//【5】 调用Camera API设置预览回调\n cameraInstance!!.setPreviewCallback { data, camera ->\n if (data == null || camera == null) {\n return@setPreviewCallback\n }\n val size = camera.parameters.previewSize\n onPreviewFrame?.invoke(data, size.width, size.height)\n }","attrs":{}}]},{"type":"numberedlist","attrs":{"start":7,"normalizeStart":7},"content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":7,"align":null,"origin":null},"content":[{"type":"text","text":"开启预览","attrs":{}}]}]}]},{"type":"codeblock","attrs":{"lang":"kotlin"},"content":[{"type":"text","text":"//【6】 调用Camera API开启预览\n cameraInstance!!.startPreview()","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"上面代码中的【3】【4】【5】【6】都是调用Camera类的API来完成，","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"经过上面的流程之后，Camera的预览会显示在传入的Surface上，并且在Camera停止前会一直回调函数","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"onPreviewFrame(byte[] data,Camera camera)","attrs":{}}],"attrs":{}},{"type":"text","text":"，其中byte[] data中存储的就是实时的YUV图像数据。","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"byte[] data的格式是YUV格式中的NV21","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"YUV图像格式","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"色彩空间","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"这里我们只讲常用到的两种色彩空间。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"RGB","attrs":{}},{"type":"text","text":"RGB的颜色模式应该是我们最熟悉的一种，在现在的电子设备中应用广泛。通过R G B三种基础色，可以混合出所有的颜色。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"YUV","attrs":{}},{"type":"text","text":"这里着重讲一下YUV，这种色彩空间并不是我们熟悉的。这是一种亮度与色度分离的色彩格式。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"早期的电视都是黑白的，即只有亮度值，即Y。有了彩色电视以后，加入了UV两种色度，形成现在的YUV，也叫YCbCr。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Y：亮度，就是灰度值。除了表示亮度信号外，还含有较多的绿色通道量。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"U：蓝色通道与亮度的差值。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"V：红色通道与亮度的差值。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"采用YUV有什么优势呢？","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"人眼对亮度敏感，对色度不敏感，因此减少部分UV的数据量，人眼却无法感知出来，这样可以通过压缩UV的分辨率，在不影响观感的前提下，减小视频的体积。","attrs":{}}]},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"RGB和YUV的换算","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":"br"}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Y = 0.299R ＋ 0.587G ＋ 0.114B","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":"br"}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"U = －0.147R － 0.289G ＋ 0.436B","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":"br"}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"V = 0.615R － 0.515G － 0.100B","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":"br"}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"——————————————————","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":"br"}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"R = Y ＋ 1.14V","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":"br"}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"G = Y － 0.39U － 0.58V","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":"br"}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"B = Y ＋ 2.03U","attrs":{}}]}],"attrs":{}},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"YUV格式","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"YUV存储方式分为两大类：planar 和 packed。","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"planar：先存储所有Y，紧接着存储所有U，最后是V；","attrs":{}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/83/8377a84eaf7785674202bc4128d659e1.webp","alt":null,"title":null,"style":null,"href":null,"fromPaste":false,"pastePass":false}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"packed：每个像素点的 Y、U、V 连续交叉存储。","attrs":{}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/f3/f3fe4e619790f2a57d9ca2905ba7fb7e.webp","alt":null,"title":null,"style":null,"href":null,"fromPaste":false,"pastePass":false}}]}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"pakced存储方式已经非常少用，大部分视频都是采用planar存储方式。","attrs":{}}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"对于planar存储方式，通过省略一些色度信息，即亮度共用一些色度信息，进而节省存储空间。因此，planar又区分了以下几种格式： ","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"YUV444、 YUV422、YUV420。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"YUV 4:4:4采样，每一个Y对应一组UV分量。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/4d/4df4662e1d245d0d5a5c5fbb744cc325.webp","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"YUV 4:2:2采样，每两个Y共用一组UV分量。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/3d/3d94171885704fc243c33f89c6b2e27b.webp","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"YUV 4:2:0采样，每四个Y共用一组UV分量。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/36/36152604e68094a92505e28b0b7672d3.webp","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"其中，最常用的就是","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"YUV420","attrs":{}},{"type":"text","text":"。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"YUV420格式存储方式又分两种类型","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"YUV420P：三平面存储。数据组成为YYYYYYYYUUVV（如I420）或YYYYYYYYVVUU（如YV12）。","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"YUV420SP：两平面存储。分为两种类型YYYYYYYYUVUV（如NV12）或YYYYYYYYVUVU（如NV21）","attrs":{}}]}]}],"attrs":{}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"Camera2","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在Andorid SDK API 21之后呢，Google就推荐使用Camera2体系来管理设备，Camera2还是与Camera1有很大的不同的。一样的，我们先来看一下Camera2体系的部分类图","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/07/0746efa58c37f400b873a19ac2a35fe5.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Camera2要比Camera1复杂的多，CameraManager CameraCaptureSession是Camera2体系的核心类，CameraManager 用来管理摄像头的打开和关闭 Camera2 引入了CameraCaptureSession来管理拍摄会话。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"我们下面来看一下更详细的流程图","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/3a/3a3eada3f6ed3d9953e7b028c9e825eb.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"numberedlist","attrs":{"start":null,"normalizeStart":1},"content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":1,"align":null,"origin":null},"content":[{"type":"text","text":"在开启Camera之前先释放Camera,这一步的目的是重置Camera的状态","attrs":{}}]}]}]},{"type":"codeblock","attrs":{"lang":"kotlin"},"content":[{"type":"text","text":"private fun releaseCamera() {\n imageReader?.close()\n cameraInstance?.close()\n captureSession?.close()\n imageReader = null\n cameraInstance = null\n captureSession = null\n }","attrs":{}}]},{"type":"numberedlist","attrs":{"start":2,"normalizeStart":2},"content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":2,"align":null,"origin":null},"content":[{"type":"text","text":"获取Camera的Id","attrs":{}}]}]}]},{"type":"codeblock","attrs":{"lang":"kotlin"},"content":[{"type":"text","text":"/**\n *【1】 获取Camera Id\n */\n private fun getCameraId(facing: Int): String? {\n return cameraManager.cameraIdList.find { id ->\n cameraManager.getCameraCharacteristics(id).get(CameraCharacteristics.LENS_FACING) == facing\n }\n }\n","attrs":{}}]},{"type":"numberedlist","attrs":{"start":3,"normalizeStart":3},"content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":3,"align":null,"origin":null},"content":[{"type":"text","text":"打开Camera","attrs":{}}]}]}]},{"type":"codeblock","attrs":{"lang":"kotlin"},"content":[{"type":"text","text":"try {\n //【2】打开Camera，传入的 CameraDeviceCallback()是摄像机设备状态回调\n cameraManager.openCamera(cameraId, CameraDeviceCallback(), null)\n } catch (e: CameraAccessException) {\n Log.e(TAG, \"Opening camera (ID: $cameraId) failed.\")\n }\n\n//设备状态回调\n private inner class CameraDeviceCallback : CameraDevice.StateCallback() {\n override fun onOpened(camera: CameraDevice) {\n cameraInstance = camera\n //【3】开启拍摄会话\n startCaptureSession()\n }\n\n override fun onDisconnected(camera: CameraDevice) {\n camera.close()\n cameraInstance = null\n }\n\n override fun onError(camera: CameraDevice, error: Int) {\n camera.close()\n cameraInstance = null\n }\n }","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"numberedlist","attrs":{"start":4,"normalizeStart":4},"content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":4,"align":null,"origin":null},"content":[{"type":"text","text":"开启拍摄会话","attrs":{}}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"kotlin"},"content":[{"type":"text","text":" //【3】开启拍摄会话\n private fun startCaptureSession() {\n val size = chooseOptimalSize()\n //创建ImageRender并设置回调\n imageReader =\n ImageReader.newInstance(size.width, size.height, ImageFormat.YUV_420_888, 2).apply {\n setOnImageAvailableListener({ reader ->\n val image = reader?.acquireNextImage() ?: return@setOnImageAvailableListener\n onPreviewFrame?.invoke(image.generateNV21Data(), image.width, image.height)\n image.close()\n }, null)\n }\n\n try {\n if (surfaceHolder == null) {\n //设置ImageRender的surface给cameraInstance，以便后面预览的时候数据呈现到ImageRender的surface，从而触发ImageRender的回调\n cameraInstance?.createCaptureSession(\n listOf(imageReader!!.surface),\n //【4】CaptureStateCallback是CameraCaptureSession的内部类，是摄像机会话状态的回调\n CaptureStateCallback(),\n null\n )\n } else {\n cameraInstance?.createCaptureSession(\n listOf(imageReader!!.surface,\n surfaceHolder!!.surface),\n CaptureStateCallback(),\n null\n )\n }\n\n } catch (e: CameraAccessException) {\n Log.e(TAG, \"Failed to start camera session\")\n }\n }\n\n //摄像机会话状态的回调\n private inner class CaptureStateCallback : CameraCaptureSession.StateCallback() {\n override fun onConfigureFailed(session: CameraCaptureSession) {\n Log.e(TAG, \"Failed to configure capture session.\")\n }\n //摄像机配置完成\n override fun onConfigured(session: CameraCaptureSession) {\n cameraInstance ?: return\n captureSession = session\n //设置预览CaptureRequest.Builder\n val builder = cameraInstance!!.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW)\n builder.addTarget(imageReader!!.surface)\n surfaceHolder?.let {\n builder.addTarget(it.surface)\n }\n\n try {\n //开启会话\n session.setRepeatingRequest(builder.build(), null, null)\n } catch (e: CameraAccessException) {\n Log.e(TAG, \"Failed to start camera preview because it couldn't access camera\", e)\n } catch (e: IllegalStateException) {\n Log.e(TAG, \"Failed to start camera preview.\", e)\n }\n }\n }\n","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PS","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"ImageRender可以直接访问呈现在Surface上得图像数据，ImageRender的工作原理是创建实例并设置回调，这个回调会在ImageRender所关联的Surface上的图像可用时调用","attrs":{}}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"我们分析了上面的Camera采集数据，完整的代码请看文末的Github 地址","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"AudioRecord","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"上面分析完了视频，我们接着来看音频，录音API我们使用AudioRecord，录音的流程相对于视频而言要简单许多，一样的，我们先来看一下简单类图","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/67/673687401e4c8b0ef701538a03285be1.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"就一个类，API也简单明了，我们来看一下流程","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/39/39db6036e73981f6f770a038e69aa8b4.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下面上代码","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"kotlin"},"content":[{"type":"text","text":" public void startRecord() {\n //开启录音\n mAudioRecord.startRecording();\n mIsRecording = true;\n //开启新线程轮询\n ExecutorService executorService = Executors.newSingleThreadExecutor();\n executorService.execute(new Runnable() {\n @Override\n public void run() {\n byte[] buffer = new byte[DEFAULT_BUFFER_SIZE_IN_BYTES];\n while (mIsRecording) {\n int len = mAudioRecord.read(buffer, 0, DEFAULT_BUFFER_SIZE_IN_BYTES);\n if (len > 0) {\n byte[] data = new byte[len];\n System.arraycopy(buffer, 0, data, 0, len);\n //处理data\n }\n }\n }\n });\n\n }\n\n\n public void stopRecord() {\n mIsRecording = false;\n mAACMediaCodecEncoder.stopEncoder();\n mAudioRecord.stop();\n }\n","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"AudioRecord生成的 byte[] data即PCM音频数据","attrs":{}}]}],"attrs":{}},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"小结","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"本章我们对音视频的原生输入API进行了详细的介绍，这个也是我们后面博客的基础，有了YUV和PCM数据之后，就可以编码了，下一篇我们再来分析MediaCodec，用MediaCodec对原生音视频数据进行硬编码生成Mp4.","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://github.com/LoveWFan/BlogDemo/tree/master/app/src/main/java/com/poney/blogdemo/base/camera","title":"","type":null},"content":[{"type":"text","text":"Camera","attrs":{}}]},{"type":"link","attrs":{"href":"https://github.com/LoveWFan/BlogDemo/tree/master/media/src/main/java/com/poney/ffmpeg/audio","title":"","type":null},"content":[{"type":"text","text":"AudioRecord","attrs":{}}]}]}]}