解決OpenCV JavaCameraView相機preview方向問題

    網上找了很多解決都是有問題的,研究了半天源碼解決了這個問題。我是從整個相機啟動和數據傳輸的過程着手的,這里撿重點介紹一下,最后會貼上修改后的兩個源文件。

    首先要知道一個概念。

    圖里的小圓圈是Home按鍵,不是攝像頭:)

    現在問題就是在什么地方進行旋轉,如何旋轉。這就需要了解JavaCameraView類的工作流程了。JavaCameraView實現了父類CameraBridgeViewBase的抽象方法connectCamera,這個方法主要做兩件事

1.初始化相機,包括選擇啟動哪個相機,選擇preview的Frame大小等等。這個過程是在UI線程實現的,因此onPreviewFrame也是在UI線程調用,當相機啟動后數據就會傳至onPreviewFrame。onPreviewFrame不處理數據只是將數據存好然后通知另一個線程去處理。

2.就是啟動那個處理onPreviewFrame存儲數據的線程。

JavaCameraView類的問題就是在這第1,第2步驟里都沒有區分SurfaceView和相機Frame的坐標是不同的。比如,在第1步驟里,選擇相機preview大小時,preview的height最大不能超過SurfaceView的width,也就是說相機Frame的高對應着SurfaceView的寬而相機Frame的寬對應着SurfaceView的高。因此所有涉及到SurfaceView寬和高的地方都需要修改,因為源碼都用的相機Frame的寬和高。最重要的就是第1步驟里會調用AllocateCache函數初始化一個Bitmap,之后在第2步驟里將相機Frame轉為這個Bitmap,然后直接畫到SurfaceView對應的畫布上的。因此這個Bitmap的寬和高應該跟相機Frame的寬和高正好相反,因為相機Frame的數據是順時針旋轉90度后使用的。在第2步驟里,相機Frame轉為Bitmap之前需要順時針旋轉相機Frame數據,旋轉后相機Frame和Bitmap就一致了。這個旋轉相機Frame的時機我選擇在JavaCameraFrame類實現,這個類很簡單它有一個Mat成員,onPreviewFrame每當有數據時就是存到這個成員里,這個Mat是Yuv420sp格式的。但是它實現了CvCameraViewFrame接口的gray,rgba方法,可直接返回Yuv420sp對應的灰度圖和rgba圖,此時返回的是相機Frame那個方向的。我在gray,rgba方法里對返回結果進行旋轉。我認為這個時機很好,因為通過onCameraFrame傳給JavaCameraView類的用戶的數據就是JavaCameraFrame類型,在onCameraFrame函數里用戶像往常一樣直接調用gray,rgba方法就可得到方向正確的Mat,且這個時機在相機Frame轉為Bitmap之前,當轉換時相機Frame的數據已經被旋轉到正確方向了。

水平一般,能力有限。說的可能不清楚,貼上代碼,所有我修改的地方都注釋了#Modified,修改的地方不是很多。

 

1.CameraBridgeViewBase

package org.opencv.android;

import java.util.List;

import org.opencv.BuildConfig;
import org.opencv.R;
import org.opencv.core.Mat;
import org.opencv.core.Size;

import android.app.Activity;
import android.app.AlertDialog;
import android.content.Context;
import android.content.DialogInterface;
import android.content.res.TypedArray;
import android.graphics.Bitmap;
import android.graphics.Canvas;
import android.graphics.Rect;
import android.util.AttributeSet;
import android.util.Log;
import android.view.SurfaceHolder;
import android.view.SurfaceView;

/**
 * This is a basic class, implementing the interaction with Camera and OpenCV library.
 * The main responsibility of it - is to control when camera can be enabled, process the frame,
 * call external listener to make any adjustments to the frame and then draw the resulting
 * frame to the screen.
 * The clients shall implement CvCameraViewListener.
 */
public abstract class CameraBridgeViewBase extends SurfaceView implements SurfaceHolder.Callback {

    private static final String TAG = "CameraBridge";
    private static final int MAX_UNSPECIFIED = -1;
    private static final int STOPPED = 0;
    private static final int STARTED = 1;

    private int mState = STOPPED;
    private Bitmap mCacheBitmap;
    private CvCameraViewListener2 mListener;
    private boolean mSurfaceExist;
    private final Object mSyncObject = new Object();

    protected int mFrameWidth;
    protected int mFrameHeight;
    protected int mMaxHeight;
    protected int mMaxWidth;
    protected float mScale = 0;
    protected int mPreviewFormat = RGBA;
    protected int mCameraIndex = CAMERA_ID_ANY;
    protected boolean mEnabled;
    protected FpsMeter mFpsMeter = null;

    public static final int CAMERA_ID_ANY   = -1;
    public static final int CAMERA_ID_BACK  = 99;
    public static final int CAMERA_ID_FRONT = 98;
    public static final int RGBA = 1;
    public static final int GRAY = 2;

    public CameraBridgeViewBase(Context context, int cameraId) {
        super(context);
        mCameraIndex = cameraId;
        getHolder().addCallback(this);
        mMaxWidth = MAX_UNSPECIFIED;
        mMaxHeight = MAX_UNSPECIFIED;
    }

    public CameraBridgeViewBase(Context context, AttributeSet attrs) {
        super(context, attrs);

        int count = attrs.getAttributeCount();
        Log.d(TAG, "Attr count: " + Integer.valueOf(count));

        TypedArray styledAttrs = getContext().obtainStyledAttributes(attrs, R.styleable.CameraBridgeViewBase);
        if (styledAttrs.getBoolean(R.styleable.CameraBridgeViewBase_show_fps, false))
            enableFpsMeter();

        mCameraIndex = styledAttrs.getInt(R.styleable.CameraBridgeViewBase_camera_id, -1);

        getHolder().addCallback(this);
        mMaxWidth = MAX_UNSPECIFIED;
        mMaxHeight = MAX_UNSPECIFIED;
        styledAttrs.recycle();
    }

    /**
     * Sets the camera index
     * @param cameraIndex new camera index
     */
    public void setCameraIndex(int cameraIndex) {
        this.mCameraIndex = cameraIndex;
    }

    public interface CvCameraViewListener {
        /**
         * This method is invoked when camera preview has started. After this method is invoked
         * the frames will start to be delivered to client via the onCameraFrame() callback.
         * @param width -  the width of the frames that will be delivered
         * @param height - the height of the frames that will be delivered
         */
        public void onCameraViewStarted(int width, int height);

        /**
         * This method is invoked when camera preview has been stopped for some reason.
         * No frames will be delivered via onCameraFrame() callback after this method is called.
         */
        public void onCameraViewStopped();

        /**
         * This method is invoked when delivery of the frame needs to be done.
         * The returned values - is a modified frame which needs to be displayed on the screen.
         * TODO: pass the parameters specifying the format of the frame (BPP, YUV or RGB and etc)
         */
        public Mat onCameraFrame(Mat inputFrame);
    }

    public interface CvCameraViewListener2 {
        /**
         * This method is invoked when camera preview has started. After this method is invoked
         * the frames will start to be delivered to client via the onCameraFrame() callback.
         * @param width -  the width of the frames that will be delivered
         * @param height - the height of the frames that will be delivered
         */
        public void onCameraViewStarted(int width, int height);

        /**
         * This method is invoked when camera preview has been stopped for some reason.
         * No frames will be delivered via onCameraFrame() callback after this method is called.
         */
        public void onCameraViewStopped();

        /**
         * This method is invoked when delivery of the frame needs to be done.
         * The returned values - is a modified frame which needs to be displayed on the screen.
         * TODO: pass the parameters specifying the format of the frame (BPP, YUV or RGB and etc)
         */
        public Mat onCameraFrame(CvCameraViewFrame inputFrame);
    };

    protected class CvCameraViewListenerAdapter implements CvCameraViewListener2  {
        public CvCameraViewListenerAdapter(CvCameraViewListener oldStypeListener) {
            mOldStyleListener = oldStypeListener;
        }

        public void onCameraViewStarted(int width, int height) {
            mOldStyleListener.onCameraViewStarted(width, height);
        }

        public void onCameraViewStopped() {
            mOldStyleListener.onCameraViewStopped();
        }

        public Mat onCameraFrame(CvCameraViewFrame inputFrame) {
             Mat result = null;
             switch (mPreviewFormat) {
                case RGBA:
                    result = mOldStyleListener.onCameraFrame(inputFrame.rgba());
                    break;
                case GRAY:
                    result = mOldStyleListener.onCameraFrame(inputFrame.gray());
                    break;
                default:
                    Log.e(TAG, "Invalid frame format! Only RGBA and Gray Scale are supported!");
            };

            return result;
        }

        public void setFrameFormat(int format) {
            mPreviewFormat = format;
        }

        private int mPreviewFormat = RGBA;
        private CvCameraViewListener mOldStyleListener;
    };

    /**
     * This class interface is abstract representation of single frame from camera for onCameraFrame callback
     * Attention: Do not use objects, that represents this interface out of onCameraFrame callback!
     */
    public interface CvCameraViewFrame {

        /**
         * This method returns RGBA Mat with frame
         */
        public Mat rgba();

        /**
         * This method returns single channel gray scale Mat with frame
         */
        public Mat gray();
    };

    /*
    重載SurfaceHolder.Callback的方法
     */
    /*
    Access to the underlying surface is provided via the SurfaceHolder interface,
    which can be retrieved by calling getHolder().
    The Surface will be created for you while the SurfaceView's window is visible;
    you should implement SurfaceHolder.Callback.surfaceCreated(SurfaceHolder)
    and SurfaceHolder.Callback.surfaceDestroyed(SurfaceHolder) to discover when the
    Surface is created and destroyed as the window is shown and hidden.
    One of the purposes of this class is to provide a surface in which a secondary
    thread can render into the screen. If you are going to use it this way,
    you need to be aware of some threading semantics:
    All SurfaceView and SurfaceHolder.Callback methods will be called from
    the thread running the SurfaceView's window (typically the main thread of the application). They thus need to correctly synchronize with any state that is also touched by the drawing thread.
    You must ensure that the drawing thread only touches the underlying Surface
    while it is valid -- between SurfaceHolder.Callback.surfaceCreated()
    and SurfaceHolder.Callback.surfaceDestroyed().
     */
    /*
    This is called immediately after any structural changes (format or size)
    have been made to the surface. You should at this point update the imagery
    in the surface. This method is always called at least once,
    after surfaceCreated(SurfaceHolder).
     */
    public void surfaceChanged(SurfaceHolder arg0, int arg1, int arg2, int arg3) {
        Log.d(TAG, "call surfaceChanged event");
        synchronized(mSyncObject) {
            if (!mSurfaceExist) {
                mSurfaceExist = true;
                checkCurrentState();
            } else {
                /** Surface changed. We need to stop camera and restart with new parameters */
                /* Pretend that old surface has been destroyed */
                mSurfaceExist = false;
                checkCurrentState();
                /* Now use new surface. Say we have it now */
                mSurfaceExist = true;
                checkCurrentState();
            }
        }
    }

    /*
    This is called immediately after the surface is first created.
    Implementations of this should start up whatever rendering code they desire.
    Note that only one thread can ever draw into a Surface,
    so you should not draw into the Surface here if your normal rendering
    will be in another thread.
     */
    public void surfaceCreated(SurfaceHolder holder) {
        /* Do nothing. Wait until surfaceChanged delivered */
    }

    /*
    This is called immediately before a surface is being destroyed.
    After returning from this call, you should no longer try to access this surface.
    If you have a rendering thread that directly accesses the surface,
    you must ensure that thread is no longer touching the Surface before returning
    from this function.
     */
    public void surfaceDestroyed(SurfaceHolder holder) {
        synchronized(mSyncObject) {
            mSurfaceExist = false;
            checkCurrentState();
        }
    }

    /**
     * This method is provided for clients, so they can enable the camera connection.
     * The actual onCameraViewStarted callback will be delivered only after both this method is called and surface is available
     */
    public void enableView() {
        synchronized(mSyncObject) {
            mEnabled = true;
            checkCurrentState();
        }
    }

    /**
     * This method is provided for clients, so they can disable camera connection and stop
     * the delivery of frames even though the surface view itself is not destroyed and still stays on the scren
     */
    public void disableView() {
        synchronized(mSyncObject) {
            mEnabled = false;
            checkCurrentState();
        }
    }

    /**
     * This method enables label with fps value on the screen
     */
    public void enableFpsMeter() {
        if (mFpsMeter == null) {
            mFpsMeter = new FpsMeter();
            mFpsMeter.setResolution(mFrameWidth, mFrameHeight);
        }
    }

    public void disableFpsMeter() {
            mFpsMeter = null;
    }

    /**
     *
     * @param listener
     */

    public void setCvCameraViewListener(CvCameraViewListener2 listener) {
        mListener = listener;
    }

    public void setCvCameraViewListener(CvCameraViewListener listener) {
        CvCameraViewListenerAdapter adapter = new CvCameraViewListenerAdapter(listener);
        adapter.setFrameFormat(mPreviewFormat);
        mListener = adapter;
    }

    /**
     * This method sets the maximum size that camera frame is allowed to be. When selecting
     * size - the biggest size which less or equal the size set will be selected.
     * As an example - we set setMaxFrameSize(200,200) and we have 176x152 and 320x240 sizes. The
     * preview frame will be selected with 176x152 size.
     * This method is useful when need to restrict the size of preview frame for some reason (for example for video recording)
     * @param maxWidth - the maximum width allowed for camera frame.
     * @param maxHeight - the maximum height allowed for camera frame
     */
    public void setMaxFrameSize(int maxWidth, int maxHeight) {
        mMaxWidth = maxWidth;
        mMaxHeight = maxHeight;
    }

    public void SetCaptureFormat(int format)
    {
        mPreviewFormat = format;
        if (mListener instanceof CvCameraViewListenerAdapter) {
            CvCameraViewListenerAdapter adapter = (CvCameraViewListenerAdapter) mListener;
            adapter.setFrameFormat(mPreviewFormat);
        }
    }

    /**
     * Called when mSyncObject lock is held
     */
    private void checkCurrentState() {
        Log.d(TAG, "call checkCurrentState");
        int targetState;
        //enableView()將設置mEnabled為true,surfaceChanged()將設置mSurfaceExist
        //getVisibility() == VISIBLE似乎總是成立的
        //當surface准備好了且client設置enableView()時設置targetState為STARTED
        if (mEnabled && mSurfaceExist && getVisibility() == VISIBLE) {
            targetState = STARTED;
        } else {
            targetState = STOPPED;
        }

        //mState初始值是STOPPED
        //若目標狀態與當前狀態不同則退出當前狀態進入目標狀態
        if (targetState != mState) {
            /* The state change detected. Need to exit the current state and enter target state */
            processExitState(mState);
            mState = targetState;
            processEnterState(mState);
        }
    }

    private void processEnterState(int state) {
        Log.d(TAG, "call processEnterState: " + state);
        switch(state) {
        case STARTED:
            //真正啟動相機的地方
            onEnterStartedState();
            if (mListener != null) {
                //進入STARTED狀態后若CameraBridgeViewBase類的成員CvCameraViewListener2 mListener
                //不為null則調用其onCameraViewStarted方法,通知mListener相機啟動了
                mListener.onCameraViewStarted(mFrameWidth, mFrameHeight);
            }
            break;
        case STOPPED:
            onEnterStoppedState();
            if (mListener != null) {
                //進入STOPPED狀態后若CameraBridgeViewBase類的成員CvCameraViewListener2 mListener
                //不為null則調用其onCameraViewStopped方法,通知mListener相機停止了
                mListener.onCameraViewStopped();
            }
            break;
        };
    }

    private void processExitState(int state) {
        Log.d(TAG, "call processExitState: " + state);
        switch(state) {
        case STARTED:
            onExitStartedState();
            break;
        case STOPPED:
            onExitStoppedState();
            break;
        };
    }

    private void onEnterStoppedState() {
        /* nothing to do */
    }

    private void onExitStoppedState() {
        /* nothing to do */
    }

    // NOTE: The order of bitmap constructor and camera connection is important for android 4.1.x
    // Bitmap must be constructed before surface
    private void onEnterStartedState() {
        Log.d(TAG, "call onEnterStartedState");
        /* Connect camera */
        //connectCamera的參數是CameraBridgeViewBase的width,height
        if (!connectCamera(getWidth(), getHeight())) {
            AlertDialog ad = new AlertDialog.Builder(getContext()).create();
            ad.setCancelable(false); // This blocks the 'BACK' button
            ad.setMessage("It seems that you device does not support camera (or it is locked). Application will be closed.");
            ad.setButton(DialogInterface.BUTTON_NEUTRAL,  "OK", new DialogInterface.OnClickListener() {
                public void onClick(DialogInterface dialog, int which) {
                    dialog.dismiss();
                    ((Activity) getContext()).finish();
                }
            });
            ad.show();

        }
    }

    private void onExitStartedState() {
        disconnectCamera();
        if (mCacheBitmap != null) {
            mCacheBitmap.recycle();
        }
    }

    /*
        onPreviewFrame在UI線程被調用,它存好數據后通知另一個線程處理
        另一個線程就調用這個方法處理數據
        onPreviewFrame相當於生產者,另一個線程相當於消費者
        當使用JavaCameraView類時frame是JavaCameraFrame類型
        其通過OpenCV實現了接口
     */
    /**
     * This method shall be called by the subclasses when they have valid
     * object and want it to be delivered to external client (via callback) and
     * then displayed on the screen.
     * @param frame - the current frame to be delivered
     */
    protected void deliverAndDrawFrame(CvCameraViewFrame frame) {
        Mat modified;

        if (mListener != null) {
            //CvCameraViewListener2 mListener是client指定的
            //這里調用客戶重載的接口方法且接收返回值
            //這里都是在數據處理線程里執行的
            modified = mListener.onCameraFrame(frame);
        } else {
            //若client沒指定CvCameraViewListener2 mListener即client不准備處理preview數據
            //則modified設置為
            //onPreviewFrame傳回的數據轉換成的rgba Mat
            modified = frame.rgba();
        }

        //Log Mat的大小和Bitmap的大小
        Log.d("FunnyAR","mScale: "+mScale+" modified.rows: "+modified.rows()
                +" modified.cols: "+modified.cols()+" mCacheBitmap.getWidth(): "+
                mCacheBitmap.getWidth()+" mCacheBitmap.getHeight() "+
                mCacheBitmap.getHeight());

        //標志modified轉Bitmap是否成功
        boolean bmpValid = true;
        //若確實有modified則將其轉為Bitmap
        if (modified != null) {
            try {
                Utils.matToBitmap(modified, mCacheBitmap);
            } catch(Exception e) {
                Log.e(TAG, "Mat type: " + modified);
                Log.e(TAG, "Bitmap type: " + mCacheBitmap.getWidth() + "*" + mCacheBitmap.getHeight());
                Log.e(TAG, "Utils.matToBitmap() throws an exception: " + e.getMessage());
                bmpValid = false;
            }
        }
        //轉換成功通過畫布畫到surface里
        if (bmpValid && mCacheBitmap != null) {
            Canvas canvas = getHolder().lockCanvas();
            if (canvas != null) {
                canvas.drawColor(0, android.graphics.PorterDuff.Mode.CLEAR);
                if (BuildConfig.DEBUG)
                    Log.d(TAG, "mStretch value: " + mScale);

                if (mScale != 0) {
                    canvas.drawBitmap(mCacheBitmap, new Rect(0,0,mCacheBitmap.getWidth(), mCacheBitmap.getHeight()),
                         new Rect((int)((canvas.getWidth() - mScale*mCacheBitmap.getWidth()) / 2),
                         (int)((canvas.getHeight() - mScale*mCacheBitmap.getHeight()) / 2),
                         (int)((canvas.getWidth() - mScale*mCacheBitmap.getWidth()) / 2 + mScale*mCacheBitmap.getWidth()),
                         (int)((canvas.getHeight() - mScale*mCacheBitmap.getHeight()) / 2 + mScale*mCacheBitmap.getHeight())), null);
                } else {
                     canvas.drawBitmap(mCacheBitmap, new Rect(0,0,mCacheBitmap.getWidth(), mCacheBitmap.getHeight()),
                         new Rect((canvas.getWidth() - mCacheBitmap.getWidth()) / 2,
                         (canvas.getHeight() - mCacheBitmap.getHeight()) / 2,
                         (canvas.getWidth() - mCacheBitmap.getWidth()) / 2 + mCacheBitmap.getWidth(),
                         (canvas.getHeight() - mCacheBitmap.getHeight()) / 2 + mCacheBitmap.getHeight()), null);
                }

                if (mFpsMeter != null) {
                    mFpsMeter.measure();
                    mFpsMeter.draw(canvas, 20, 30);
                }
                getHolder().unlockCanvasAndPost(canvas);
            }
        }
    }

    /**
     * This method is invoked shall perform concrete operation to initialize the camera.
     * CONTRACT: as a result of this method variables mFrameWidth and mFrameHeight MUST be
     * initialized with the size of the Camera frames that will be delivered to external processor.
     * @param width - the width of this SurfaceView
     * @param height - the height of this SurfaceView
     */
    //具體啟動相機的過程由子類實現
    protected abstract boolean connectCamera(int width, int height);

    /**
     * Disconnects and release the particular camera object being connected to this surface view.
     * Called when syncObject lock is held
     */
    protected abstract void disconnectCamera();

    // NOTE: On Android 4.1.x the function must be called before SurfaceTexture constructor!
    protected void AllocateCache()
    {
        //mCacheBitmap = Bitmap.createBitmap(mFrameWidth, mFrameHeight, Bitmap.Config.ARGB_8888);
        //#Modified portrait step2
        //為了方向正確mCacheBitmap存儲的時相機frame旋轉90度之后的數據
        //旋轉90度后mFrameWidth,mFrameHeight互換
        int portraitWidth=mFrameHeight;
        int portraitHeight=mFrameWidth;
        mCacheBitmap = Bitmap.createBitmap(portraitWidth, portraitHeight, Bitmap.Config.ARGB_8888);
    }

    public interface ListItemAccessor {
        public int getWidth(Object obj);
        public int getHeight(Object obj);
    };

    /**
     * This helper method can be called by subclasses to select camera preview size.
     * It goes over the list of the supported preview sizes and selects the maximum one which
     * fits both values set via setMaxFrameSize() and surface frame allocated for this view
     * @param supportedSizes
     * @param surfaceWidth
     * @param surfaceHeight
     * @return optimal frame size
     */
    protected Size calculateCameraFrameSize(List<?> supportedSizes, ListItemAccessor accessor, int surfaceWidth, int surfaceHeight) {
        //選擇一個相機frame大小
        int calcWidth = 0;
        int calcHeight = 0;

        //允許的最大width和height
        //#Modified step4
        //相機Frame的mMaxWidth應該與surface的surfaceHeight比
        //相機Frame的mMaxHeight應該與surface的surfaceWidth比
        //int maxAllowedWidth = (mMaxWidth != MAX_UNSPECIFIED && mMaxWidth < surfaceWidth)? mMaxWidth : surfaceWidth;
        //int maxAllowedHeight = (mMaxHeight != MAX_UNSPECIFIED && mMaxHeight < surfaceHeight)? mMaxHeight : surfaceHeight;
        int maxAllowedWidth = (mMaxWidth != MAX_UNSPECIFIED && mMaxWidth < surfaceHeight)? mMaxWidth : surfaceHeight;
        int maxAllowedHeight = (mMaxHeight != MAX_UNSPECIFIED && mMaxHeight < surfaceWidth)? mMaxHeight : surfaceWidth;

        for (Object size : supportedSizes) {
            int width = accessor.getWidth(size);
            int height = accessor.getHeight(size);

            //在允許的范圍內選擇最大的size
            //client是可通過設置小的mMaxWidth,mMaxHeight來選擇低分辨率frame的
            if (width <= maxAllowedWidth && height <= maxAllowedHeight) {
                if (width >= calcWidth && height >= calcHeight) {
                    calcWidth = (int) width;
                    calcHeight = (int) height;
                }
            }
        }

        return new Size(calcWidth, calcHeight);
    }
}

 2.JavaCameraView

package org.opencv.android;

import java.util.List;

import android.content.Context;
import android.graphics.ImageFormat;
import android.graphics.SurfaceTexture;
import android.hardware.Camera;
import android.hardware.Camera.PreviewCallback;
import android.os.Build;
import android.util.AttributeSet;
import android.util.Log;
import android.view.ViewGroup.LayoutParams;

import org.opencv.BuildConfig;
import org.opencv.core.Core;
import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.core.Size;
import org.opencv.imgproc.Imgproc;

/**
 * This class is an implementation of the Bridge View between OpenCV and Java Camera.
 * This class relays on the functionality available in base class and only implements
 * required functions:
 * connectCamera - opens Java camera and sets the PreviewCallback to be delivered.
 * disconnectCamera - closes the camera and stops preview.
 * When frame is delivered via callback from Camera - it processed via OpenCV to be
 * converted to RGBA32 and then passed to the external callback for modifications if required.
 */
public class JavaCameraView extends CameraBridgeViewBase implements PreviewCallback {

    private static final int MAGIC_TEXTURE_ID = 10;
    private static final String TAG = "JavaCameraView";

    private byte mBuffer[];
    private Mat[] mFrameChain;
    private int mChainIdx = 0;
    private Thread mThread;
    private boolean mStopThread;

    protected Camera mCamera;
    protected JavaCameraFrame[] mCameraFrame;
    private SurfaceTexture mSurfaceTexture;
    private int mPreviewFormat = ImageFormat.NV21;

    public static class JavaCameraSizeAccessor implements ListItemAccessor {

        @Override
        public int getWidth(Object obj) {
            Camera.Size size = (Camera.Size) obj;
            return size.width;
        }

        @Override
        public int getHeight(Object obj) {
            Camera.Size size = (Camera.Size) obj;
            return size.height;
        }
    }

    public JavaCameraView(Context context, int cameraId) {
        super(context, cameraId);
    }

    public JavaCameraView(Context context, AttributeSet attrs) {
        super(context, attrs);
    }

    //傳入的是JavaCameraView的width,height
    protected boolean initializeCamera(int width, int height) {
        Log.d(TAG, "Initialize java camera");
        boolean result = true;
        synchronized (this){
            mCamera = null;
            //mCameraIndex是指定相機的類型,是應用里的設置不是指相機ID
            //相機ID需根據類型找
            //繼承父類CameraBridgeViewBase
            //初值為CAMERA_ID_ANY
            if (mCameraIndex == CAMERA_ID_ANY) {
                Log.d(TAG, "Trying to open camera with old open()");
                try {
                    //先嘗試不指定相機類型啟動相機
                    mCamera = Camera.open();
                }
                catch (Exception e){
                    Log.e(TAG, "Camera is not available (in use or does not exist): " + e.getLocalizedMessage());
                }

                if(mCamera == null && Build.VERSION.SDK_INT >= Build.VERSION_CODES.GINGERBREAD) {
                    boolean connected = false;
                    for (int camIdx = 0; camIdx < Camera.getNumberOfCameras(); ++camIdx) {
                        Log.d(TAG, "Trying to open camera with new open(" + Integer.valueOf(camIdx) + ")");
                        try {
                            //若不指定相機類型啟動相機失敗則遍歷所有相機ID一個個嘗試啟動,一旦成功
                            //就選擇當前成功啟動的相機
                            mCamera = Camera.open(camIdx);
                            connected = true;
                        } catch (RuntimeException e) {
                            Log.e(TAG, "Camera #" + camIdx + "failed to open: " + e.getLocalizedMessage());
                        }
                        if (connected) break;
                    }
                }
            } else {
                //這里是指定相機類型的情況
                if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.GINGERBREAD) {
                    int localCameraIndex = mCameraIndex;
                    if (mCameraIndex == CAMERA_ID_BACK) {
                        Log.i(TAG, "Trying to open back camera");
                        Camera.CameraInfo cameraInfo = new Camera.CameraInfo();
                        //根據相機類型找此類型對應的相機ID
                        for (int camIdx = 0; camIdx < Camera.getNumberOfCameras(); ++camIdx) {
                            Camera.getCameraInfo( camIdx, cameraInfo );
                            if (cameraInfo.facing == Camera.CameraInfo.CAMERA_FACING_BACK) {
                                localCameraIndex = camIdx;
                                break;
                            }
                        }
                    } else if (mCameraIndex == CAMERA_ID_FRONT) {
                        Log.i(TAG, "Trying to open front camera");
                        Camera.CameraInfo cameraInfo = new Camera.CameraInfo();
                        for (int camIdx = 0; camIdx < Camera.getNumberOfCameras(); ++camIdx) {
                            Camera.getCameraInfo( camIdx, cameraInfo );
                            if (cameraInfo.facing == Camera.CameraInfo.CAMERA_FACING_FRONT) {
                                localCameraIndex = camIdx;
                                break;
                            }
                        }
                    }
                    if (localCameraIndex == CAMERA_ID_BACK) {
                        //localCameraIndex初賦值為CAMERA_ID_BACK類型,指定要啟動背面相機
                        //若有背面相機此處localCameraIndex值已經被賦值為背面相機的相機ID了
                        Log.e(TAG, "Back camera not found!");
                    } else if (localCameraIndex == CAMERA_ID_FRONT) {
                        Log.e(TAG, "Front camera not found!");
                    } else {
                        Log.d(TAG, "Trying to open camera with new open(" + Integer.valueOf(localCameraIndex) + ")");
                        try {
                            //根據找到的相機ID啟動相機
                            mCamera = Camera.open(localCameraIndex);
                        } catch (RuntimeException e) {
                            Log.e(TAG, "Camera #" + localCameraIndex + "failed to open: " + e.getLocalizedMessage());
                        }
                    }
                }
            }

            //若啟動相機失敗則返回false
            if (mCamera == null)
                return false;

            /* Now set camera parameters */
            try {
                Camera.Parameters params = mCamera.getParameters();
                Log.d(TAG, "getSupportedPreviewSizes()");
                List<android.hardware.Camera.Size> sizes = params.getSupportedPreviewSizes();

                if (sizes != null) {
                    //選擇預覽size
                    /* Select the size that fits surface considering maximum size allowed */
                    Size frameSize = calculateCameraFrameSize(sizes, new JavaCameraSizeAccessor(), width, height);
                    //這里width,height是connectCamera(getWidth(), getHeight())傳進來的
                    //是surfaceView的大小也是surface的大小
                    //Log相機frame大小和surface大小
                    Log.d("FunnyAR","surface width: "+width+" surface height: "+height+
                            "frameSize: "+frameSize.toString());

                    //選擇預覽格式
                    /* Image format NV21 causes issues in the Android emulators */
                    if (Build.FINGERPRINT.startsWith("generic")
                            || Build.FINGERPRINT.startsWith("unknown")
                            || Build.MODEL.contains("google_sdk")
                            || Build.MODEL.contains("Emulator")
                            || Build.MODEL.contains("Android SDK built for x86")
                            || Build.MANUFACTURER.contains("Genymotion")
                            || (Build.BRAND.startsWith("generic") && Build.DEVICE.startsWith("generic"))
                            || "google_sdk".equals(Build.PRODUCT))
                        params.setPreviewFormat(ImageFormat.YV12);  // "generic" or "android" = android emulator
                    else
                        params.setPreviewFormat(ImageFormat.NV21);

                    //預覽格式記錄到成員變量里
                    mPreviewFormat = params.getPreviewFormat();

                    Log.d(TAG, "Set preview size to " + Integer.valueOf((int)frameSize.width) + "x" + Integer.valueOf((int)frameSize.height));
                    params.setPreviewSize((int)frameSize.width, (int)frameSize.height);

                    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.ICE_CREAM_SANDWICH && !android.os.Build.MODEL.equals("GT-I9100"))
                        params.setRecordingHint(true);

                    //JavaCameraView的聚焦模式也是寫定下的
                    List<String> FocusModes = params.getSupportedFocusModes();
                    if (FocusModes != null && FocusModes.contains(Camera.Parameters.FOCUS_MODE_CONTINUOUS_VIDEO))
                    {
                        params.setFocusMode(Camera.Parameters.FOCUS_MODE_CONTINUOUS_VIDEO);
                    }

                    mCamera.setParameters(params);
                    params = mCamera.getParameters();

                    //設置frame大小
                    mFrameWidth = params.getPreviewSize().width;
                    mFrameHeight = params.getPreviewSize().height;

                    //這里涉及到縮放
                    /*
                        #Modified portrait step1
                        為了在deliverAndDrawFrame里往畫布上畫時應用縮放
                        <JavaCameraView>里
                        android:layout_width="match_parent"
                        android:layout_height="match_parent"
                        若又想指定縮放后的大小可將<JavaCameraView>放在一個有大小的
                        LinearLayout里
                        且當方向是portrait時比率是
                        surface的width/相機frame的mFrameHeight
                        surface的height/相機frame的mFrameWidth
                        若不想設置<JavaCameraView>則這里直接去掉if語句應該也可
                     */
                    if ((getLayoutParams().width == LayoutParams.MATCH_PARENT) && (getLayoutParams().height == LayoutParams.MATCH_PARENT))
                        //mScale = Math.min(((float)height)/mFrameHeight, ((float)width)/mFrameWidth);
                        mScale = Math.min(((float)width)/mFrameHeight, ((float)height)/mFrameWidth);
                    else
                        mScale = 0;

                    //Log縮放和相機Frame大小
                    Log.d("FunnyAR","mScale: "+mScale+" mFrameWidth: "+mFrameWidth+
                            " mFrameHeight: "+mFrameHeight);

                    if (mFpsMeter != null) {
                        mFpsMeter.setResolution(mFrameWidth, mFrameHeight);
                    }

                    //算frame的字節大小,設置相應大小的緩沖區接收數據
                    //像素個數
                    int size = mFrameWidth * mFrameHeight;
                    //像素個數x當前格式每個像素所需bit個數/一個字節8bit==frame所需byte數
                    size  = size * ImageFormat.getBitsPerPixel(params.getPreviewFormat()) / 8;
                    mBuffer = new byte[size];

                    /*
                    Adds a pre-allocated buffer to the preview callback buffer queue.
                    Applications can add one or more buffers to the queue.
                    When a preview frame arrives and there is still at least
                    one available buffer, the buffer will be used and removed from the queue.
                    Then preview callback is invoked with the buffer.
                    If a frame arrives and there is no buffer left, the frame is discarded.
                    Applications should add buffers back when they finish processing the data
                     in them.
                     */
                    /*
                    This method is only necessary when setPreviewCallbackWithBuffer(PreviewCallback)
                     is used. When setPreviewCallback(PreviewCallback) or
                     setOneShotPreviewCallback(PreviewCallback) are used,
                     buffers are automatically allocated.
                     When a supplied buffer is too small to hold the preview frame data,
                     preview callback will return null and the buffer will be removed from the
                     buffer queue.
                     */
                    mCamera.addCallbackBuffer(mBuffer);
                    /*
                    Installs a callback to be invoked for every preview frame,
                    using buffers supplied with addCallbackBuffer(byte[]),
                    in addition to displaying them on the screen.
                    he callback will be repeatedly called for as long as preview is active
                    and buffers are available. Any other preview callbacks are overridden.
                     */
                    mCamera.setPreviewCallbackWithBuffer(this);

                    //一個Mat數組
                    //注意Yuv420sp格式
                    mFrameChain = new Mat[2];
                    mFrameChain[0] = new Mat(mFrameHeight + (mFrameHeight/2), mFrameWidth, CvType.CV_8UC1);
                    mFrameChain[1] = new Mat(mFrameHeight + (mFrameHeight/2), mFrameWidth, CvType.CV_8UC1);

                    //繼承的方法為繼承的Bitmap mCacheBitmap初始化內存
                    AllocateCache();

                    //JavaCameraFrame內部有對Mat的引用
                    //mCameraFrame[0].mYuvFrameData就是Mat mFrameChain[0]
                    mCameraFrame = new JavaCameraFrame[2];
                    mCameraFrame[0] = new JavaCameraFrame(mFrameChain[0], mFrameWidth, mFrameHeight);
                    mCameraFrame[1] = new JavaCameraFrame(mFrameChain[1], mFrameWidth, mFrameHeight);

                    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.HONEYCOMB) {
                        mSurfaceTexture = new SurfaceTexture(MAGIC_TEXTURE_ID);
                        mCamera.setPreviewTexture(mSurfaceTexture);
                    } else
                       mCamera.setPreviewDisplay(null);

                    /* Finally we are ready to start the preview */
                    Log.d(TAG, "startPreview");
                    mCamera.startPreview();
                }
                else
                    result = false;
            } catch (Exception e) {
                result = false;
                e.printStackTrace();
            }
        }

        return result;
    }

    protected void releaseCamera() {
        synchronized (this) {
            if (mCamera != null) {
                mCamera.stopPreview();
                mCamera.setPreviewCallback(null);

                mCamera.release();
            }
            mCamera = null;
            if (mFrameChain != null) {
                mFrameChain[0].release();
                mFrameChain[1].release();
            }
            if (mCameraFrame != null) {
                mCameraFrame[0].release();
                mCameraFrame[1].release();
            }
        }
    }

    private boolean mCameraFrameReady = false;

    //重載父類的抽象方法,負責啟動相機
    @Override
    protected boolean connectCamera(int width, int height) {

        /* 1. We need to instantiate camera
         * 2. We need to start thread which will be getting frames
         */
        /* First step - initialize camera connection */
        Log.d(TAG, "Connecting to camera");
        //用initializeCamera函數實現初始化相機連接
        if (!initializeCamera(width, height))
            return false;

        mCameraFrameReady = false;

        /* now we can start update thread */
        Log.d(TAG, "Starting processing thread");
        mStopThread = false;
        mThread = new Thread(new CameraWorker());
        mThread.start();

        return true;
    }

    @Override
    protected void disconnectCamera() {
        /* 1. We need to stop thread which updating the frames
         * 2. Stop camera and release it
         */
        Log.d(TAG, "Disconnecting from camera");
        try {
            mStopThread = true;
            Log.d(TAG, "Notify thread");
            synchronized (this) {
                this.notify();
            }
            Log.d(TAG, "Waiting for thread");
            if (mThread != null)
                mThread.join();
        } catch (InterruptedException e) {
            e.printStackTrace();
        } finally {
            mThread =  null;
        }

        /* Now release camera */
        releaseCamera();

        mCameraFrameReady = false;
    }

    /*
    重載Camera.PreviewCallback onPreviewFrame方法
    這里onPreviewFrame時在UI線程里被處理的,因為相機時在主線程里被啟動的
    但是數據將被另一個線程取走處理
     */
    /*
    Callback interface used to deliver copies of preview frames as they are displayed.
    Called as preview frames are displayed. This callback is invoked
     on the event thread Camera.open(int) was called from.
     */
    @Override
    public void onPreviewFrame(byte[] frame, Camera arg1) {
        if (BuildConfig.DEBUG)
            Log.d(TAG, "Preview Frame received. Frame size: " + frame.length);
        synchronized (this) {
            //mChainIdx在0,1間切換,由另一個線程負責管理
            //OpenCV Java層特有的方法
            //mFrameChain[mChainIdx]的大小是1.5height x 1.0width,將數據存進去
            mFrameChain[mChainIdx].put(0, 0, frame);
            //設置標志表示數據存好了
            mCameraFrameReady = true;
            //喚醒一個等待當前JavaCameraView.this的線程

            this.notify();
        }
        /*
        onPreviewFrame處理數據時addCallbackBuffer()的buffer將出隊列被處理
        處理完后為了下次onPreviewFrame需再次將buffer給回調
         */
        if (mCamera != null)
            mCamera.addCallbackBuffer(mBuffer);
    }

    /*
    JavaCameraFrame實現CvCameraViewFrame的rgba(),gray()方法
    這個類型將通過deliverAndDrawFrame()里的mListener.onCameraFrame(frame)傳給用戶處理
    在JavaCameraFrame的接口里實現Mat的旋轉是最好的時機了
    如此client通過gray(),rgba()獲得的Mat就是方向portrait的了
    #Modified portrait step3
     */
    private class JavaCameraFrame implements CvCameraViewFrame {
        @Override
        public Mat gray() {
            //返回Mat里的選定區域,這跟Yuv420sp格式緊密相關
            //return mYuvFrameData.submat(0, mHeight, 0, mWidth);
            //#Modified step3.1
            Core.rotate(mYuvFrameData.submat(0, mHeight, 0, mWidth),
                    portrait_gray,Core.ROTATE_90_CLOCKWISE);
            return portrait_gray;
        }

        @Override
        public Mat rgba() {
            if (mPreviewFormat == ImageFormat.NV21)
                Imgproc.cvtColor(mYuvFrameData, mRgba, Imgproc.COLOR_YUV2RGBA_NV21, 4);
            else if (mPreviewFormat == ImageFormat.YV12)
                Imgproc.cvtColor(mYuvFrameData, mRgba, Imgproc.COLOR_YUV2RGB_I420, 4);  // COLOR_YUV2RGBA_YV12 produces inverted colors
            else
                throw new IllegalArgumentException("Preview Format can be NV21 or YV12");

            //#Modified step3.2
            Core.rotate(mYuvFrameData.submat(0, mHeight, 0, mWidth),
                    portrait_rgba,Core.ROTATE_90_CLOCKWISE);

            return portrait_rgba;
        }

        public JavaCameraFrame(Mat Yuv420sp, int width, int height) {
            super();
            mWidth = width;
            mHeight = height;
            //#Modified
            portrait_mHeight=mWidth;
            portrait_mWidth=mHeight;
            portrait_gray=new Mat(portrait_mHeight,portrait_mWidth,CvType.CV_8UC1);
            portrait_rgba=new Mat(portrait_mHeight,portrait_mWidth,CvType.CV_8UC4);
            mYuvFrameData = Yuv420sp;
            mRgba = new Mat();
        }

        public void release() {
            mRgba.release();
        }

        private Mat mYuvFrameData;
        private Mat mRgba;
        private int mWidth;
        private int mHeight;
        //#Modified
        private int portrait_mHeight;
        private int portrait_mWidth;
        private Mat portrait_gray;
        private Mat portrait_rgba;
    };

    private class CameraWorker implements Runnable {

        @Override
        public void run() {
            do {
                boolean hasFrame = false;
                synchronized (JavaCameraView.this) {
                    try {
                        //onPreviewFrame里frame准備好了會設置mCameraFrameReady為true然后喚醒此線程
                        //只要相機啟動着mStopThread就為false
                        //當相機啟動着且onPreviewFrame里frame沒准備好時線程就等待
                        //等待語句放在while里防止條件沒滿足時線程被喚醒
                        while (!mCameraFrameReady && !mStopThread) {
                            JavaCameraView.this.wait();
                        }
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                    //線程被喚醒是因為onPreviewFrame里frame准備好了
                    if (mCameraFrameReady)
                    {
                        //mChainIdx在0,1之間切換表示mCameraFrame當前的緩沖區
                        mChainIdx = 1 - mChainIdx;
                        //設置mCameraFrameReady為false用來等下次onPreviewFrame里frame准備好
                        mCameraFrameReady = false;
                        //表示當前有frame可用
                        hasFrame = true;
                    }
                }

                //線程沒停止且有frame可用
                if (!mStopThread && hasFrame) {
                    //當前的緩沖區不為空則處理它
                    //mChainIdx初值為0,mChainIdx = 1 - mChainIdx設置其為1
                    //這里1 - mChainIdx為0
                    //之后mChainIdx值為1,mChainIdx = 1 - mChainIdx設置其為0
                    //這里1 - mChainIdx為1
                    //如此循環
                    //mCameraFrame[1 - mChainIdx].mYuvFrameData就是對mFrameChain[1 - mChainIdx]
                    //的引用,即JavaCameraFrame類里有對Mat的引用
                    if (!mFrameChain[1 - mChainIdx].empty())
                        deliverAndDrawFrame(mCameraFrame[1 - mChainIdx]);
                }
            } while (!mStopThread);
            Log.d(TAG, "Finish processing thread");
        }
    }
}

3.這是layout文件

<?xml version="1.0" encoding="utf-8"?>
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:tools="http://schemas.android.com/tools"
    android:layout_width="match_parent"
    android:layout_height="match_parent"
    android:orientation="vertical"
    tools:context=".MainActivity">

    <!-- 這里1080px,1440px是硬編碼,適合redmi note4x屏幕 -->
    <!--<org.opencv.android.JavaCameraView
        android:id="@+id/javaCameraView"
        android:layout_width="1080px"
        android:layout_height="1440px" />-->

    <!--為了應用縮放-->
    <LinearLayout
        android:layout_width="1080px"
        android:layout_height="1440px"
        android:orientation="vertical">

        <org.opencv.android.JavaCameraView
            android:id="@+id/javaCameraView"
            android:layout_width="match_parent"
            android:layout_height="match_parent" />

    </LinearLayout>

    <TextView
        android:layout_width="1080px"
        android:layout_height="480px"
        android:text="FunnyAR!" />

</LinearLayout>