Optimizing Access to Raw Camera Frames on Android – Part 3.

This is the final post in this series. In the previous posts I have shown a way to gain access to the raw camera frames by using private APIs. As it is very dangerous to use hidden APIs, extra care must be taken during error handling. As I mentioned in the first post, there is no need for hacks on Android 2.2 and above, as there is an improved setPreviewCallbackWithBuffer method there that makes all these hacks unnecessary.

We are done with 1.5 and 1.6 versions. All that is left is 2.0 and 2.1. The good news is that 2.0 is basically extinct so we don’t have to worry about it. The even better news is that the improved previewCallback function can be used on 2.1 versions too! It was already there in the public APIs, but was hidden. So we have to use reflection to reach it.

Before starting to write a single line of code, let’s think for a second about what is the best way to add this new functionality to the app. In the previous posts we followed the pattern that most of the “logic” was in the native parts of the code. The camera handling was managed from the native code. But this new API is in Java. So if we want to keep this logic to not mess up the existing code base, we have to find a way to add this new type of camera handling to the LoadManager class. One way to do achieve this, is to create a fake native Camera class – that implements MyICam – and implement their methods as that they do nothing else just calling back to the Java layer.

Let’s start with the CameraPreview.java class. This is where we would use the new camera API.

CameraPreview.java


class CameraPreview extends SurfaceView implements SurfaceHolder.Callback {
        private final static String TAG = "CameraPreview";

	private static Method addCBBuffer=null;
	private static Method setPreviewCB=null;
	private static boolean hasNewCameraApi = false;
	private static int bufSize = 115200;
	private static PreviewCallback cb = null;

	protected static SurfaceHolder mHolder;
	private static Camera camera=null;

	static {
		checkForNewCameraApi();
	};

	private static void checkForNewCameraApi() {
		try {
			Class clazz = Class.forName("android.hardware.Camera");
			addCBBuffer = clazz.getMethod("addCallbackBuffer", new Class[]{ byte[].class });
			setPreviewCB = clazz.getMethod("setPreviewCallbackWithBuffer", new Class[]{ PreviewCallback.class });
			hasNewCameraApi = true;
		} catch (ClassNotFoundException e) {
			Log.e(TAG, "Can't find android.hardware.Camera class");
			e.printStackTrace();
		}  catch (NoSuchMethodException e) {
			hasNewCameraApi = false;
		}
	}

	private static boolean addCallbacks(){
    	if(hasNewCameraApi){
    		PixelFormat p = new PixelFormat();
    		PixelFormat.getPixelFormatInfo(PixelFormat.YCbCr_420_SP, p);
    		byte[] buffer1 = new byte[bufSize];
    		byte[] buffer2 = new byte[bufSize];
    		byte[] buffer3 = new byte[bufSize];
    		try {
				addCBBuffer.invoke(camera, buffer1);
				addCBBuffer.invoke(camera, buffer2);
				addCBBuffer.invoke(camera, buffer3);
			} catch (IllegalArgumentException e) {
				Log.e(TAG, "..." , e);
				return false;
			} catch (IllegalAccessException e) {
				Log.e(TAG, "..." , e);
				return false;
			} catch (InvocationTargetException e) {
				Log.e(TAG, "..." , e);
				return false;
			}
			return true;
    	}
    	return false;
	}

//these will be called from native code only
    public static void initCameraFromNative(){
    	camera = Camera.open();
    	if (camera!=null) {
	    	try {
				camera.setPreviewDisplay(mHolder);
			} catch (IOException e) {
				Log.e(TAG, "..." , e);
			}
	    	Camera.Parameters parameters = camera.getParameters();
	    	parameters.setPreviewSize(320, 240);
	    	parameters.setPreviewFormat(PixelFormat.YCbCr_420_SP);
	    	parameters.setPreviewFrameRate(15);
	    	camera.setParameters(parameters);
    	}
   }
   public static void releaseCameraFromNative(){
    	if (camera != null){
    		camera.release();
    		camera = null;    		
    	}
    }

    //true means: start
    //false means: stop
    public static void setRecordingCallbackFromNative(boolean action){
    	if(action){
    		if(!addCallbacks()){
    			hasNewCameraApi = false;
    		}
    		if(hasNewCameraApi){
    			try {
					setPreviewCB.invoke(camera, new Object[]{
							(cb = new PreviewCallback() {

								@Override
								public void onPreviewFrame(byte[] data, Camera camera) {
data.length);
									Native.previewCallback(data);
									try {										
										addCBBuffer.invoke(camera, data);
									} catch (IllegalArgumentException e) {
										Log.e(TAG, "..." , e);
									} catch (IllegalAccessException e) {
										Log.e(TAG, "..." , e);
									} catch (InvocationTargetException e) {
										Log.e(TAG, "..." , e);
									}
								}
							}) });
				} catch (IllegalArgumentException e) {
					Log.e(TAG, "..." , e);
					hasNewCameraApi = false;
				} catch (IllegalAccessException e) {
					Log.e(TAG, "..." , e);
					hasNewCameraApi = false;
				} catch (InvocationTargetException e) {
					Log.e(TAG, "..." , e);
					hasNewCameraApi = false;
				}
				//fallback to the old setPreviewCallback
				if(!hasNewCameraApi){
					camera.setPreviewCallback(new Camera.PreviewCallback() {

	    				@Override
	    				public void onPreviewFrame(byte[] data, Camera camera) {
	    					Native.previewCallback(data);
	    				}
	    			});
				}
    		} else{
			//old setPreviewCallback
    			camera.setPreviewCallback(new Camera.PreviewCallback() {

    				@Override
    				public void onPreviewFrame(byte[] data, Camera camera) {
    					Native.previewCallback(data);
    				}
    			});
    		}
    	} else {
    		camera.setPreviewCallback(null);
    	}
    }

//...other methods

The class above checks for the existence of the new Preview callback API in its static constructor. If it founds the new APIs, then it it will try to use them instead of the old APIs. All of these methods make sense only if we keep in mind that we want to control these events from native code. All functions ending in …FromNative will be called from native code. As it can be seen, the code will fall back to the old public APIs if something goes wrong with the new one. This might or might not what you want in your app.

We have to create a native Camera class that calls back to Java on Android 2.1 and above. Let’s call this class JCamera. It is very easy to implement, because it does nothing else just calls back to the appropriate method in CameraPreview.java. So let’s take a look instead at the changes of LoadManager.cpp.

LoadManager.cpp


MyICam* LoadManager::createCam(){
	char *error=NULL;
	if(fallback){
		return new JCamera;
	} else{
		if(!handle){
			char dir[150];
			switch(SDK){
			case -1:
				LOGE("SDK is not set!");
				fallback = true;
				return new JCamera;
				break;
			case 3:
				snprintf(dir, 150, "%s/lib/libcupcakecam.so",dataDir);
				handle = dlopen(dir, RTLD_LAZY);
				if(!handle){
					fallback = true;
					return new JCamera;
				}
				break;
			case 4:
				snprintf(dir, 150, "%s/lib/libdonutcam.so",dataDir);
				handle = dlopen(dir, RTLD_LAZY);
				if(!handle){
					fallback = true;
					return new JCamera;
				}
				break;
			}

			createCamera = (createCamera_t*) dlsym(handle, "create");
			const char* dlsym_error = dlerror();
			if(dlsym_error){				
				fallback = true;
				handle = NULL;
				return new JCamera;
			}

			destroyCamera = (destroyCamera_t*) dlsym(handle, "destroy");
			dlsym_error = dlerror();
			if (dlsym_error) {
				fallback = true;
				handle = NULL;
				return new JCamera;
			}
		}
		return createCamera();
	}
}

void LoadManager::destroyCam(MyICam* iCam){
	if(fallback){
		delete iCam;
	} else{
		destroyCamera(iCam);
	}
}

Summary

The problem was to find a better way to get access to the preview frames from the camera than what the official API offered, as it had a major flaw under Android 2.2.
The solution was to use private APIs, specific to 1.5 and 1.6 versions, and to use reflection on 2.1. To avoid polluting the code base with dangerous private API calls we used the bridge pattern and dynamic loading to separate them into their own so files. The camera handling was managed from the native code at all steps even on 2.1 and above versions. This made available that the main code base remained untouched.