I want to integrate flutter app with ArGear SDK.
There is no package or plugin to use this SDK directly in flutter, and we should implement SDK in platform side and call corresponding method with method channel.
I want to open camera in platform side and perform image processing on the platform side, then show camera that implemented with frame layout beside flutter widgets with platform view.
Implementation worked fine in an android native project. But a problem occurred after move implementation code to flutter app in platform side. app running but screen is black and there is no error in console.
Error fixed after comment initGLView() method and setContentView(R.layout.camera_activity); in code.
SDK has a listener to detect camera, I tested another way and open camera in flutter with camera plugin but SDK couldn't detect camera and in this case i should open camera in platform side.
Implementation in android platform:
public static final String CHANNEL = "flutter.native/argear";
public static final String STREAM = "argear";
public static final String API_URL = "";
public static final String API_KEY = "";
public static final String SECRET_KEY = "";
public static final String AUTH_KEY = "";
private ARGFrame.Ratio mScreenRatio = ARGFrame.Ratio.RATIO_FULL;
private boolean mUseARGSessionDestroy = false;
private GLView mGlView;
private ReferenceCamera mCamera;
private ARGConfig config;
private Set<ARGInferenceConfig.Feature> inferenceConfig;
private ARGSession argsession;
private ARGMedia argMedia;
private ScreenRenderer mScreenRenderer;
private CameraTexture mCameraTexture;
private CameraActivityBinding mDataBinding;
private boolean mIsShooting = false;
// preview size
private int mDeviceWidth = 0;
private int mDeviceHeight = 0;
private int mGLViewWidth = 0;
private int mGLViewHeight = 0;
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
//todo : this method cause error on flutter app
setContentView(R.layout.camera_activity);
Point realSize = new Point();
Display display = ((WindowManager) this.getSystemService(Context.WINDOW_SERVICE)).getDefaultDisplay();
display.getRealSize(realSize);
mDeviceWidth = realSize.x;
mDeviceHeight = realSize.y;
mGLViewWidth = realSize.x;
mGLViewHeight = realSize.y;
//todo : this method cause error on flutter app
mDataBinding = CameraActivityBinding.inflate(getLayoutInflater());
View view = mDataBinding.getRoot();
setContentView(view);
}
@Override
protected void onResume() {
super.onResume();
if (argsession == null) {
if (!PermissionHelper.hasPermission(this)) {
if (PermissionHelper.shouldShowRequestPermissionRationale(this)) {
Toast.makeText(this, "Please check your permissions!", Toast.LENGTH_SHORT).show();
return;
}
PermissionHelper.requestPermission(this);
return;
}
config = new ARGConfig(API_URL, API_KEY, SECRET_KEY, AUTH_KEY);
inferenceConfig
= EnumSet.of(ARGInferenceConfig.Feature.FACE_MESH_TRACKING);
argsession = new ARGSession(this, config, inferenceConfig);
argMedia = new ARGMedia(argsession);
mScreenRenderer = new ScreenRenderer();
mCameraTexture = new CameraTexture();
//todo : this method cause error on flutter app
initGLView();
initCamera();
}
try {
mCamera.startCamera();
} catch (Exception e) {
System.out.println("error is : " + e);
}
argsession.resume();
setGLViewSize(mCamera.getPreviewSize());
addFaceEffect();
}
public void addFaceEffect() {
argsession.contents().setBulge(ARGContents.BulgeType.FUN6);
}
public void changeCameraFace() {
argsession.pause();
mCamera.changeCameraFacing();
argsession.resume();
}
public void setMeasureSurfaceView(View view) {
if (view.getParent() instanceof FrameLayout) {
view.setLayoutParams(new FrameLayout.LayoutParams(mGLViewWidth, mGLViewHeight));
} else if (view.getParent() instanceof RelativeLayout) {
view.setLayoutParams(new RelativeLayout.LayoutParams(mGLViewWidth, mGLViewHeight));
}
/* to align center */
if ((mScreenRatio == ARGFrame.Ratio.RATIO_FULL) && (mGLViewWidth > mDeviceWidth)) {
view.setX((mDeviceWidth - mGLViewWidth) / 2);
} else {
view.setX(0);
}
}
public int getGLViewWidth() {
return mGLViewWidth;
}
public int getGLViewHeight() {
return mGLViewHeight;
}
private void setGLViewSize(int[] cameraPreviewSize) {
int previewWidth = cameraPreviewSize[1];
int previewHeight = cameraPreviewSize[0];
if (mScreenRatio == ARGFrame.Ratio.RATIO_FULL) {
mGLViewHeight = mDeviceHeight;
mGLViewWidth = (int) ((float) mDeviceHeight * previewWidth / previewHeight);
} else {
mGLViewWidth = mDeviceWidth;
mGLViewHeight = (int) ((float) mDeviceWidth * previewHeight / previewWidth);
}
if (mGlView != null
&& (mGLViewWidth != mGlView.getViewWidth() || mGLViewHeight != mGlView.getHeight())) {
mDataBinding.cameraLayout.removeView(mGlView);
mGlView.getHolder().setFixedSize(mGLViewWidth, mGLViewHeight);
mDataBinding.cameraLayout.addView(mGlView);
}
}
@Override
protected void onPause() {
super.onPause();
if (argsession != null) {
mCamera.stopCamera();
argsession.pause();
}
}
@Override
protected void onDestroy() {
super.onDestroy();
if (argsession != null) {
mCamera.destroy();
mUseARGSessionDestroy = true;
}
}
//todo : this method cause error on flutter app
private void initGLView() {
final FrameLayout cameraLayout = findViewById(R.id.camera_layout);
FrameLayout.LayoutParams params = new FrameLayout.LayoutParams(ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT);
mGlView = new GLView(this, glViewListener);
mGlView.setZOrderMediaOverlay(true);
cameraLayout.addView(mGlView, params);
}
private void initCamera() {
mCamera = new ReferenceCamera1(this, cameraListener);
}
ReferenceCamera.CameraListener cameraListener = new ReferenceCamera.CameraListener() {
@Override
public void setConfig(int previewWidth,
int previewHeight,
float verticalFov,
float horizontalFov,
int orientation,
boolean isFrontFacing,
float fps) {
argsession.setCameraConfig(new ARGCameraConfig(previewWidth,
previewHeight,
verticalFov,
horizontalFov,
orientation,
isFrontFacing,
fps));
}
@Override
public void feedRawData(byte[] data) {
Log.d("FEED_RAW_DATA", "feedRawData: Send preview frame raw data from camera device to ARGear");
System.out.println("****************************");
System.out.println("camera detected");
System.out.println("****************************");
// Send preview frame raw data from camera device to ARGear
argsession.feedRawData(data);
}
@Override
public void feedRawData(Image data) {
argsession.feedRawData(data);
}
};
GLView.GLViewListener glViewListener = new GLView.GLViewListener() {
@Override
public void onSurfaceCreated(GL10 gl, EGLConfig config) {
mScreenRenderer.create(gl, config);
mCameraTexture.createCameraTexture();
}
@Override
public void onDrawFrame(GL10 gl, int width, int height) {
if (mCameraTexture.getSurfaceTexture() == null) {
return;
}
if (mCamera != null) {
mCamera.setCameraTexture(mCameraTexture.getTextureId(), mCameraTexture.getSurfaceTexture());
}
ARGFrame frame = argsession.drawFrame(gl, mScreenRatio, width, height);
mScreenRenderer.draw(frame, ARGFrame.Ratio.RATIO_4_3, width, height);
if (argMedia != null) {
if (argMedia.isRecording()) argMedia.updateFrame(frame.getTextureId());
}
// check destroy boolean to destroy arg session
if (mUseARGSessionDestroy)
argsession.destroy();
}
};
