Need to play Video in ARCORE

Question

As we know in ARCore, we can detect 3d object while click on the horizontal plane surfaces. Instead of 3d object, I need to show Video when User is going to click the PLane Surfaces. The Look and feel should be same like 3d object is displaying. Instead of 3d object, the video should be displayed in preview mode.

In ARcore they are currently using one Relativelayout with Surfaceview. So for displaying Video, I am using the Surfaceview and attaching with mediaplayer.

public void onsurfacecreatedvideo(){
    mProgram = createProgram(mVertexShader, mFragmentShader);
    if (mProgram == 0) {
        return;
    }
    maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition");
    checkGlError("glGetAttribLocation aPosition");
    if (maPositionHandle == -1) {
        throw new RuntimeException("Could not get attrib location for aPosition");
    }
    maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord");
    checkGlError("glGetAttribLocation aTextureCoord");
    if (maTextureHandle == -1) {
        throw new RuntimeException("Could not get attrib location for aTextureCoord");
    }

    muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
    checkGlError("glGetUniformLocation uMVPMatrix");
    if (muMVPMatrixHandle == -1) {
        throw new RuntimeException("Could not get attrib location for uMVPMatrix");
    }

    muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix");
    checkGlError("glGetUniformLocation uSTMatrix");
    if (muSTMatrixHandle == -1) {
        throw new RuntimeException("Could not get attrib location for uSTMatrix");
    }


    int[] textures = new int[1];
    GLES20.glGenTextures(1, textures, 0);

    mTextureID = textures[0];
    GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID);
    checkGlError("glBindTexture mTextureID");

    GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
            GLES20.GL_NEAREST);
    GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
            GLES20.GL_LINEAR);

        /*
         * Create the SurfaceTexture that will feed this textureID,
         * and pass it to the MediaPlayer
         */
    mSurface = new SurfaceTexture(mTextureID);
    mSurface.setOnFrameAvailableListener(this);



    Surface surface = new Surface(mSurface);
    mMediaPlayer.setSurface(surface);
    mMediaPlayer.setScreenOnWhilePlaying(true);

    surface.release();

    mMediaPlayer.setOnPreparedListener(new MediaPlayer.OnPreparedListener() {
        @Override
        public void onPrepared(MediaPlayer mp) {
            Log.i(TAG,"ONPREPArED abhilash");
            setVideoSize();
            mp.start();
        }
    });
    try {
        mMediaPlayer.prepare();
    } catch (IOException t) {
        Log.e(TAG, "media player prepare failed");
    }

    synchronized(this) {
        updateSurface = false;
    }

    mMediaPlayer.start();

}

void ondrawvideo(){
    synchronized(this) {
        if (updateSurface) {
            mSurface.updateTexImage();
            mSurface.getTransformMatrix(mSTMatrix);
            updateSurface = false;
        }
    }

    /////////////
    GLES20.glClearColor(0.0f, 1.0f, 0.0f, 1.0f);
    GLES20.glClear( GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT);


    GLES20.glUseProgram(mProgram);
    checkGlError("glUseProgram");

    GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
    GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID);



    mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET);
    GLES20.glVertexAttribPointer(maPositionHandle, 3, GLES20.GL_FLOAT, false,
            TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
    checkGlError("glVertexAttribPointer maPosition");
    GLES20.glEnableVertexAttribArray(maPositionHandle);
    checkGlError("glEnableVertexAttribArray maPositionHandle");

    mTriangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET);
    GLES20.glVertexAttribPointer(maTextureHandle, 3, GLES20.GL_FLOAT, false,
            TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
    checkGlError("glVertexAttribPointer maTextureHandle");
    GLES20.glEnableVertexAttribArray(maTextureHandle);
    checkGlError("glEnableVertexAttribArray maTextureHandle");

    Matrix.setIdentityM(mMVPMatrix, 0);
    GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0);
    GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0);

    GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
    checkGlError("glDrawArrays");
    GLES20.glFinish();

}


        // Visualize planes.
        mPlaneRenderer.drawPlanes(mSession.getAllPlanes(), frame.getPose(), projmtx);


        // Visualize anchors created by touch.
        float scaleFactor = 1.0f;
        for (PlaneAttachment planeAttachment : mTouches) {
            ondrawvideo();
            if (!planeAttachment.isTracking()) {
                continue;
            }


            // Get the current combined pose of an Anchor and Plane in world space. The Anchor
            // and Plane poses are updated during calls to session.update() as ARCore refines
            // its estimate of the world.
            planeAttachment.getPose().toMatrix(mAnchorMatrix, 0);

            // Update and draw the model and its shadow.
            mVirtualObject.updateModelMatrix(mAnchorMatrix, scaleFactor);
            mVirtualObjectShadow.updateModelMatrix(mAnchorMatrix, scaleFactor);
            mVirtualObject.draw(viewmtx, projmtx, lightIntensity);
            mVirtualObjectShadow.draw(viewmtx, projmtx, lightIntensity);
        }

    } catch (Throwable t) {
        // Avoid crashing the application due to unhandled exceptions.
        Log.e(TAG, "Exception on the OpenGL thread", t);
    }
}

Currently my output is coming like this. When I am click on plane Surfaces, its showing like this:

enter image description here

As you can see, below Image, I need to achieve it like this. I just marked that in this particular bugdroid image the video should be played, the video should not be exceed full screen; it should only be shown just like the bugdroid image size:

enter image description here

Answer 1

I did this by creating a new class called MovieClipRenderer - which modeled after the ObjectRenderer class in the HelloAR sample. This creates a quad geometry and renders the texture from the media player in the quad. The quad is anchored to a plane, so it does not move as the user looks around.

To test with, I used a stock movie from: https://www.videvo.net/video/chicken-on-green-screen/3435/ and added it to src/main/assets

Then I added the member variable for the renderer to HelloArActivity

  private final MovieClipRenderer mMovieClipRenderer = new MovieClipRenderer();

In onSurfaceCreated() I initialized the renderer with the others

 mMovieClipRenderer.createOnGlThread();

To try it out, I made the first tap on a plane create the movie anchor by changing hit test code slightly to be:

if (mMovieAnchor == null) {
    mMovieAnchor = hit.createAnchor();
} else {
    mAnchors.add(hit.createAnchor());
}

Then at the bottom of onDrawFrame() I checked for the anchor and started playing it:

    if (mMovieAnchor != null) {
        // Draw chickens!
        if (!mMovieClipRenderer.isStarted()) {
            mMovieClipRenderer.play("chicken.mp4", this);
        }
        mMovieAnchor.getPose().toMatrix(mAnchorMatrix,0);
        mMovieClipRenderer.update(mAnchorMatrix, 0.25f);
        mMovieClipRenderer.draw(mMovieAnchor.getPose(), viewmtx, projmtx);
    }

The rendering class is pretty long, but is pretty standard GLES code of creating the OES texture and initializing the video player, creating the vertices of a quad and loading a fragment shader that draws an OES texture.

/**
 * Renders a movie clip with a green screen aware shader.
 * <p>
 * Copyright 2018 Google LLC
 * <p>
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * <p>
 * http://www.apache.org/licenses/LICENSE-2.0
 * <p>
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
public class MovieClipRenderer implements 
     SurfaceTexture.OnFrameAvailableListener {
  private static final String TAG = MovieClipRenderer.class.getSimpleName();

  // Quad geometry
  private static final int COORDS_PER_VERTEX = 3;
  private static final int TEXCOORDS_PER_VERTEX = 2;
  private static final int FLOAT_SIZE = 4;
  private static final float[] QUAD_COORDS = new float[]{
          -1.0f, -1.0f, 0.0f,
          -1.0f, +1.0f, 0.0f,
          +1.0f, -1.0f, 0.0f,
          +1.0f, +1.0f, 0.0f,
  };

  private static final float[] QUAD_TEXCOORDS = new float[]{
          0.0f, 1.0f,
          0.0f, 0.0f,
          1.0f, 1.0f,
          1.0f, 0.0f,
  };

  // Shader for a flat quad.
  private static final String VERTEX_SHADER =
      "uniform mat4 u_ModelViewProjection;\n\n" +
      "attribute vec4 a_Position;\n" +
      "attribute vec2 a_TexCoord;\n\n" +
      "varying vec2 v_TexCoord;\n\n" +
      "void main() {\n" +
      "   gl_Position = u_ModelViewProjection * vec4(a_Position.xyz, 1.0);\n" +
     "   v_TexCoord = a_TexCoord;\n" +
     "}";

  // The fragment shader samples the video texture, blending to
  //  transparent for the green screen
  //  color.  The color was determined by sampling a screenshot
  //  of the video in an image editor.
  private static final String FRAGMENT_SHADER =
      "#extension GL_OES_EGL_image_external : require\n" +
      "\n" +
      "precision mediump float;\n" +
      "varying vec2 v_TexCoord;\n" +
      "uniform samplerExternalOES sTexture;\n" +
      "\n" +
      "void main() {\n" +
      "    //TODO make this a uniform variable - " +
      " but this is the color of the background. 17ad2b\n" +
      "  vec3 keying_color = vec3(23.0f/255.0f, 173.0f/255.0f, 43.0f/255.0f);\n" +
      "  float thresh = 0.4f; // 0 - 1.732\n" +
      "  float slope = 0.2;\n" +
      "  vec3 input_color = texture2D(sTexture, v_TexCoord).rgb;\n" +
      "  float d = abs(length(abs(keying_color.rgb - input_color.rgb)));\n" +
      "  float edge0 = thresh * (1.0f - slope);\n" +
      "  float alpha = smoothstep(edge0,thresh,d);\n" +
      "  gl_FragColor = vec4(input_color, alpha);\n" +
      "}";

  // Geometry data in GLES friendly data structure.
  private FloatBuffer mQuadVertices;
  private FloatBuffer mQuadTexCoord;

  // Shader program id and parameters.
  private int mQuadProgram;
  private int mQuadPositionParam;
  private int mQuadTexCoordParam;
  private int mModelViewProjectionUniform;
  private int mTextureId = -1;

  // Matrix for the location and perspective of the quad.
  private float[] mModelMatrix = new float[16];

  // Media player,  texture and other bookkeeping.
  private MediaPlayer player;
  private SurfaceTexture videoTexture;
  private boolean frameAvailable = false;
  private boolean started = false;
  private boolean done;
  private boolean prepared;
  private static Handler handler;


  // Lock used for waiting if the player was not yet created.
  private final Object lock = new Object();

  /**
   * Update the model matrix based on the location and scale to draw the quad.
   */
  public void update(float[] modelMatrix, float scaleFactor) {
    float[] scaleMatrix = new float[16];
    Matrix.setIdentityM(scaleMatrix, 0);
    scaleMatrix[0] = scaleFactor;
    scaleMatrix[5] = scaleFactor;
    scaleMatrix[10] = scaleFactor;
    Matrix.multiplyMM(mModelMatrix, 0, modelMatrix, 0, scaleMatrix, 0);
  }

  /**
   * Initialize the GLES objects.  
   * This is called from the GL render thread to make sure
   * it has access to the EGLContext.
   */
  public void createOnGlThread() {

    // 1 texture to hold the video frame.
    int textures[] = new int[1];
    GLES20.glGenTextures(1, textures, 0);
    mTextureId = textures[0];
    int mTextureTarget = GLES11Ext.GL_TEXTURE_EXTERNAL_OES;
    GLES20.glBindTexture(mTextureTarget, mTextureId);
    GLES20.glTexParameteri(mTextureTarget, GLES20.GL_TEXTURE_WRAP_S,
       GLES20.GL_CLAMP_TO_EDGE);
    GLES20.glTexParameteri(mTextureTarget, GLES20.GL_TEXTURE_WRAP_T,
       GLES20.GL_CLAMP_TO_EDGE);
    GLES20.glTexParameteri(mTextureTarget, GLES20.GL_TEXTURE_MIN_FILTER,
       GLES20.GL_NEAREST);
    GLES20.glTexParameteri(mTextureTarget, GLES20.GL_TEXTURE_MAG_FILTER,
       GLES20.GL_NEAREST);

    videoTexture = new SurfaceTexture(mTextureId);
    videoTexture.setOnFrameAvailableListener(this);

    // Make a quad to hold the movie
    ByteBuffer bbVertices = ByteBuffer.allocateDirect(
         QUAD_COORDS.length * FLOAT_SIZE);
    bbVertices.order(ByteOrder.nativeOrder());
    mQuadVertices = bbVertices.asFloatBuffer();
    mQuadVertices.put(QUAD_COORDS);
    mQuadVertices.position(0);

    int numVertices = 4;
    ByteBuffer bbTexCoords = ByteBuffer.allocateDirect(
            numVertices * TEXCOORDS_PER_VERTEX * FLOAT_SIZE);
    bbTexCoords.order(ByteOrder.nativeOrder());
    mQuadTexCoord = bbTexCoords.asFloatBuffer();
    mQuadTexCoord.put(QUAD_TEXCOORDS);
    mQuadTexCoord.position(0);

    int vertexShader = loadGLShader(TAG, GLES20.GL_VERTEX_SHADER, VERTEX_SHADER);
    int fragmentShader = loadGLShader(TAG,
         GLES20.GL_FRAGMENT_SHADER, FRAGMENT_SHADER);

    mQuadProgram = GLES20.glCreateProgram();
    GLES20.glAttachShader(mQuadProgram, vertexShader);
    GLES20.glAttachShader(mQuadProgram, fragmentShader);
    GLES20.glLinkProgram(mQuadProgram);
    GLES20.glUseProgram(mQuadProgram);

    ShaderUtil.checkGLError(TAG, "Program creation");

    mQuadPositionParam = GLES20.glGetAttribLocation(mQuadProgram, "a_Position");
    mQuadTexCoordParam = GLES20.glGetAttribLocation(mQuadProgram, "a_TexCoord");
    mModelViewProjectionUniform = GLES20.glGetUniformLocation(
            mQuadProgram, "u_ModelViewProjection");

    ShaderUtil.checkGLError(TAG, "Program parameters");

    Matrix.setIdentityM(mModelMatrix, 0);

    initializeMediaPlayer();
  }

  public void draw(Pose pose, float[] cameraView, float[] cameraPerspective) {
    if (done || !prepared) {
      return;
    }
    synchronized (this) {
      if (frameAvailable) {
        videoTexture.updateTexImage();
        frameAvailable = false;
      }
    }

    float[] modelMatrix = new float[16];
    pose.toMatrix(modelMatrix, 0);

    float[] modelView = new float[16];
    float[] modelViewProjection = new float[16];
    Matrix.multiplyMM(modelView, 0, cameraView, 0, mModelMatrix, 0);
    Matrix.multiplyMM(modelViewProjection, 0, cameraPerspective, 0, modelView, 0);

    ShaderUtil.checkGLError(TAG, "Before draw");

    GLES20.glEnable(GL10.GL_BLEND);
    GLES20.glBlendFunc(GL10.GL_SRC_ALPHA, GL10.GL_ONE_MINUS_SRC_ALPHA);
    GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureId);
    GLES20.glUseProgram(mQuadProgram);

    // Set the vertex positions.
    GLES20.glVertexAttribPointer(
            mQuadPositionParam, COORDS_PER_VERTEX, GLES20.GL_FLOAT,
            false, 0, mQuadVertices);
    // Set the texture coordinates.
    GLES20.glVertexAttribPointer(mQuadTexCoordParam, TEXCOORDS_PER_VERTEX,
            GLES20.GL_FLOAT, false, 0, mQuadTexCoord);

    // Enable vertex arrays
    GLES20.glEnableVertexAttribArray(mQuadPositionParam);
    GLES20.glEnableVertexAttribArray(mQuadTexCoordParam);
    GLES20.glUniformMatrix4fv(mModelViewProjectionUniform, 1, false,
                              modelViewProjection, 0);

    GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);

    // Disable vertex arrays
    GLES20.glDisableVertexAttribArray(mQuadPositionParam);
    GLES20.glDisableVertexAttribArray(mQuadTexCoordParam);

    ShaderUtil.checkGLError(TAG, "Draw");
  }

  private void initializeMediaPlayer() {
    if (handler == null)
      handler = new Handler(Looper.getMainLooper());

    handler.post(new Runnable() {
      @Override
      public void run() {
        synchronized (lock) {
          player = new MediaPlayer();
          lock.notify();
        }
      }
    });
  }

  @Override
  public void onFrameAvailable(SurfaceTexture surfaceTexture) {
    synchronized (this) {
      frameAvailable = true;
    }
  }

  public boolean play(final String filename, Context context)
                     throws FileNotFoundException {
    // Wait for the player to be created.
    if (player == null) {
      synchronized (lock) {
        while (player == null) {
          try {
            lock.wait();
          } catch (InterruptedException e) {
            return false;
          }
        }
      }
    }

    player.reset();
    done = false;

    player.setOnPreparedListener(new MediaPlayer.OnPreparedListener() {
      @Override
      public void onPrepared(MediaPlayer mp) {
        prepared = true;
        mp.start();
      }
    });
    player.setOnErrorListener(new MediaPlayer.OnErrorListener() {
      @Override
      public boolean onError(MediaPlayer mp, int what, int extra) {
        done = true;
        Log.e("VideoPlayer",
            String.format("Error occured: %d, %d\n", what, extra));
        return false;
      }
    });

    player.setOnCompletionListener(new MediaPlayer.OnCompletionListener() {
      @Override
      public void onCompletion(MediaPlayer mp) {
        done = true;
      }
    });

    player.setOnInfoListener(new MediaPlayer.OnInfoListener() {
      @Override
      public boolean onInfo(MediaPlayer mediaPlayer, int i, int i1) {
        return false;
      }
    });

    try {
      AssetManager assets = context.getAssets();
      AssetFileDescriptor descriptor = assets.openFd(filename);
      player.setDataSource(descriptor.getFileDescriptor(),
                           descriptor.getStartOffset(),
                           descriptor.getLength());
      player.setSurface(new Surface(videoTexture));
      player.prepareAsync();
      synchronized (this) {
        started = true;
      }
    } catch (IOException e) {
      Log.e(TAG, "Exception preparing movie", e);
      return false;
    }

    return true;
  }

  public synchronized boolean isStarted() {
    return started;
  }

  static int loadGLShader(String tag, int type, String code) {
    int shader = GLES20.glCreateShader(type);
    GLES20.glShaderSource(shader, code);
    GLES20.glCompileShader(shader);

    // Get the compilation status.
    final int[] compileStatus = new int[1];
    GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compileStatus, 0);

    // If the compilation failed, delete the shader.
    if (compileStatus[0] == 0) {
      Log.e(tag, "Error compiling shader: " + GLES20.glGetShaderInfoLog(shader));
      GLES20.glDeleteShader(shader);
      shader = 0;
    }

    if (shader == 0) {
      throw new RuntimeException("Error creating shader.");
    }

    return shader;
  }
}

Answer 2

If you are in need to play a video on ar plane using sceneform, create video node and add media player to it. Refer to this link- https://github.com/SceneView/sceneform-android/blob/master/samples/video-texture/src/main/java/com/google/ar/sceneform/samples/videotexture/MainActivity.java