Combine EncodeAndMux with HelloEffects in Android

Question

I am trying to create a video with a series of images using MediaCodec and MediaMuxer. I want to also apply an Effect on each Bitmap before the encoding.

The video gets created with the correct duration but the output is only black screen.

I have been looking at the EncodeAndMux example at bigflake.com (Modified the generateFrame() and added a shared context when creating the CodecInputSurface )

and the HelloEffects from Google to setup the EffectFactory.

My idea was to create 2 EGLContexts - one for rendering the effect and one for the MediaCodec(encoder) and swapping the buffers

I am new to OpenGl and Android so please tell me if I am going about this the wrong way. Any help with helping me understand what the problem is would be greatly appreciated

Here is my code :

public class WriteMovieActivity extends Activity {

private static final String TAG = "WRITE MOVIE";
private static final File OUTPUT_DIR = Environment.getExternalStorageDirectory();

private int BPM;
private int numberOfPhotos;
private List durations;
private int durationInSec;
private Cursor mCursor;
private MyApplication mApp;
private String filterName;
private String size;
private int firstImageIndex;
private int lastImageIndex;
private String albumName;
private String orientation;
private int[] imageIDs;

private static final String MIME_TYPE = "video/avc";    // H.264 Advanced Video Coding
private static final int FRAME_RATE = 10;               // 10fps
private static final int IFRAME_INTERVAL = 5;           // 5 seconds between I-frames
private ImageLoader loader;
private CodecInputSurface mInputSurface;
private MediaMuxer mMuxer;
private MediaCodec mEncoder;
private MediaCodec.BufferInfo mBufferInfo;
private int mBitRate = -1;
private int mTrackIndex;
private boolean mMuxerStarted;

private int mWidth;
private int mHeight;
private int mImageWidth;
private int mImageHeight;

private DisplayImageOptions options;
private long presentationTime;
private long durationInNanosec;

private int[] mTextures = new int[2];
private int texId = -1;
private EffectContext mEffectcontext;
private Effect mEffect;
private TextureRenderer mTexRenderer = new TextureRenderer();

private GLEnv mEnv;

@Override
protected void onCreate(Bundle savedInstanceState) {
    super.onCreate(savedInstanceState);
    setContentView(R.layout.activity_write_movie);

    mApp = (MyApplication) getApplication();

    firstImageIndex = mApp.getFirstImageIndex();
    lastImageIndex = mApp.getLastImageIndex();
    numberOfPhotos = lastImageIndex - firstImageIndex +1 ;
    Log.v(TAG,"FirstImage " + firstImageIndex +"------ lastImage "+ lastImageIndex );

    albumName = mApp.getAlbumName();
    filterName = mApp.getFilter();
    size = mApp.getSize();
    orientation = mApp.getOrientation();

    loader = ImageLoader.getInstance();


    if (size == "small") {
        mBitRate = 1000000;
        if (orientation == "portrait") {
            mWidth = 144;
            mHeight = 176;
        } else {
            mWidth = 176;
            mHeight = 144;
        }
    } else if (size == "default") {
        mBitRate = 2000000;
        if (orientation == "portrait") {
            mWidth = 240;
            mHeight = 320;
        } else {
            mWidth = 320;
            mHeight = 240;
        }
    } else {
        mBitRate = 6000000;
        if (orientation == "portrait") {
            mWidth = 720;
            mHeight = 1280;
        } else {
            mWidth = 1280;
            mHeight = 720;
        }
    }

    //Get the image ids for bucket
    imageIDs = getImagesFromBucket(albumName);

    BitmapFactory.Options resizeOptions = new BitmapFactory.Options();
    resizeOptions.inSampleSize = 1; // decrease size 1 time
    resizeOptions.inScaled = true;

    options = new DisplayImageOptions.Builder()
            .cacheOnDisk(true)
            .considerExifParams(true)
            .bitmapConfig(Bitmap.Config.RGB_565)
            .decodingOptions(resizeOptions)
            .postProcessor(new BitmapProcessor() {
                @Override
                public Bitmap process(Bitmap bmp) {

                    return scaleBitmap(bmp);
                }
            })
            .build();


    BPM = Math.round(mApp.getBPM());

    durationInSec = mApp.getDuration();
    if (durationInSec == 0) {
        //TODO: Get the duration of the song
    }

    durations = getDurationsArray(durationInSec);

    mEnv = new GLEnv();
    mEnv.makeCurrent();
    mEffectcontext = EffectContext.createWithCurrentGlContext();
    mTexRenderer.init();



}

private Bitmap scaleBitmap(Bitmap bm) {
    int width = bm.getWidth();
    int height = bm.getHeight();

    //Log.v("Pictures", "Width and height are " + width + "--" + height);

    if (width > height) {
        // landscape
        float ratio = (float) width / mWidth;
        width = mWidth;
        height = (int)(height / ratio);
    } else if (height > width) {
        // portrait
        float ratio = (float) height / mHeight;
        height = mHeight;
        width = (int)(width / ratio);
    } else {
        // square
        height = mHeight;
        width = mWidth;
    }

    //Log.v("Pictures", "after scaling Width and height are " + width + "--" + height);

    bm = Bitmap.createScaledBitmap(bm, width, height, true);
    return bm;
}


public void createMovie(View view) throws  Throwable {

    Log.v(TAG,"CREATING MOVIE");


    try {
        prepareEncoder();

        presentationTime = 0;
        int j = 0;

        for (int i = firstImageIndex ; i <= lastImageIndex; i++) {

            drainEncoder(false);

            generateSurfaceFrame(i);

            mInputSurface.setPresentationTime(presentationTime);

            Log.v(TAG,"sending frame " + i + " to encoder");
            mInputSurface.swapBuffers();

            //Convert frame duration from milliseconds to nanoseconds
            durationInNanosec = (long)((float)durations.get(j) * 1000000);

            presentationTime += durationInNanosec;
            j++;


        }
        drainEncoder(true);

    } catch(Throwable e) {
        Log.e(TAG,e.getMessage(),e);
    } finally {
        // release encoder, muxer, and input Surface
        releaseEncoder();
        Log.v(TAG,"VIDEO CREATED");
        Toast.makeText(this, "Video Created!",
                Toast.LENGTH_LONG).show();
    }
}

private void generateSurfaceFrame(int frameIndex) {

    mEnv.makeCurrent();

    if (texId >= 0){
        mEnv.releaseTexture(texId);
    }
    int imageID = imageIDs[frameIndex];
    Uri imageURI = Uri.withAppendedPath(MediaStore.Images.Media.EXTERNAL_CONTENT_URI, "" + imageID);
    texId = loadTextures(loader.loadImageSync(imageURI.toString()));
    initAndApplyEffect();
    renderResult();

    GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
    ByteBuffer pixelBuffer = ByteBuffer.allocateDirect(mImageWidth * mImageHeight * 4).order(ByteOrder.nativeOrder());
    GLES20.glReadPixels(0, 0, mImageWidth, mImageHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, pixelBuffer);
    GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
    mEnv.checkForEGLErrors("store Pixels");


    mInputSurface.makeCurrent();


}

private void initAndApplyEffect()
{
    EffectFactory effectFactory = mEffectcontext.getFactory();
    if (mEffect != null)
    {
        mEffect.release();
    }
    mEffect = effectFactory.createEffect(EffectFactory.EFFECT_BRIGHTNESS);
    mEffect.setParameter("brightness", 2.0f);
    mEffect.apply(mTextures[0], mWidth, mHeight, mTextures[1]);

}

private int loadTextures(Bitmap bitmap)
{
    // Generate textures
    GLES20.glGenTextures(2, mTextures, 0);

    mImageWidth = bitmap.getWidth();
    mImageHeight = bitmap.getHeight();
    mTexRenderer.updateTextureSize(mImageWidth, mImageHeight);


    // Upload to texture
    GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextures[0]);
    GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, bitmap, 0);

    // Set texture parameters
    GLToolbox.initTexParams();

    return mTextures[0];
}

private void renderResult()
{
    mTexRenderer.renderTexture(mTextures[1]);
    //mTexRenderer.renderTexture(mTextures[0]);
}

private void prepareEncoder() {
    mBufferInfo = new MediaCodec.BufferInfo();

    MediaFormat format = MediaFormat.createVideoFormat(MIME_TYPE, mWidth, mHeight);

    // Set some properties.  Failing to specify some of these can cause the MediaCodec
    // configure() call to throw an unhelpful exception.
    format.setInteger(MediaFormat.KEY_COLOR_FORMAT,
            MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
    format.setInteger(MediaFormat.KEY_BIT_RATE, mBitRate);
    format.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE);
    format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL);
    Log.v(TAG, "format: " + format);

    // Create a MediaCodec encoder, and configure it with our format.  Get a Surface
    // we can use for input and wrap it with a class that handles the EGL work.
    //
    // If you want to have two EGL contexts -- one for display, one for recording --
    // you will likely want to defer instantiation of CodecInputSurface until after the
    // "display" EGL context is created, then modify the eglCreateContext call to
    // take eglGetCurrentContext() as the share_context argument.
    try {
        mEncoder = MediaCodec.createEncoderByType(MIME_TYPE);
    } catch (IOException e) {
        e.printStackTrace();
    }
    mEncoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
    mInputSurface = new CodecInputSurface(mEncoder.createInputSurface());
    mEncoder.start();

    // Output filename.  Ideally this would use Context.getFilesDir() rather than a
    // hard-coded output directory.
    String outputPath = new File(OUTPUT_DIR,
            "test." + mWidth + "x" + mHeight + ".mp4").toString();
    Log.d(TAG, "output file is " + outputPath);


    // Create a MediaMuxer.  We can't add the video track and start() the muxer here,
    // because our MediaFormat doesn't have the Magic Goodies.  These can only be
    // obtained from the encoder after it has started processing data.
    //
    // We're not actually interested in multiplexing audio.  We just want to convert
    // the raw H.264 elementary stream we get from MediaCodec into a .mp4 file.
    try {
        mMuxer = new MediaMuxer(outputPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
    } catch (IOException ioe) {
        throw new RuntimeException("MediaMuxer creation failed", ioe);
    }

    mTrackIndex = -1;
    mMuxerStarted = false;
}

private void releaseEncoder() {
    Log.v(TAG, "releasing encoder objects");
    if (mEncoder != null) {
        mEncoder.stop();
        mEncoder.release();
        mEncoder = null;
    }
    if (mInputSurface != null) {
        mInputSurface.release();
        mInputSurface = null;
    }
    if (mMuxer != null) {
        mMuxer.stop();
        mMuxer.release();
        mMuxer = null;
    }
}

private void drainEncoder(boolean endOfStream) {
    final int TIMEOUT_USEC = 10000;
    Log.v(TAG, "drainEncoder(" + endOfStream + ")");

    if (endOfStream) {
        Log.v(TAG, "sending EOS to encoder");
        mEncoder.signalEndOfInputStream();
    }

    ByteBuffer[] encoderOutputBuffers = mEncoder.getOutputBuffers();
    while (true) {
        int encoderStatus = mEncoder.dequeueOutputBuffer(mBufferInfo, TIMEOUT_USEC);
        if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
            // no output available yet
            if (!endOfStream) {
                break;      // out of while
            } else {
                Log.v(TAG, "no output available, spinning to await EOS");
            }
        } else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
            // not expected for an encoder
            encoderOutputBuffers = mEncoder.getOutputBuffers();
        } else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
            // should happen before receiving buffers, and should only happen once
            if (mMuxerStarted) {
                throw new RuntimeException("format changed twice");
            }
            MediaFormat newFormat = mEncoder.getOutputFormat();
            Log.d(TAG, "encoder output format changed: " + newFormat);

            // now that we have the Magic Goodies, start the muxer
            mTrackIndex = mMuxer.addTrack(newFormat);
            mMuxer.start();
            mMuxerStarted = true;
        } else if (encoderStatus < 0) {
            Log.w(TAG, "unexpected result from encoder.dequeueOutputBuffer: " +
                    encoderStatus);
            // let's ignore it
        } else {
            ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
            if (encodedData == null) {
                throw new RuntimeException("encoderOutputBuffer " + encoderStatus +
                        " was null");
            }

            if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
                // The codec config data was pulled out and fed to the muxer when we got
                // the INFO_OUTPUT_FORMAT_CHANGED status.  Ignore it.
                Log.v(TAG, "ignoring BUFFER_FLAG_CODEC_CONFIG");
                mBufferInfo.size = 0;
            }

            if (mBufferInfo.size != 0) {
                if (!mMuxerStarted) {
                    throw new RuntimeException("muxer hasn't started");
                }

                // adjust the ByteBuffer values to match BufferInfo (not needed?)
                encodedData.position(mBufferInfo.offset);
                encodedData.limit(mBufferInfo.offset + mBufferInfo.size);

                mMuxer.writeSampleData(mTrackIndex, encodedData, mBufferInfo);
                Log.v(TAG, "sent " + mBufferInfo.size + " bytes to muxer");
            }

            mEncoder.releaseOutputBuffer(encoderStatus, false);

            if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
                if (!endOfStream) {
                    Log.w(TAG, "reached end of stream unexpectedly");
                } else {
                    Log.v(TAG, "end of stream reached");
                }
                break;      // out of while
            }
        }
    }
}

private static class CodecInputSurface {
    private static final int EGL_RECORDABLE_ANDROID = 0x3142;

    private EGLDisplay mEGLDisplay = EGL14.EGL_NO_DISPLAY;
    private EGLContext mEGLContext = EGL14.EGL_NO_CONTEXT;
    private EGLSurface mEGLSurface = EGL14.EGL_NO_SURFACE;

    private Surface mSurface;

    /**
     * Creates a CodecInputSurface from a Surface.
     */
    public CodecInputSurface(Surface surface) {
        if (surface == null) {
            throw new NullPointerException();
        }
        mSurface = surface;

        eglSetup();
    }

    /**
     * Prepares EGL.  We want a GLES 2.0 context and a surface that supports recording.
     */
    private void eglSetup() {
        mEGLDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY);
        if (mEGLDisplay == EGL14.EGL_NO_DISPLAY) {
            throw new RuntimeException("unable to get EGL14 display");
        }
        int[] version = new int[2];
        if (!EGL14.eglInitialize(mEGLDisplay, version, 0, version, 1)) {
            throw new RuntimeException("unable to initialize EGL14");
        }

        // Configure EGL for recording and OpenGL ES 2.0.
        int[] attribList = {
                EGL14.EGL_RED_SIZE, 8,
                EGL14.EGL_GREEN_SIZE, 8,
                EGL14.EGL_BLUE_SIZE, 8,
                EGL14.EGL_ALPHA_SIZE, 8,
                EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT,
                EGL_RECORDABLE_ANDROID, 1,
                EGL14.EGL_NONE
        };
        EGLConfig[] configs = new EGLConfig[1];
        int[] numConfigs = new int[1];
        EGL14.eglChooseConfig(mEGLDisplay, attribList, 0, configs, 0, configs.length,
                numConfigs, 0);
        checkEglError("eglCreateContext RGB888+recordable ES2");

        // Configure context for OpenGL ES 2.0.
        int[] attrib_list = {
                EGL14.EGL_CONTEXT_CLIENT_VERSION, 2,
                EGL14.EGL_NONE
        };
        mEGLContext = EGL14.eglCreateContext(mEGLDisplay, configs[0], EGL14.eglGetCurrentContext(),
                attrib_list, 0);
        checkEglError("eglCreateContext");

        // Create a window surface, and attach it to the Surface we received.
        int[] surfaceAttribs = {
                EGL14.EGL_NONE
        };
        mEGLSurface = EGL14.eglCreateWindowSurface(mEGLDisplay, configs[0], mSurface,
                surfaceAttribs, 0);
        checkEglError("eglCreateWindowSurface");
    }

    /**
     * Discards all resources held by this class, notably the EGL context.  Also releases the
     * Surface that was passed to our constructor.
     */
    public void release() {
        if (mEGLDisplay != EGL14.EGL_NO_DISPLAY) {
            EGL14.eglMakeCurrent(mEGLDisplay, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_SURFACE,
                    EGL14.EGL_NO_CONTEXT);
            EGL14.eglDestroySurface(mEGLDisplay, mEGLSurface);
            EGL14.eglDestroyContext(mEGLDisplay, mEGLContext);
            EGL14.eglReleaseThread();
            EGL14.eglTerminate(mEGLDisplay);
        }

        mSurface.release();

        mEGLDisplay = EGL14.EGL_NO_DISPLAY;
        mEGLContext = EGL14.EGL_NO_CONTEXT;
        mEGLSurface = EGL14.EGL_NO_SURFACE;

        mSurface = null;
    }

    /**
     * Makes our EGL context and surface current.
     */
    public void makeCurrent() {
        EGL14.eglMakeCurrent(mEGLDisplay, mEGLSurface, mEGLSurface, mEGLContext);
        checkEglError("eglMakeCurrent");
    }

    /**
     * Calls eglSwapBuffers.  Use this to "publish" the current frame.
     */
    public boolean swapBuffers() {
        boolean result = EGL14.eglSwapBuffers(mEGLDisplay, mEGLSurface);
        checkEglError("eglSwapBuffers");
        return result;
    }

    /**
     * Sends the presentation time stamp to EGL.  Time is expressed in nanoseconds.
     */
    public void setPresentationTime(long nsecs) {
        EGLExt.eglPresentationTimeANDROID(mEGLDisplay, mEGLSurface, nsecs);
        checkEglError("eglPresentationTimeANDROID");
    }

    /**
     * Checks for EGL errors.  Throws an exception if one is found.
     */
    private void checkEglError(String msg) {
        int error;
        if ((error = EGL14.eglGetError()) != EGL14.EGL_SUCCESS) {
            throw new RuntimeException(msg + ": EGL error: 0x" + Integer.toHexString(error));
        }
    }
}

}

This is the logcat:

05-06 17:27:19.871  17114-17114/com.example.andreaskaitis.myapplication V/WRITE MOVIE﹕ drainEncoder(false)
05-06 17:27:19.876  17114-17114/com.example.andreaskaitis.myapplication V/WRITE MOVIE﹕ sent 10 bytes to muxer
05-06 17:27:19.926  17114-17114/com.example.andreaskaitis.myapplication D/dalvikvm﹕ GC_FOR_ALLOC freed 51250K, 55% free 24821K/54524K, paused 31ms, total 31ms
05-06 17:27:20.011  17114-17114/com.example.andreaskaitis.myapplication I/dalvikvm-heap﹕ Grow heap (frag case) to 62.052MB for 38340880-byte allocation
05-06 17:27:20.021  17114-17124/com.example.andreaskaitis.myapplication D/dalvikvm﹕ GC_FOR_ALLOC freed <1K, 33% free 62263K/91968K, paused 13ms, total 13ms
05-06 17:27:20.286  17114-17114/com.example.andreaskaitis.myapplication D/dalvikvm﹕ GC_FOR_ALLOC freed 48K, 33% free 62215K/91968K, paused 9ms, total 10ms
05-06 17:27:20.336  17114-17114/com.example.andreaskaitis.myapplication I/dalvikvm-heap﹕ Grow heap (frag case) to 87.003MB for 26211856-byte allocation
05-06 17:27:20.346  17114-17124/com.example.andreaskaitis.myapplication D/dalvikvm﹕ GC_FOR_ALLOC freed <1K, 26% free 87813K/117568K, paused 12ms, total 12ms
05-06 17:27:20.491  17114-17114/com.example.andreaskaitis.myapplication D/dalvikvm﹕ GC_FOR_ALLOC freed 37454K, 58% free 50370K/117568K, paused 15ms, total 15ms
05-06 17:27:20.506  17114-17114/com.example.andreaskaitis.myapplication I/dalvikvm-heap﹕ Grow heap (frag case) to 75.435MB for 26211856-byte allocation
05-06 17:27:20.521  17114-17114/com.example.andreaskaitis.myapplication D/dalvikvm﹕ GC_FOR_ALLOC freed <1K, 36% free 75967K/117568K, paused 13ms, total 13ms
05-06 17:27:20.521  17114-17114/com.example.andreaskaitis.myapplication V/WRITE MOVIE﹕ sending frame 58 to encoder
05-06 17:27:20.521  17114-17114/com.example.andreaskaitis.myapplication V/WRITE MOVIE﹕ drainEncoder(true)
05-06 17:27:20.521  17114-17114/com.example.andreaskaitis.myapplication V/WRITE MOVIE﹕ sending EOS to encoder
05-06 17:27:20.526  17114-17114/com.example.andreaskaitis.myapplication V/WRITE MOVIE﹕ sent 10 bytes to muxer
05-06 17:27:20.531  17114-17114/com.example.andreaskaitis.myapplication V/WRITE MOVIE﹕ end of stream reached
05-06 17:27:20.531  17114-17114/com.example.andreaskaitis.myapplication V/WRITE MOVIE﹕ releasing encoder objects
05-06 17:27:20.531  17114-17709/com.example.andreaskaitis.myapplication I/ACodec﹕ [OMX.Exynos.AVC.Encoder] Now Executing->Idle
05-06 17:27:20.541  17114-17709/com.example.andreaskaitis.myapplication I/ACodec﹕ [OMX.Exynos.AVC.Encoder] Now Idle->Loaded
05-06 17:27:20.541  17114-17709/com.example.andreaskaitis.myapplication I/ACodec﹕ [OMX.Exynos.AVC.Encoder] Now Loaded
05-06 17:27:20.541  17114-17709/com.example.andreaskaitis.myapplication I/ACodec﹕ [OMX.Exynos.AVC.Encoder] Now uninitialized
05-06 17:27:20.546  17114-17114/com.example.andreaskaitis.myapplication D/MPEG4Writer﹕ Stopping Video track
05-06 17:27:20.546  17114-17724/com.example.andreaskaitis.myapplication I/MPEG4Writer﹕ Received total/0-length (59/0) buffers and encoded 59 frames. - video
05-06 17:27:20.546  17114-17114/com.example.andreaskaitis.myapplication D/MPEG4Writer﹕ Stopping Video track source
05-06 17:27:20.546  17114-17114/com.example.andreaskaitis.myapplication D/MPEG4Writer﹕ Video track stopped
05-06 17:27:20.546  17114-17114/com.example.andreaskaitis.myapplication D/MPEG4Writer﹕ Stopping writer thread
05-06 17:27:20.546  17114-17723/com.example.andreaskaitis.myapplication D/MPEG4Writer﹕ 0 chunks are written in the last batch
05-06 17:27:20.546  17114-17114/com.example.andreaskaitis.myapplication D/MPEG4Writer﹕ Writer thread stopped
05-06 17:27:20.551  17114-17114/com.example.andreaskaitis.myapplication I/MPEG4Writer﹕ The mp4 file will not be streamable.
05-06 17:27:20.551  17114-17114/com.example.andreaskaitis.myapplication D/MPEG4Writer﹕ Stopping Video track
05-06 17:27:20.551  17114-17114/com.example.andreaskaitis.myapplication V/WRITE MOVIE﹕ VIDEO CREATED