I have been at this for three days straight and it's driving me insane.
I have an event notification set up on an S3 bucket to field file.wav uploads, transcode them using a static ffmpeg binary hosted on an node.js AWS Lambda instance, then re-upload the file to the same bucket.
The issue I'm having - and this does work on my local XUbuntu machine - is that the file is read, encoded, and written synchronously, yet some sort of asynchronous access is distorting the audio file. A symptom of the distortion is static white noise, and previously (although it may have been fixed) there was overlapping audio.
This is the process:
- File is uploaded via frontend
- Lambda is invoked
- Handler is entered and all necessary endpoint construction is performed
exports.handler = async (event, context, callback) => {
callback();
const buckinfo = event.Records[0].s3;
const filename = buckinfo.object.key.toString().replace(/\+/g, " ");
var des = filename.split('/');
var newFile = des[des.length - 1];
console.log('filename : ', filename);
const logKey = event.logKey || `${'logKey'}.log`;
s3Bucket = event.s3Bucket || process.env.S3_UPLOADS;
var meta = null;
const inputFilename = tempy.file({extension: 'wav'});
const temp3Filename = tempy.file({extension: 'mp3'});
console.log('input file', inputFilename);
console.log('output file', temp3Filename);
var mp3Filename = filename.split('/');
mp3Filename[4] = 'mp3';
mp3Filename = mp3Filename[0] + '/' +
mp3Filename[1] + '/' +
mp3Filename[2] + '/' +
mp3Filename[3] + '/' +
mp3Filename[4] + '/' +
newFile.split('.').slice(0, -1).join('.') + '.mp3';
console.log('mp3Filename : ', mp3Filename);
await readIn(filename, inputFilename, false);
await encodeFile(inputFilename, temp3Filename, false);
await writeOut(temp3Filename, mp3Filename, false);
};
(YES it's gross, but it works, so only worry about the last three lines)
- The file is read in from the key passed to it via the upload event
async function readIn(inputFilename, transfer, local) {
console.log('reading in locally?', local);
console.log(`reading from ${inputFilename} to ${transfer}`);
const writeStream = fs.createWriteStream(transfer);
var file;
local? await new Promise((resolve, revoke) => {
file = fs.createReadStream(inputFilename);
writeStream.on('finish', () => {
console.log('file finished reading');
resolve();
});
writeStream.on('error', (err) => {
console.log('things messed up');
revoke();
});
file.pipe(writeStream);
}) : await new Promise((resolve, revoke) => {
writeStream.on('finish', () => {
console.log('file finished reading');
resolve();
});
writeStream.on('error', (err) => {
console.log('things messed up');
revoke();
});
s3.getObject({
Bucket :s3Bucket,
Key :inputFilename
}, function(err, data) {
if (err) {
console.log(err, err.stack);
} else {
console.log('data got!');
}
}).createReadStream().pipe(writeStream);
});
console.log('returning to main from reading');
return ;
}
- The file is then transcoded using the Linux x86-64 static binary
async function encodeFile(inputFilename, temp3Filename) {
const ffmpeg = path.resolve(__dirname, 'ffmpeg');
const ffmpegArgs = ['-i', inputFilename, '-vn', '-acodec', 'libmp3lame', temp3Filename];
console.log('arguments to ffmpeg', ffmpegArgs);
const ps = child_process.spawn(ffmpeg, ffmpegArgs);
await new Promise((resolve, revoke) => {
console.log('beginning encoding process');
ps.stderr.on('data', (data) => {console.log(`stderr: ${data}`);});
ps.on('exit', function(code, signal) {
console.log('child process exited with ' + `code ${code} and signal ${signal}`);
});
ps.on('error', function(code, signal) {
console.log('child process erred with ' + `code ${code} and signal ${signal}`);
});
ps.on('close', (code) => {
if (code === 0) {
resolve(code);
} else {
revoke(code);
}
});
});
console.log('returning to main from encoding');
return ;
}
- The temporary encoded file is then re-uploaded to the S3 bucket async
function writeOut(transfer, outFile, local) {
console.log('writing out locally?', local);
console.log(`writing to ${outFile} from ${transfer}`);
const data = fs.createReadStream(transfer);
local? await new Promise((resolve, revoke) => {
const file = fs.createWriteStream(outFile);
file.on('finish', () => {
console.log('file finished writing');
resolve();
});
file.on('error', (err) => {
console.log('things messed up');
revoke();
});
data.pipe(file);
}) : await new Promise((resolve, revoke) => {
const params = {
Bucket: s3Bucket,
Key: outFile,
Body: data,
ContentType: 'audio/mpeg'
};
const options = {
partSize: 40 * 1024 * 1024,
queueSize: 1
};
s3.upload(params, options, function(err, data) {
if (err) {
console.log('error uploading', err, err.stack);
revoke(err);
} else {
console.log(data);
resolve();
}
});
});
console.log('returning to main from writing');
}
Any tips or suggestions are more than welcome. Here is the main() I use for local testing:
async function main(input, output) {
const inputFilename = tempy.file({extension: 'wav'});
const temp3Filename = tempy.file({extension: 'mp3'});
await readIn(input, inputFilename, true);
const logs = await encodeFile(inputFilename, temp3Filename, true);
await writeOut(temp3Filename, output, true);
}
main('../../../../../../Downloads/<file>.wav', './local.mp3');
You will need to uncomment the s3 modules require statements to get this to run. All you need are the node_modules required above and the static fmmpeg binary dependent on your OS.
Thanks!
S3 Logs:
START RequestId: eb6dfbc7-e927-11e8-9a7e-c172e5d12174 Version: $LATEST
2018-11-15T22:43:50.558Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 filename : <S3/endpoint.wav>
2018-11-15T22:43:50.573Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 input file /tmp/80e9f1a9539d322a6a9cfd429b78a2f3.wav
2018-11-15T22:43:50.573Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 output file /tmp/0e1119ad256b7bfa6f5cc270eaa273cc.mp3
2018-11-15T22:43:50.573Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 mp3Filename : <S3/endpoint.mp3>
2018-11-15T22:43:50.573Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 reading in locally? false
2018-11-15T22:43:50.573Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 reading from <S3/endpoint.wav> to /tmp/80e9f1a9539d322a6a9cfd429b78a2f3.wav
2018-11-15T22:43:53.073Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 data got!
2018-11-15T22:43:53.113Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 file finished reading
2018-11-15T22:43:53.113Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 returning to main from reading
2018-11-15T22:43:53.113Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 arguments to ffmpeg [ '-i',
'/tmp/80e9f1a9539d322a6a9cfd429b78a2f3.wav',
'-vn',
'-acodec',
'libmp3lame',
'/tmp/0e1119ad256b7bfa6f5cc270eaa273cc.mp3' ]
2018-11-15T22:43:53.153Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 beginning encoding process
2018-11-15T22:43:53.913Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: ffmpeg version 4.1-static https://johnvansickle.com/ffmpeg/ Copyright (c) 2000-2018 the FFmpeg developers
built with gcc 6.3.0 (Debian 6.3.0-18+deb9u1) 20170516
configuration: --enable-gpl --enable-version3 --enable-static --disable-debug --disable-ffplay --disable-indev=sndio --disable-outdev=sndio --cc=gcc-6 --enable-fontconfig --enable-frei0r --enable-gnutls --enable-gray --enable-libaom --enable-libfribidi --enable-libass --enable-libvmaf --enable-libfreetype --enable-libmp3lame --enable-libopencore-amrnb --enable-libopencore-amrwb --enable-libopenjpeg --enable-librubberband --enable-libsoxr --enable-libspeex --enable-libvorbis --enable-libopus --enable-libtheora --enable-libvidstab --enable-libvo-amrwbenc --enable-libvpx --enable-libwebp --enable-libx264 --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzimg
2018-11-15T22:43:53.972Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: libavutil 56. 22.100 / 56. 22.100
libavcodec 58. 35.100 / 58. 35.100
libavformat 58. 20.100 / 58. 20.100
libavdevice 58. 5.100 / 58. 5.100
libavfilter 7. 40.101 / 7. 40.101
libswscale 5. 3.100 / 5. 3.100
libswresample 3. 3.100 / 3. 3.100
libpostproc 55. 3.100 / 55. 3.100
2018-11-15T22:43:54.453Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: Guessed Channel Layout for Input Stream #0.0 : mono
Input #0, wav, from '/tmp/80e9f1a9539d322a6a9cfd429b78a2f3.wav':
Duration: 00:00:56.57, bitrate: 1059 kb/s
Stream #0:0: Audio: pcm_s16le ([1][0][0][0] / 0x0001), 44100 Hz, mono, s16, 705 kb/s
2018-11-15T22:43:54.472Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: Stream mapping:
Stream #0:0 -> #0:0 (pcm_s16le (native) -> mp3 (libmp3lame))
Press [q] to stop, [?] for help
2018-11-15T22:43:54.612Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: Output #0, mp3, to '/tmp/0e1119ad256b7bfa6f5cc270eaa273cc.mp3':
Metadata:
TSSE : Lavf58.20.100
2018-11-15T22:43:54.612Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: Stream #0:0: Audio: mp3 (libmp3lame), 44100 Hz, mono, s16p
Metadata:
encoder : Lavc58.35.100 libmp3lame
2018-11-15T22:43:54.994Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 10kB time=00:00:01.25 bitrate= 66.7kbits/s speed=2.41x
2018-11-15T22:43:55.494Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 26kB time=00:00:03.24 bitrate= 65.1kbits/s speed=3.17x
2018-11-15T22:43:56.013Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 42kB time=00:00:05.33 bitrate= 64.6kbits/s speed=3.46x
2018-11-15T22:43:56.514Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 59kB time=00:00:07.47 bitrate= 64.5kbits/s speed=3.66x
2018-11-15T22:43:57.014Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 75kB time=00:00:09.56 bitrate= 64.4kbits/s speed=3.76x
2018-11-15T22:43:57.515Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 91kB time=00:00:11.57 bitrate= 64.3kbits/s speed= 3.8x
2018-11-15T22:43:58.034Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 105kB time=00:00:13.42 bitrate= 64.3kbits/s speed=3.77x
2018-11-15T22:43:58.534Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 122kB time=00:00:15.51 bitrate= 64.2kbits/s speed=3.82x
2018-11-15T22:43:59.035Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 138kB time=00:00:17.60 bitrate= 64.2kbits/s speed=3.86x
2018-11-15T22:43:59.535Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 153kB time=00:00:19.54 bitrate= 64.2kbits/s speed=3.86x
2018-11-15T22:44:00.054Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 169kB time=00:00:21.60 bitrate= 64.2kbits/s speed=3.87x
2018-11-15T22:44:00.574Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 187kB time=00:00:23.87 bitrate= 64.1kbits/s speed=3.91x
2018-11-15T22:44:01.093Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 204kB time=00:00:26.04 bitrate= 64.1kbits/s speed=3.93x
2018-11-15T22:44:01.614Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 221kB time=00:00:28.23 bitrate= 64.1kbits/s speed=3.95x
2018-11-15T22:44:02.114Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 236kB time=00:00:30.09 bitrate= 64.1kbits/s speed=3.94x
2018-11-15T22:44:02.614Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 252kB time=00:00:32.26 bitrate= 64.1kbits/s speed=3.96x
2018-11-15T22:44:03.152Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 256kB time=00:00:34.40 bitrate= 61.0kbits/s speed=3.98x
2018-11-15T22:44:03.653Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 256kB time=00:00:36.54 bitrate= 57.4kbits/s speed= 4x
2018-11-15T22:44:04.153Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 256kB time=00:00:38.58 bitrate= 54.4kbits/s speed= 4x
2018-11-15T22:44:04.634Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 256kB time=00:00:40.59 bitrate= 51.7kbits/s speed= 4x
2018-11-15T22:44:05.134Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 256kB time=00:00:42.81 bitrate= 49.0kbits/s speed=4.02x
2018-11-15T22:44:05.672Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 256kB time=00:00:44.95 bitrate= 46.6kbits/s speed=4.03x
2018-11-15T22:44:06.192Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 256kB time=00:00:46.99 bitrate= 44.6kbits/s speed=4.02x
2018-11-15T22:44:06.674Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 256kB time=00:00:49.18 bitrate= 42.6kbits/s speed=4.03x
2018-11-15T22:44:07.954Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 256kB time=00:00:51.22 bitrate= 40.9kbits/s speed=4.03x
size= 256kB time=00:00:53.36 bitrate= 39.3kbits/s speed=4.03x
2018-11-15T22:44:08.214Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 256kB time=00:00:54.83 bitrate= 38.2kbits/s speed=3.99x
2018-11-15T22:44:08.613Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 stderr: size= 442kB time=00:00:56.58 bitrate= 64.1kbits/s speed= 4x
video:0kB audio:442kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.050126%
2018-11-15T22:44:08.713Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 child process exited with code 0 and signal null
2018-11-15T22:44:08.713Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 returning to main from encoding
2018-11-15T22:44:08.713Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 writing out locally? false
2018-11-15T22:44:08.713Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 writing to <S3/endpoint.mp3> from /tmp/0e1119ad256b7bfa6f5cc270eaa273cc.mp3
2018-11-15T22:44:09.045Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 { ETag: <ETag>,
Location: <S3/endpoint.mp3>,
key: <S3/endpoint.mp3>,
Key: <S3/endpoint.mp3>,
Bucket: <S3> }
2018-11-15T22:44:09.072Z eb6dfbc7-e927-11e8-9a7e-c172e5d12174 returning to main from writing
END RequestId: eb6dfbc7-e927-11e8-9a7e-c172e5d12174
REPORT RequestId: eb6dfbc7-e927-11e8-9a7e-c172e5d12174 Duration: 18536.93 ms Billed Duration: 18600 ms Memory Size: 192 MB Max Memory Used: 107 MB
