sudo service mongod status
● mongod.service - MongoDB Database Server
Loaded: loaded (/lib/systemd/system/mongod.service; enabled; vendor preset: enabled)
Active: failed (Result: signal) since Sun 2019-08-04 14:44:43 UTC; 1h 54min ago
Docs: https://docs.mongodb.org/manual
Process: 2013 ExecStart=/usr/bin/mongod --config /etc/mongod.conf (code=killed, signal=ABRT)
Main PID: 2013 (code=killed, signal=ABRT)
root@mongodb-s-2vcpu-4gb-nyc3-01:~#
I'm running mongodb on digitalocean using this preinstall: https://marketplace.digitalocean.com/apps/mongodb
Why did the service stop? Isn't the service supposed to restart automatically on crash? How do I check if that's working.
Edit: Found journal
The logs:
2019-08-04T14:36:05.173+0000 I COMMAND [LogicalSessionCacheRefresh] command config.$cmd command: update { update: "system.sessions", ordered: false, allowImplicitCollectionCreation: false, writeConcern: { w: "majority", wtimeout: 15000 }, $db: "config" } numYields:0 reslen:60 locks:{ Global: { acquireCount: { r: 3, w: 3 } }, Database: { acquireCount: { w: 3 } }, Collection: { acquireCount: { w: 2 } } } protocol:op_msg 102ms
2019-08-04T14:41:40.606+0000 E STORAGE [conn4809] WiredTiger error (12) [1564929700:584465][2013:0x7f44d3d68700], file:collection-34-7943544189608520692.wt, WT_CURSOR.search: __realloc_func, 133: memory allocation of 87330 bytes failed: Cannot allocate memory Raw: [1564929700:584465][2013:0x7f44d3d68700], file:collection-34-7943544189608520692.wt, WT_CURSOR.search: __realloc_func, 133: memory allocation of 87330 bytes failed: Cannot allocate memory
2019-08-04T14:41:40.686+0000 F - [conn4809] Invariant failure: seekRet resulted in status UnknownError: 12: Cannot allocate memory at src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp 1911
2019-08-04T14:41:40.730+0000 F - [conn4809]
***aborting after invariant() failure
2019-08-04T14:41:40.869+0000 F - [conn4809] Got signal: 6 (Aborted).
0x55be6af18cb1 0x55be6af17ec9 0x55be6af183ad 0x7f44e6eae890 0x7f44e6ae9e97 0x7f44e6aeb801 0x55be695573c8 0x55be6962eea5 0x55be69e36aeb 0x55be69de8216 0x55be69e0e39b 0x55be69e1fc2f 0x55be69e0e39b 0x55be69e1c6de 0x55be69e0e39b 0x55be69e10007 0x55be69e0e39b 0x55be69e03039 0x55be69e0e39b 0x55be69e5ee4d 0x55be69e5f7db 0x55be69becb95 0x55be6a988769 0x55be69603e9c 0x55be69605c29 0x55be69606b61 0x55be695f413a 0x55be695feeaa 0x55be695f9b77 0x55be695fd381 0x55be6a7b7a42 0x55be695f7d8f 0x55be695faf25 0x55be695f92b7 0x55be695f9bfd 0x55be695fd381 0x55be6a7b7fa5 0x55be6ae722e4 0x7f44e6ea36db 0x7f44e6bcc88f
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"55BE68B4B000","o":"23CDCB1","s":"_ZN5mongo15printStackTraceERSo"},{"b":"55BE68B4B000","o":"23CCEC9"},{"b":"55BE68B4B000","o":"23CD3AD"},{"b":"7F44E6E9C000","o":"12890"},{"b":"7F44E6AAB000","o":"3EE97","s":"gsignal"},{"b":"7F44E6AAB000","o":"40801","s":"abort"},{"b":"55BE68B4B000","o":"A0C3C8","s":"_ZN5mongo24invariantOKFailedWithMsgEPKcRKNS_6StatusERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES1_j"},{"b":"55BE68B4B000","o":"AE3EA5","s":"_ZN5mongo31WiredTigerRecordStoreCursorBase9seekExactERKNS_8RecordIdE"},{"b":"55BE68B4B000","o":"12EBAEB","s":"_ZN5mongo16WorkingSetCommon5fetchEPNS_16OperationContextEPNS_10WorkingSetEmNS_11unowned_ptrINS_20SeekableRecordCursorEEE"},{"b":"55BE68B4B000","o":"129D216","s":"_ZN5mongo10FetchStage6doWorkEPm"},{"b":"55BE68B4B000","o":"12C339B","s":"_ZN5mongo9PlanStage4workEPm"},{"b":"55BE68B4B000","o":"12D4C2F","s":"_ZN5mongo21SortKeyGeneratorStage6doWorkEPm"},{"b":"55BE68B4B000","o":"12C339B","s":"_ZN5mongo9PlanStage4workEPm"},{"b":"55BE68B4B000","o":"12D16DE","s":"_ZN5mongo9SortStage6doWorkEPm"},{"b":"55BE68B4B000","o":"12C339B","s":"_ZN5mongo9PlanStage4workEPm"},{"b":"55BE68B4B000","o":"12C5007","s":"_ZN5mongo15ProjectionStage6doWorkEPm"},{"b":"55BE68B4B000","o":"12C339B","s":"_ZN5mongo9PlanStage4workEPm"},{"b":"55BE68B4B000","o":"12B8039","s":"_ZN5mongo14MultiPlanStage6doWorkEPm"},{"b":"55BE68B4B000","o":"12C339B","s":"_ZN5mongo9PlanStage4workEPm"},{"b":"55BE68B4B000","o":"1313E4D","s":"_ZN5mongo12PlanExecutor11getNextImplEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE"},{"b":"55BE68B4B000","o":"13147DB","s":"_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE"},{"b":"55BE68B4B000","o":"10A1B95"},{"b":"55BE68B4B000","o":"1E3D769","s":"_ZN5mongo12BasicCommand10Invocation3runEPNS_16OperationContextEPNS_19CommandReplyBuilderE"},{"b":"55BE68B4B000","o":"AB8E9C"},{"b":"55BE68B4B000","o":"ABAC29"},{"b":"55BE68B4B000","o":"ABBB61","s":"_ZN5mongo23ServiceEntryPointCommon13handleRequestEPNS_16OperationContextERKNS_7MessageERKNS0_5HooksE"},{"b":"55BE68B4B000","o":"AA913A","s":"_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE"},{"b":"55BE68B4B000","o":"AB3EAA","s":"_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE"},{"b":"55BE68B4B000","o":"AAEB77","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE"},{"b":"55BE68B4B000","o":"AB2381"},{"b":"55BE68B4B000","o":"1C6CA42","s":"_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlagsENS0_23ServiceExecutorTaskNameE"},{"b":"55BE68B4B000","o":"AACD8F","s":"_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScheduleFlagsENS2_23ServiceExecutorTaskNameENS0_9OwnershipE"},{"b":"55BE68B4B000","o":"AAFF25","s":"_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE"},{"b":"55BE68B4B000","o":"AAE2B7","s":"_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE"},{"b":"55BE68B4B000","o":"AAEBFD","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE"},{"b":"55BE68B4B000","o":"AB2381"},{"b":"55BE68B4B000","o":"1C6CFA5"},{"b":"55BE68B4B000","o":"23272E4"},{"b":"7F44E6E9C000","o":"76DB"},{"b":"7F44E6AAB000","o":"12188F","s":"clone"}],"processInfo":{ "mongodbVersion" : "4.0.3", "gitVersion" : "7ea530946fa7880364d88c8d8b6026bbc9ffa48c", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "4.15.0-54-generic", "version" : "#58-Ubuntu SMP Mon Jun 24 10:55:24 UTC 2019", "machine" : "x86_64" }, "somap" : [ { "b" : "55BE68B4B000", "elfType" : 3, "buildId" : "7F3B9CF97446F8FDDD1B41488C40342571A3243B" }, { "b" : "7FFFFAD01000", "path" : "linux-vdso.so.1", "elfType" : 3, "buildId" : "02C5B0C16E57B3FCDB27AED1A50434F21797AEC7" }, { "b" : "7F44E83F0000", "path" : "/usr/lib/x86_64-linux-gnu/libcurl.so.4", "elfType" : 3, "buildId" : "20310A72634F3373FA7BCAB64DFB64CC204554B2" }, { "b" : "7F44E81D5000", "path" : "/lib/x86_64-linux-gnu/libresolv.so.2", "elfType" : 3, "buildId" : "390E9CC4C215314B6D8ADE6D6E28F8518418039C" }, { "b" : "7F44E7D0A000", "path" : "/usr/lib/x86_64-linux-gnu/libcrypto.so.1.1", "elfType" : 3, "buildId" : "4A1C995324942CDF5D843213896E6C7C0C9C67CC" }, { "b" : "7F44E7A7D000", "path" : "/usr/lib/x86_64-linux-gnu/libssl.so.1.1", "elfType" : 3, "buildId" : "439A262CC0127BA401707DEC7A28884D617550E0" }, { "b" : "7F44E7879000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "25AD56E902E23B490A9CCDB08A9744D89CB95BCC" }, { "b" : "7F44E7671000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "9826FBDF57ED7D6965131074CB3C08B1009C1CD8" }, { "b" : "7F44E72D3000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "A33761AB8FB485311B3C85BF4253099D7CABE653" }, { "b" : "7F44E70BB000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "41BDC55C07D5E5B1D8AB38E2C19B1F535855E084" }, { "b" : "7F44E6E9C000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "28C6AADE70B2D40D1F0F3D0A1A0CAD1AB816448F" }, { "b" : "7F44E6AAB000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "B417C0BA7CC5CF06D1D1BED6652CEDB9253C60D0" }, { "b" : "7F44E866F000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "64DF1B961228382FE18684249ED800AB1DCEAAD4" }, { "b" : "7F44E6886000", "path" : "/usr/lib/x86_64-linux-gnu/libnghttp2.so.14", "elfType" : 3, "buildId" : "4F00E5207693FDC249DA42EC6472ACA6A7B929AE" }, { "b" : "7F44E6669000", "path" : "/usr/lib/x86_64-linux-gnu/libidn2.so.0", "elfType" : 3, "buildId" : "BA5BF9A5C44F48C647E9D8270A5421AE81CCAD61" }, { "b" : "7F44E644D000", "path" : "/usr/lib/x86_64-linux-gnu/librtmp.so.1", "elfType" : 3, "buildId" : "69465D8AA6B19086ABF2455A703F9168BF82A69F" }, { "b" : "7F44E623F000", "path" : "/usr/lib/x86_64-linux-gnu/libpsl.so.5", "elfType" : 3, "buildId" : "CDAF1F1946846941F9D06414EC8C812D131A168E" }, { "b" : "7F44E5FF4000", "path" : "/usr/lib/x86_64-linux-gnu/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "00F419F64B0E70D8C5EEF7050369AA40B2A6E090" }, { "b" : "7F44E5DA2000", "path" : "/usr/lib/x86_64-linux-gnu/libldap_r-2.4.so.2", "elfType" : 3, "buildId" : "E8A505FFF184C54BB6652424E44B852FBE27AACD" }, { "b" : "7F44E5B94000", "path" : "/usr/lib/x86_64-linux-gnu/liblber-2.4.so.2", "elfType" : 3, "buildId" : "51B4F4B81488AADD19CC5817B9ABBB5B94BDE07F" }, { "b" : "7F44E5977000", "path" : "/lib/x86_64-linux-gnu/libz.so.1", "elfType" : 3, "buildId" : "EF3E006DFE3132A41D4D4DC0E407D6EA658E11C4" }, { "b" : "7F44E55F9000", "path" : "/usr/lib/x86_64-linux-gnu/libunistring.so.2", "elfType" : 3, "buildId" : "0E2784298E7D3F4D894FE130ACEFA77C3E624F72" }, { "b" : "7F44E5294000", "path" : "/usr/lib/x86_64-linux-gnu/libgnutls.so.30", "elfType" : 3, "buildId" : "E5AE5C31F804BE96532D0DB2091F19E472F2D4A0" }, { "b" : "7F44E5060000", "path" : "/usr/lib/x86_64-linux-gnu/libhogweed.so.4", "elfType" : 3, "buildId" : "842BDF0B0EAAB82E19F1EABFC38769F4040FBE31" }, { "b" : "7F44E4E2A000", "path" : "/usr/lib/x86_64-linux-gnu/libnettle.so.6", "elfType" : 3, "buildId" : "C20D4B3BA13FCDCC3BF6857689BA9FC70BE3F6A5" }, { "b" : "7F44E4BA9000", "path" : "/usr/lib/x86_64-linux-gnu/libgmp.so.10", "elfType" : 3, "buildId" : "D40EA9B5EC5BC46799E4A412319617BD38BE9341" }, { "b" : "7F44E48D3000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5.so.3", "elfType" : 3, "buildId" : "69FBCF425EE6DF03DE93B82FBC2FC33790E68A96" }, { "b" : "7F44E46A1000", "path" : "/usr/lib/x86_64-linux-gnu/libk5crypto.so.3", "elfType" : 3, "buildId" : "F400D5D643A7F9696DF0E6148FA99BEE6C1BDDF7" }, { "b" : "7F44E449D000", "path" : "/lib/x86_64-linux-gnu/libcom_err.so.2", "elfType" : 3, "buildId" : "C0CB7E35A4566A443F99DFBC1A54D3A0677C8A10" }, { "b" : "7F44E4292000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5support.so.0", "elfType" : 3, "buildId" : "D78D71E8E016A534281B25B97CD7E5E9DB5FE00A" }, { "b" : "7F44E4077000", "path" : "/usr/lib/x86_64-linux-gnu/libsasl2.so.2", "elfType" : 3, "buildId" : "ABB7E3F40302E6509DAD1F91DFB1F04B6A5FD072" }, { "b" : "7F44E3E36000", "path" : "/usr/lib/x86_64-linux-gnu/libgssapi.so.3", "elfType" : 3, "buildId" : "A1A98DB481968073636BBAECB561A3EA8ED198AE" }, { "b" : "7F44E3B07000", "path" : "/usr/lib/x86_64-linux-gnu/libp11-kit.so.0", "elfType" : 3, "buildId" : "8DBD451EA5651283905E16FA7DFA9908688893A3" }, { "b" : "7F44E38F4000", "path" : "/usr/lib/x86_64-linux-gnu/libtasn1.so.6", "elfType" : 3, "buildId" : "6036B89A3BB671B32E01464C0C82BFA016186352" }, { "b" : "7F44E36F0000", "path" : "/lib/x86_64-linux-gnu/libkeyutils.so.1", "elfType" : 3, "buildId" : "F463E107B099910463BC32E837C73D341A52C27B" }, { "b" : "7F44E34E7000", "path" : "/usr/lib/x86_64-linux-gnu/libheimntlm.so.0", "elfType" : 3, "buildId" : "C2376C5B831991591F1A67B976758185F86896D8" }, { "b" : "7F44E325A000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5.so.26", "elfType" : 3, "buildId" : "69BDEE5FA0FEEDF317308BE850F78761861D520A" }, { "b" : "7F44E2FB8000", "path" : "/usr/lib/x86_64-linux-gnu/libasn1.so.8", "elfType" : 3, "buildId" : "315D74995AAA32DE4D15BA25F335066988B1B230" }, { "b" : "7F44E2D82000", "path" : "/usr/lib/x86_64-linux-gnu/libhcrypto.so.4", "elfType" : 3, "buildId" : "6673972A1C24A89EBAFBAE696188A4CB26C6DDEB" }, { "b" : "7F44E2B6C000", "path" : "/usr/lib/x86_64-linux-gnu/libroken.so.18", "elfType" : 3, "buildId" : "430827C33259C12248CF44B91A9A9821114376F5" }, { "b" : "7F44E2964000", "path" : "/usr/lib/x86_64-linux-gnu/libffi.so.6", "elfType" : 3, "buildId" : "3555B5F599C9787DFDDBF9E8DF6F706B9044D985" }, { "b" : "7F44E273B000", "path" : "/usr/lib/x86_64-linux-gnu/libwind.so.0", "elfType" : 3, "buildId" : "93A0931B1C2818F0EA224CE6FE5E31E84A9B55BB" }, { "b" : "7F44E252C000", "path" : "/usr/lib/x86_64-linux-gnu/libheimbase.so.1", "elfType" : 3, "buildId" : "669D4CCE42FA4382796EFFCF0C16F459F4382C4C" }, { "b" : "7F44E22E2000", "path" : "/usr/lib/x86_64-linux-gnu/libhx509.so.5", "elfType" : 3, "buildId" : "4B80C543356EE0AF9039EFE7C9EA1CC1F74C426A" }, { "b" : "7F44E1FD9000", "path" : "/usr/lib/x86_64-linux-gnu/libsqlite3.so.0", "elfType" : 3, "buildId" : "BCE351987CF42B3D258B09F0CAC867758D935086" }, { "b" : "7F44E1DA1000", "path" : "/lib/x86_64-linux-gnu/libcrypt.so.1", "elfType" : 3, "buildId" : "810686AF0D5FD350A4FB1CC4B5AFF44A05C102CB" } ] }}
mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x55be6af18cb1]
mongod(+0x23CCEC9) [0x55be6af17ec9]
mongod(+0x23CD3AD) [0x55be6af183ad]
libpthread.so.0(+0x12890) [0x7f44e6eae890]
libc.so.6(gsignal+0xC7) [0x7f44e6ae9e97]
libc.so.6(abort+0x141) [0x7f44e6aeb801]
mongod(_ZN5mongo24invariantOKFailedWithMsgEPKcRKNS_6StatusERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES1_j+0x0) [0x55be695573c8]
mongod(_ZN5mongo31WiredTigerRecordStoreCursorBase9seekExactERKNS_8RecordIdE+0x1C5) [0x55be6962eea5]
mongod(_ZN5mongo16WorkingSetCommon5fetchEPNS_16OperationContextEPNS_10WorkingSetEmNS_11unowned_ptrINS_20SeekableRecordCursorEEE+0xAB) [0x55be69e36aeb]
mongod(_ZN5mongo10FetchStage6doWorkEPm+0x106) [0x55be69de8216]
mongod(_ZN5mongo9PlanStage4workEPm+0x6B) [0x55be69e0e39b]
mongod(_ZN5mongo21SortKeyGeneratorStage6doWorkEPm+0x3F) [0x55be69e1fc2f]
mongod(_ZN5mongo9PlanStage4workEPm+0x6B) [0x55be69e0e39b]
mongod(_ZN5mongo9SortStage6doWorkEPm+0xFE) [0x55be69e1c6de]
mongod(_ZN5mongo9PlanStage4workEPm+0x6B) [0x55be69e0e39b]
mongod(_ZN5mongo15ProjectionStage6doWorkEPm+0x47) [0x55be69e10007]
mongod(_ZN5mongo9PlanStage4workEPm+0x6B) [0x55be69e0e39b]
mongod(_ZN5mongo14MultiPlanStage6doWorkEPm+0xB9) [0x55be69e03039]
mongod(_ZN5mongo9PlanStage4workEPm+0x6B) [0x55be69e0e39b]
mongod(_ZN5mongo12PlanExecutor11getNextImplEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE+0x50D) [0x55be69e5ee4d]
mongod(_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE+0x4B) [0x55be69e5f7db]
mongod(+0x10A1B95) [0x55be69becb95]
mongod(_ZN5mongo12BasicCommand10Invocation3runEPNS_16OperationContextEPNS_19CommandReplyBuilderE+0xD9) [0x55be6a988769]
mongod(+0xAB8E9C) [0x55be69603e9c]
mongod(+0xABAC29) [0x55be69605c29]
mongod(_ZN5mongo23ServiceEntryPointCommon13handleRequestEPNS_16OperationContextERKNS_7MessageERKNS0_5HooksE+0x3C1) [0x55be69606b61]
mongod(_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE+0x3A) [0x55be695f413a]
mongod(_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE+0xBA) [0x55be695feeaa]
mongod(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0x97) [0x55be695f9b77]
mongod(+0xAB2381) [0x55be695fd381]
mongod(_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlagsENS0_23ServiceExecutorTaskNameE+0x1A2) [0x55be6a7b7a42]
mongod(_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScheduleFlagsENS2_23ServiceExecutorTaskNameENS0_9OwnershipE+0x15F) [0x55be695f7d8f]
mongod(_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE+0xAF5) [0x55be695faf25]
mongod(_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE+0x357) [0x55be695f92b7]
mongod(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0x11D) [0x55be695f9bfd]
mongod(+0xAB2381) [0x55be695fd381]
mongod(+0x1C6CFA5) [0x55be6a7b7fa5]
mongod(+0x23272E4) [0x55be6ae722e4]
libpthread.so.0(+0x76DB) [0x7f44e6ea36db]
libc.so.6(clone+0x3F) [0x7f44e6bcc88f]
----- END BACKTRACE -----
The logs report some memory errors. How do I fix those and ensure they don't happen again?
