mongodb 跑15W数据量的数据老挂，内存爆满，CPU占用很高，VPS 是KVM 512的，有没有解决办法啊？

[zhfsxtx]

这是我之前 发的帖子

http://www.v2ex.com/t/88732

网站是 http://www.peinimai.com

[humiaozuzu]

贴一下 log？还有 mongoshell 的 db.serverStatus()

[zhfsxtx]

@humiaozuzu


> db.serverStatus()
{
"host" : "hhVPS",
"version" : "2.4.8",
"process" : "mongod",
"pid" : 30181,
"uptime" : 744,
"uptimeMillis" : NumberLong(744564),
"uptimeEstimate" : 736,
"localTime" : ISODate("2013-11-10T15:19:11.837Z"),
"asserts" : {
"regular" : 0,
"warning" : 0,
"msg" : 0,
"user" : 0,
"rollovers" : 0
},
"backgroundFlushing" : {
"flushes" : 12,
"total_ms" : 76,
"average_ms" : 6.333333333333333,
"last_ms" : 1,
"last_finished" : ISODate("2013-11-10T15:18:47.290Z")
},
"connections" : {
"current" : 11,
"available" : 808,
"totalCreated" : NumberLong(11)
},
"cursors" : {
"totalOpen" : 0,
"clientCursors_size" : 0,
"timedOut" : 0
},
"dur" : {
"commits" : 30,
"journaledMB" : 0,
"writeToDataFilesMB" : 0,
"compression" : 0,
"commitsInWriteLock" : 0,
"earlyCommits" : 0,
"timeMs" : {
"dt" : 3070,
"prepLogBuffer" : 0,
"writeToJournal" : 0,
"writeToDataFiles" : 0,
"remapPrivateView" : 0
}
},
"extra_info" : {
"note" : "fields vary by platform",
"heap_usage_bytes" : 122573112,
"page_faults" : 245
},
"globalLock" : {
"totalTime" : NumberLong(744564000),
"lockTime" : NumberLong(14523),
"currentQueue" : {
"total" : 0,
"readers" : 0,
"writers" : 0
},
"activeClients" : {
"total" : 0,
"readers" : 0,
"writers" : 0
}
},
"indexCounters" : {
"accesses" : 0,
"hits" : 0,
"misses" : 0,
"resets" : 0,
"missRatio" : 0
},
"locks" : {
"." : {
"timeLockedMicros" : {
"R" : NumberLong(19785),
"W" : NumberLong(14523)
},
"timeAcquiringMicros" : {
"R" : NumberLong(12923),
"W" : NumberLong(159266)
}
},
"admin" : {
"timeLockedMicros" : {
"r" : NumberLong(1190),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(1554),
"w" : NumberLong(0)
}
},
"local" : {
"timeLockedMicros" : {
"r" : NumberLong(1287),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(139),
"w" : NumberLong(0)
}
},
"double11" : {
"timeLockedMicros" : {
"r" : NumberLong(28627609),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(10399787),
"w" : NumberLong(0)
}
}
},
"network" : {
"bytesIn" : 9707,
"bytesOut" : 2832815,
"numRequests" : 131
},
"opcounters" : {
"insert" : 1,
"query" : 118,
"update" : 0,
"delete" : 0,
"getmore" : 22,
"command" : 29
},
"opcountersRepl" : {
"insert" : 0,
"query" : 0,
"update" : 0,
"delete" : 0,
"getmore" : 0,
"command" : 0
},
"recordStats" : {
"accessesNotInMemory" : 43,
"pageFaultExceptionsThrown" : 2,
"admin" : {
"accessesNotInMemory" : 0,
"pageFaultExceptionsThrown" : 0
},
"double11" : {
"accessesNotInMemory" : 43,
"pageFaultExceptionsThrown" : 2
},
"local" : {
"accessesNotInMemory" : 0,
"pageFaultExceptionsThrown" : 0
}
},
"writeBacksQueued" : false,
"mem" : {
"bits" : 64,
"resident" : 291,
"virtual" : 1964,
"supported" : true,
"mapped" : 288,
"mappedWithJournal" : 576
},
"metrics" : {
"document" : {
"deleted" : NumberLong(0),
"inserted" : NumberLong(1),
"returned" : NumberLong(4846),
"updated" : NumberLong(0)
},
"getLastError" : {
"wtime" : {
"num" : 0,
"totalMillis" : 0
},
"wtimeouts" : NumberLong(0)
},
"operation" : {
"fastmod" : NumberLong(0),
"idhack" : NumberLong(0),
"scanAndOrder" : NumberLong(0)
},
"queryExecutor" : {
"scanned" : NumberLong(4804690)
},
"record" : {
"moves" : NumberLong(0)
},
"repl" : {
"apply" : {
"batches" : {
"num" : 0,
"totalMillis" : 0
},
"ops" : NumberLong(0)
},
"buffer" : {
"count" : NumberLong(0),
"maxSizeBytes" : 268435456,
"sizeBytes" : NumberLong(0)
},
"network" : {
"bytes" : NumberLong(0),
"getmores" : {
"num" : 0,
"totalMillis" : 0
},
"ops" : NumberLong(0),
"readersCreated" : NumberLong(0)
},
"oplog" : {
"insert" : {
"num" : 0,
"totalMillis" : 0
},
"insertBytes" : NumberLong(0)
},
"preload" : {
"docs" : {
"num" : 0,
"totalMillis" : 0
},
"indexes" : {
"num" : 0,
"totalMillis" : 0
}
}
},
"ttl" : {
"deletedDocuments" : NumberLong(0),
"passes" : NumberLong(12)
}
},
"ok" : 1
}

[bengtuo]

512的内存跑mongodb 有点小

[zhfsxtx]

@humiaozuzu log 太多不知道 怎么贴

[moname]

15万数据，搞个大点的vps吧。

[webflier]

1.要么用大点内存。
2.要么用多台512M的机器做sharding。
我能偷偷告诉你我有一个10多台512M的机器做的mongodbb sharding cluster嘛......物美价廉，跑得非常的欢畅
另外你有没有在数据库前面做适当的caching挡一下？这样数据库的压力会小很多？

[so898]

512MB内存也敢上mongodb，楼主你太小瞧这玩意了吧……

[zhfsxtx]

@webflier
什么caching啊，我没有，我都是自己瞎折腾，门外汉
10多台512M ，你真是高富帅啊

[likuku]

挂的时候啥报错啊？

[mikale]

mongodb有small file模式，你用用，这适合你这种环境，不然就这点小内存，根本不适合

[zhfsxtx]

@mikale 怎么开户smalll file 模式啊，是不是我要先把数据导出来，再用--smallfiles 重新导入数据啊

[zhfsxtx]

@likuku


Sun Nov 10 15:05:45.852 [initandlisten] git version: a350fc38922fbda2cec8d5dd842237b904eafc14
Sun Nov 10 15:05:45.852 [initandlisten] build info: Linux ip-10-2-29-40 2.6.21.7-2.ec2.v1.2.fc8xen #1 SMP Fri Nov 20 17:48:28 EST 2009 x86_64 BOOST_LIB_VERSION=1_49
Sun Nov 10 15:05:45.852 [initandlisten] allocator: tcmalloc
Sun Nov 10 15:05:45.852 [initandlisten] options: { dbpath: "/data/db", fork: true, logappend: true, logpath: "/root/mongodb/work.log", port: 27017, quiet: true }
Sun Nov 10 15:05:46.485 [initandlisten] journal dir=/data/db/journal
Sun Nov 10 15:05:46.485 [initandlisten] recover begin
Sun Nov 10 15:05:46.485 [initandlisten] info no lsn file in journal/ directory
Sun Nov 10 15:05:46.485 [initandlisten] recover lsn: 0
Sun Nov 10 15:05:46.485 [initandlisten] recover /data/db/journal/j._0
Sun Nov 10 15:05:46.616 [initandlisten] recover cleaning up
Sun Nov 10 15:05:46.616 [initandlisten] removeJournalFiles
Sun Nov 10 15:05:46.753 [initandlisten] recover done
Sun Nov 10 15:05:46.754 [initandlisten] preallocating a journal file /data/db/journal/prealloc.0
Sun Nov 10 15:05:49.055 [initandlisten] File Preallocator Progress: 167772160/1073741824 15%
Sun Nov 10 15:05:52.981 [initandlisten] File Preallocator Progress: 272629760/1073741824 25%
Sun Nov 10 15:05:55.019 [initandlisten] File Preallocator Progress: 408944640/1073741824 38%
Sun Nov 10 15:05:58.063 [initandlisten] File Preallocator Progress: 555745280/1073741824 51%
Sun Nov 10 15:06:01.267 [initandlisten] File Preallocator Progress: 681574400/1073741824 63%
Sun Nov 10 15:06:04.246 [initandlisten] File Preallocator Progress: 796917760/1073741824 74%
Sun Nov 10 15:06:07.414 [initandlisten] File Preallocator Progress: 880803840/1073741824 82%
Sun Nov 10 15:06:10.098 [initandlisten] File Preallocator Progress: 891289600/1073741824 83%
Sun Nov 10 15:06:13.134 [initandlisten] File Preallocator Progress: 975175680/1073741824 90%
Sun Nov 10 15:06:20.004 [initandlisten] command local.$cmd command: { create: "startup_log", size: 10485760, capped: true } ntoreturn:1 keyUpdates:0 reslen:75 212ms
Sun Nov 10 15:06:20.028 [websvr] admin web console waiting for connections on port 28017
Sun Nov 10 15:06:20.449 [initandlisten] waiting for connections on port 27017
Sun Nov 10 15:06:31.422 [signalProcessingThread] got signal 2 (Interrupt), will terminate after current cmd ends
Sun Nov 10 15:06:31.422 [signalProcessingThread] now exiting
Sun Nov 10 15:06:31.422 dbexit:
Sun Nov 10 15:06:31.422 [signalProcessingThread] shutdown: going to close listening sockets...
Sun Nov 10 15:06:31.422 [signalProcessingThread] closing listening socket: 9
Sun Nov 10 15:06:31.422 [signalProcessingThread] closing listening socket: 10
Sun Nov 10 15:06:31.422 [signalProcessingThread] closing listening socket: 11
Sun Nov 10 15:06:31.422 [signalProcessingThread] removing socket file: /tmp/mongodb-27017.sock
Sun Nov 10 15:06:31.422 [signalProcessingThread] shutdown: going to flush diaglog...
Sun Nov 10 15:06:31.422 [signalProcessingThread] shutdown: going to close sockets...
Sun Nov 10 15:06:31.422 [signalProcessingThread] shutdown: waiting for fs preallocator...
Sun Nov 10 15:06:31.422 [signalProcessingThread] shutdown: lock for final commit...
Sun Nov 10 15:06:31.422 [signalProcessingThread] shutdown: final commit...
Sun Nov 10 15:06:31.478 [signalProcessingThread] shutdown: closing all files...
Sun Nov 10 15:06:31.478 [signalProcessingThread] closeAllFiles() finished
Sun Nov 10 15:06:31.478 [signalProcessingThread] journalCleanup...
Sun Nov 10 15:06:31.478 [signalProcessingThread] removeJournalFiles
Sun Nov 10 15:06:31.607 [signalProcessingThread] shutdown: removing fs lock...
Sun Nov 10 15:06:31.620 dbexit: really exiting now

[webflier]

看着信息，你是不是硬盘没空间了啊

[zhfsxtx]

@webflier

Filesystem Size Used Avail Use% Mounted on
/dev/sda1 9.5G 5.5G 3.6G 61% /
udev 242M 4.0K 242M 1% /dev
tmpfs 99M 208K 99M 1% /run
none 5.0M 0 5.0M 0% /run/lock
none 246M 0 246M 0% /run/shm


硬盘有空间的，而且这个网站只会查询数据，不会写数据库的

[zhfsxtx]

fdisk -l 看是这样的

Disk /dev/sda: 10.7 GB, 10737418240 bytes
255 heads, 63 sectors/track, 1305 cylinders, total 20971520 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x00032c86

Device Boot Start End Blocks Id System
/dev/sda1 2048 19924991 9961472 83 Linux
/dev/sda2 19924992 20971519 523264 82 Linux swap / Solaris



但是为什么free的时候 看，swap 是0呢

total used free shared buffers cached
Mem: 503528 434740 68788 0 8584 174816
-/+ buffers/cache: 251340 252188
Swap: 0 0 0

[thai9quohs6jae1C]

如果换作是mariadb毫无压力啊 哈哈

[zhfsxtx]

今天加挂了512M SWAP，不知道 会不会好点