Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- ceilometer:PRIMARY> db.currentOp()
- {
- "inprog" : [
- {
- "opid" : 135695275,
- "active" : true,
- "secs_running" : 3,
- "microsecs_running" : NumberLong(3957478),
- "op" : "getmore",
- "ns" : "local.oplog.rs",
- "query" : {
- },
- "client" : "192.168.0.11:58507",
- "desc" : "conn439815",
- "threadId" : "0x7f425f2a3700",
- "connectionId" : 439815,
- "waitingForLock" : false,
- "numYields" : 0,
- "lockStats" : {
- "timeLockedMicros" : {
- "r" : NumberLong(163),
- "w" : NumberLong(0)
- },
- "timeAcquiringMicros" : {
- "r" : NumberLong(9),
- "w" : NumberLong(0)
- }
- }
- },
- {
- "opid" : 1330,
- "active" : false,
- "op" : "none",
- "ns" : "",
- "query" : {
- },
- "desc" : "repl writer worker 1",
- "threadId" : "0x7f4d68bfa700",
- "waitingForLock" : false,
- "numYields" : 0,
- "lockStats" : {
- "timeLockedMicros" : {
- "r" : NumberLong(0),
- "w" : NumberLong(117)
- },
- "timeAcquiringMicros" : {
- }
- }
- },
- {
- "opid" : 1294,
- "active" : false,
- "op" : "none",
- "ns" : "",
- "query" : {
- },
- "desc" : "repl writer worker 2",
- "threadId" : "0x7f4d693fb700",
- "waitingForLock" : false,
- "numYields" : 0,
- "lockStats" : {
- "timeLockedMicros" : {
- "r" : NumberLong(0),
- "w" : NumberLong(173)
- },
- "timeAcquiringMicros" : {
- }
- }
- },
- {
- "opid" : 135695276,
- "active" : true,
- "secs_running" : 3,
- "microsecs_running" : NumberLong(3956053),
- "op" : "getmore",
- "ns" : "local.oplog.rs",
- "query" : {
- },
- "client" : "192.168.0.12:41920",
- "desc" : "conn440769",
- "threadId" : "0x7f4252ddf700",
- "connectionId" : 440769,
- "waitingForLock" : false,
- "numYields" : 0,
- "lockStats" : {
- "timeLockedMicros" : {
- "r" : NumberLong(125),
- "w" : NumberLong(0)
- },
- "timeAcquiringMicros" : {
- "r" : NumberLong(7),
- "w" : NumberLong(0)
- }
- }
- },
- {
- "opid" : 135683872,
- "active" : true,
- "secs_running" : 75,
- "microsecs_running" : NumberLong(75030714),
- "op" : "query",
- "ns" : "ceilometer.meter",
- "query" : {
- "$msg" : "query not recording (too large)"
- },
- "client" : "192.168.0.7:33338",
- "desc" : "conn441390",
- "threadId" : "0x7f42633dd700",
- "connectionId" : 441390,
- "locks" : {
- "^" : "r",
- "^ceilometer" : "R"
- },
- "waitingForLock" : false,
- "numYields" : 19741,
- "lockStats" : {
- "timeLockedMicros" : {
- "r" : NumberLong(131258027),
- "w" : NumberLong(0)
- },
- "timeAcquiringMicros" : {
- "r" : NumberLong(2456487),
- "w" : NumberLong(0)
- }
- }
- }
- ]
- }
- ceilometer:PRIMARY> db.runCommand( { serverStatus: 1, workingSet: 1, metrics: 0, locks: 0 } )
- {
- "host" : "node-5.fuel",
- "version" : "2.6.10",
- "process" : "mongod",
- "pid" : NumberLong(22112),
- "uptime" : 1112982,
- "uptimeMillis" : NumberLong(1112981660),
- "uptimeEstimate" : 1080241,
- "localTime" : ISODate("2016-07-13T09:59:33.164Z"),
- "asserts" : {
- "regular" : 0,
- "warning" : 0,
- "msg" : 0,
- "user" : 6,
- "rollovers" : 0
- },
- "backgroundFlushing" : {
- "flushes" : 18549,
- "total_ms" : 2663624,
- "average_ms" : 143.59933150035042,
- "last_ms" : 21,
- "last_finished" : ISODate("2016-07-13T09:58:59.101Z")
- },
- "connections" : {
- "current" : 262,
- "available" : 50938,
- "totalCreated" : NumberLong(441403)
- },
- "cursors" : {
- "note" : "deprecated, use server status metrics",
- "clientCursors_size" : 2,
- "totalOpen" : 2,
- "pinned" : 0,
- "totalNoTimeout" : 0,
- "timedOut" : 6
- },
- "dur" : {
- "commits" : 29,
- "journaledMB" : 0.008192,
- "writeToDataFilesMB" : 0.001082,
- "compression" : 6.720262510254307,
- "commitsInWriteLock" : 0,
- "earlyCommits" : 0,
- "timeMs" : {
- "dt" : 3000,
- "prepLogBuffer" : 0,
- "writeToJournal" : 1,
- "writeToDataFiles" : 0,
- "remapPrivateView" : 0
- }
- },
- "extra_info" : {
- "note" : "fields vary by platform",
- "heap_usage_bytes" : 197870984,
- "page_faults" : 250712221
- },
- "globalLock" : {
- "totalTime" : NumberLong("1112981662000"),
- "lockTime" : NumberLong("6290549859"),
- "currentQueue" : {
- "total" : 0,
- "readers" : 0,
- "writers" : 0
- },
- "activeClients" : {
- "total" : 0,
- "readers" : 0,
- "writers" : 0
- }
- },
- "indexCounters" : {
- "accesses" : 1022072542,
- "hits" : 1022072542,
- "misses" : 0,
- "resets" : 0,
- "missRatio" : 0
- },
- "network" : {
- "bytesIn" : 59222662374,
- "bytesOut" : 96910035531,
- "numRequests" : 80899312
- },
- "opcounters" : {
- "insert" : 15482042,
- "query" : 354407,
- "update" : 11868408,
- "delete" : 0,
- "getmore" : 15753099,
- "command" : 37759718
- },
- "opcountersRepl" : {
- "insert" : 184,
- "query" : 0,
- "update" : 146,
- "delete" : 714,
- "getmore" : 0,
- "command" : 0
- },
- "recordStats" : {
- "accessesNotInMemory" : 2264768,
- "pageFaultExceptionsThrown" : 23056,
- "admin" : {
- "accessesNotInMemory" : 5,
- "pageFaultExceptionsThrown" : 0
- },
- "ceilometer" : {
- "accessesNotInMemory" : 2261201,
- "pageFaultExceptionsThrown" : 23056
- },
- "local" : {
- "accessesNotInMemory" : 3562,
- "pageFaultExceptionsThrown" : 0
- }
- },
- "repl" : {
- "setName" : "ceilometer",
- "setVersion" : 3,
- "ismaster" : true,
- "secondary" : false,
- "hosts" : [
- "192.168.0.7:27017",
- "192.168.0.11:27017",
- "192.168.0.12:27017"
- ],
- "primary" : "192.168.0.7:27017",
- "me" : "192.168.0.7:27017",
- "electionId" : ObjectId("577515a61ee824d38b47d7bd")
- },
- "workingSet" : {
- "note" : "thisIsAnEstimate",
- "pagesInMemory" : 3700127,
- "computationTimeMicros" : 592762,
- "overSeconds" : 288
- },
- "ok" : 1
- }
- ceilometer:PRIMARY>
- some interesting logs:
- <14>Jul 13 10:08:42 node-5 mongod.27017[22112]: [conn441497] command ceilometer.$cmd command: aggregate { aggregate: "meter", pipeline: [ { $match: { timestamp: { $lte: new Date(1468404521711), $gte: new Date(1468403921711) }, counter_name: "cpu_util", project_id: "65f8d3591f2c478e8373cbabcdcd2302", resource_id: "350fe65e-6805-426e-8cc8-a0bd303eac44" } }, { $sort: { timestamp: 1 } }, { $group: { count: { $sum: 1 }, avg: { $avg: "$counter_volume" }, first_timestamp: { $min: "$timestamp" }, min: { $min: "$counter_volume" }, last_timestamp: { $max: "$timestamp" }, max: { $max: "$counter_volume" }, sum: { $sum: "$counter_volume" }, _id: { period_start: { $divide: [ { $subtract: [ { $subtract: [ "$timestamp", new Date(1468403921711) ] }, { $mod: [ { $subtract: [ "$timestamp", new Date(1468403921711) ] }, 300000 ] } ] }, 300000 ] }, name: "$counter_name", unit: "$counter_unit" } } }, { $sort: { _id.period_start: 1 } }, { $project: { count: "$count", min: "$min", first_timestamp: "$first_timestamp", name: "$_id.name", last_timestamp: "$last_timestamp", max: "$max", period_start: "$_id.period_start", avg: "$avg", sum: "$sum", unit: "$_id.unit" } } ], cursor: {}, allowDiskUse: true } keyUpdates:0 numYields:6 locks(micros) r:229097 reslen:104 198ms
- <14>Jul 13 10:08:44 node-5 mongod.27017[22112]: [conn441471] command ceilometer.$cmd command: aggregate { aggregate: "meter", pipeline: [ { $match: { timestamp: { $gt: new Date(1467504000000), $lt: new Date(1468404377000) }, counter_name: "memory.gb", project_id: "29a7498491d843f39bca3fa90b1e442e" } }, { $sort: { timestamp: 1 } }, { $group: { count: { $sum: 1 }, avg: { $avg: "$counter_volume" }, first_timestamp: { $min: "$timestamp" }, min: { $min: "$counter_volume" }, last_timestamp: { $max: "$timestamp" }, max: { $max: "$counter_volume" }, sum: { $sum: "$counter_volume" }, _id: { name: "$counter_name", unit: "$counter_unit" } } }, { $sort: { _id.period_start: 1 } }, { $project: { count: "$count", min: "$min", first_timestamp: "$first_timestamp", name: "$_id.name", last_timestamp: "$last_timestamp", max: "$max", period_start: "$_id.period_start", avg: "$avg", sum: "$sum", unit: "$_id.unit" } } ], cursor: {}, allowDiskUse: true } keyUpdates:0 numYields:42830 locks(micros) r:219517031 reslen:300 145224ms
- <12>Jul 13 10:08:56 node-5 mongod.27017[22112]: [conn398160] ClientCursor::staticYield can't unlock b/c of recursive lock ns: top: { opid: 135779926, active: true, secs_running: 0, microsecs_running: 325, op: "query", ns: "ceilometer.resource", query: { $msg: "query not recording (too large)" }, client: "192.168.0.7:36238", desc: "conn398160", threadId: "0x7f42637e1700", connectionId: 398160, locks: { ^: "w", ^ceilometer: "W" }, waitingForLock: false, numYields: 0, lockStats: { timeLockedMicros: {}, timeAcquiringMicros: { r: 0, w: 164301 } } }
- <14>Jul 13 10:09:20 node-5 mongod.27017[22112]: [conn441479] command ceilometer.$cmd command: aggregate { aggregate: "meter", pipeline: [ { $match: { timestamp: { $gt: new Date(1467331200000), $lt: new Date(1470009600000) }, counter_name: "vcpus" } }, { $sort: { timestamp: 1 } }, { $group: { count: { $sum: 1 }, avg: { $avg: "$counter_volume" }, first_timestamp: { $min: "$timestamp" }, min: { $min: "$counter_volume" }, last_timestamp: { $max: "$timestamp" }, max: { $max: "$counter_volume" }, sum: { $sum: "$counter_volume" }, _id: { name: "$counter_name", unit: "$counter_unit" } } }, { $sort: { _id.period_start: 1 } }, { $project: { count: "$count", min: "$min", first_timestamp: "$first_timestamp", name: "$_id.name", last_timestamp: "$last_timestamp", max: "$max", period_start: "$_id.period_start", avg: "$avg", sum: "$sum", unit: "$_id.unit" } } ], cursor: {}, allowDiskUse: true } keyUpdates:0 numYields:11664 locks(micros) r:245425709 reslen:284 140423ms
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement