ceph-pg_unfound

# ceph -s
   health HEALTH_WARN 1 pgs recovering; 50 pgs stale; 50 pgs stuck stale; 1 pgs stuck unclean; recovery 7714/2278940 degraded (0.338%); 3857/1139470 unfound (0.338%)
   monmap e10: 3 mons at {a=172.16.0.3:6789/0,b=172.16.0.4:6789/0,c=172.16.0.2:6789/0}, election epoch 350, quorum 0,1,2 a,b,c
   osdmap e4832: 8 osds: 8 up, 8 in
    pgmap v2124426: 1192 pgs: 1140 active+clean, 50 stale+active+clean, 1 active+clean+scrubbing+deep, 1 active+recovering; 4427 GB data, 8834 GB used, 9660 GB / 18588 GB avail; 7714/2278940 degraded (0.338%); 3857/1139470 unfound (0.338%)
   mdsmap e1378: 1/1/1 up {0=0=up:active}

# ceph health detail
HEALTH_WARN 1 pgs recovering; 50 pgs stale; 50 pgs stuck stale; 1 pgs stuck unclean; recovery 7714/2278940 degraded (0.338%); 3857/1139470 unfound (0.338%)
pg 11.2 is stuck unclean since forever, current state active+recovering, last acting [6,8]
pg 2.65 is stuck stale for 2067254.371209, current state stale+active+clean, last acting [7,0]
pg 0.67 is stuck stale for 2067254.371228, current state stale+active+clean, last acting [7,0]
pg 1.66 is stuck stale for 2067254.371243, current state stale+active+clean, last acting [7,0]
pg 7.60 is stuck stale for 2067254.371256, current state stale+active+clean, last acting [7,0]
pg 2.64 is stuck stale for 2067254.371258, current state stale+active+clean, last acting [0,8]
pg 0.66 is stuck stale for 2067254.371270, current state stale+active+clean, last acting [0,8]
pg 1.65 is stuck stale for 2067254.371274, current state stale+active+clean, last acting [0,8]
pg 1.60 is stuck stale for 2067254.371273, current state stale+active+clean, last acting [1,7]
pg 0.61 is stuck stale for 2067254.371283, current state stale+active+clean, last acting [1,7]
pg 7.5a is stuck stale for 2067254.371283, current state stale+active+clean, last acting [1,7]
pg 2.5f is stuck stale for 2067254.371291, current state stale+active+clean, last acting [1,7]
pg 7.5f is stuck stale for 2067254.371288, current state stale+active+clean, last acting [0,8]
pg 0.42 is stuck stale for 2067254.371221, current state stale+active+clean, last acting [7,1]
pg 2.40 is stuck stale for 2067254.371231, current state stale+active+clean, last acting [7,1]
pg 1.41 is stuck stale for 2067254.371234, current state stale+active+clean, last acting [7,1]
pg 7.3b is stuck stale for 2067254.371232, current state stale+active+clean, last acting [7,1]
pg 1.2c is stuck stale for 2067254.371192, current state stale+active+clean, last acting [1,8]
pg 0.2d is stuck stale for 2067254.371202, current state stale+active+clean, last acting [1,8]
pg 2.2b is stuck stale for 2067254.371199, current state stale+active+clean, last acting [1,8]
pg 0.28 is stuck stale for 2067254.371206, current state stale+active+clean, last acting [0,8]
pg 7.21 is stuck stale for 2067254.371211, current state stale+active+clean, last acting [0,8]
pg 1.27 is stuck stale for 2067254.371221, current state stale+active+clean, last acting [0,8]
pg 2.26 is stuck stale for 2067254.371220, current state stale+active+clean, last acting [0,8]
pg 7.26 is stuck stale for 2067254.371224, current state stale+active+clean, last acting [1,8]
pg 7.10 is stuck stale for 2067254.371200, current state stale+active+clean, last acting [7,0]
pg 1.16 is stuck stale for 2067254.371208, current state stale+active+clean, last acting [7,0]
pg 0.17 is stuck stale for 2067254.371218, current state stale+active+clean, last acting [7,0]
pg 2.15 is stuck stale for 2067254.371226, current state stale+active+clean, last acting [7,0]
pg 7.bb is stuck stale for 2067254.371886, current state stale+active+clean, last acting [7,1]
pg 7.a1 is stuck stale for 2067254.371823, current state stale+active+clean, last acting [0,8]
pg 7.a6 is stuck stale for 2067254.371817, current state stale+active+clean, last acting [1,8]
pg 0.9e is stuck stale for 2067254.371814, current state stale+active+clean, last acting [7,1]
pg 2.9c is stuck stale for 2067254.371824, current state stale+active+clean, last acting [7,1]
pg 1.9d is stuck stale for 2067254.371830, current state stale+active+clean, last acting [7,1]
pg 2.98 is stuck stale for 2067254.371832, current state stale+active+clean, last acting [7,0]
pg 0.9a is stuck stale for 2067254.371844, current state stale+active+clean, last acting [7,0]
pg 1.99 is stuck stale for 2067254.371847, current state stale+active+clean, last acting [7,0]
pg 2.95 is stuck stale for 2067254.371848, current state stale+active+clean, last acting [1,7]
pg 0.97 is stuck stale for 2067254.371860, current state stale+active+clean, last acting [1,7]
pg 1.96 is stuck stale for 2067254.371870, current state stale+active+clean, last acting [1,7]
pg 7.90 is stuck stale for 2067254.371885, current state stale+active+clean, last acting [1,7]
pg 7.93 is stuck stale for 2067254.371879, current state stale+active+clean, last acting [7,0]
pg 1.88 is stuck stale for 2067254.371855, current state stale+active+clean, last acting [7,1]
pg 0.89 is stuck stale for 2067254.371865, current state stale+active+clean, last acting [7,1]
pg 7.82 is stuck stale for 2067254.371861, current state stale+active+clean, last acting [7,1]
pg 2.87 is stuck stale for 2067254.371868, current state stale+active+clean, last acting [7,1]
pg 2.80 is stuck stale for 2067254.371867, current state stale+active+clean, last acting [1,8]
pg 0.82 is stuck stale for 2067254.371879, current state stale+active+clean, last acting [1,8]
pg 1.81 is stuck stale for 2067254.371875, current state stale+active+clean, last acting [1,8]
pg 7.7b is stuck stale for 2067254.371870, current state stale+active+clean, last acting [1,8]
pg 11.2 is active+recovering, acting [6,8], 3857 unfound
recovery 7714/2278940 degraded (0.338%); 3857/1139470 unfound (0.338%)

# ceph pg 1.81 query
i don't have pgid 1.81


# ceph pg 11.2 query

{ "state": "active+recovering",
  "epoch": 4832,
  "up": [
        6,
        8],
  "acting": [
        6,
        8],
  "info": { "pgid": "11.2",
      "last_update": "4084'7092",
      "last_complete": "2757'407",
      "log_tail": "0'0",
      "last_backfill": "MAX",
      "purged_snaps": "[]",
      "history": { "epoch_created": 2726,
          "last_epoch_started": 4827,
          "last_epoch_clean": 3294,
          "last_epoch_split": 0,
          "same_up_since": 4826,
          "same_interval_since": 4826,
          "same_primary_since": 4813,
          "last_scrub": "0'0",
          "last_scrub_stamp": "2013-03-04 15:30:15.335201",
          "last_deep_scrub": "0'0",
          "last_deep_scrub_stamp": "2013-03-04 15:30:15.335201",
          "last_clean_scrub_stamp": "2013-03-04 15:30:15.335201"},
      "stats": { "version": "4084'7092",
          "reported": "4813'3473",
          "state": "active+recovering",
          "last_fresh": "2013-04-11 14:45:19.262633",
          "last_change": "2013-04-05 12:54:57.430937",
          "last_active": "2013-04-11 14:45:19.262633",
          "last_clean": "0.000000",
          "last_unstale": "2013-04-11 14:45:19.262633",
          "mapping_epoch": 4820,
          "log_start": "0'0",
          "ondisk_log_start": "0'0",
          "created": 2726,
          "last_epoch_clean": 2726,
          "parent": "0.0",
          "parent_split_bits": 0,
          "last_scrub": "0'0",
          "last_scrub_stamp": "2013-03-04 15:30:15.335201",
          "last_deep_scrub": "0'0",
          "last_deep_scrub_stamp": "2013-03-04 15:30:15.335201",
          "last_clean_scrub_stamp": "2013-03-04 15:30:15.335201",
          "log_size": 0,
          "ondisk_log_size": 0,
          "stats_invalid": "0",
          "stat_sum": { "num_bytes": 17382162544,
              "num_objects": 4146,
              "num_object_clones": 0,
              "num_object_copies": 0,
              "num_objects_missing_on_primary": 0,
              "num_objects_degraded": 0,
              "num_objects_unfound": 0,
              "num_read": 0,
              "num_read_kb": 0,
              "num_write": 7078,
              "num_write_kb": 18952638,
              "num_scrub_errors": 0,
              "num_objects_recovered": 8237,
              "num_bytes_recovered": 34517590688,
              "num_keys_recovered": 0},
          "stat_cat_sum": {},
          "up": [
                6,
                8],
          "acting": [
                6,
                8]},
      "empty": 0,
      "dne": 0,
      "incomplete": 0,
      "last_epoch_started": 4827},
  "recovery_state": [
        { "name": "Started\/Primary\/Active",
          "enter_time": "2013-04-05 12:54:57.241401",
          "might_have_unfound": [
                { "osd": 7,
                  "status": "querying"},
                { "osd": 8,
                  "status": "already probed"}],
          "recovery_progress": { "backfill_target": -1,
              "waiting_on_backfill": 0,
              "backfill_pos": "0\/\/0\/\/-1",
              "backfill_info": { "begin": "0\/\/0\/\/-1",
                  "end": "0\/\/0\/\/-1",
                  "objects": []},
              "peer_backfill_info": { "begin": "0\/\/0\/\/-1",
                  "end": "0\/\/0\/\/-1",
                  "objects": []},
              "backfills_in_flight": [],
              "pull_from_peer": [],
              "pushing": []},
          "scrub": { "scrubber.epoch_start": "0",
              "scrubber.active": 0,
              "scrubber.block_writes": 0,
              "scrubber.finalizing": 0,
              "scrubber.waiting_on": 0,
              "scrubber.waiting_on_whom": []}},
        { "name": "Started",
          "enter_time": "2013-04-05 12:54:56.037918"}]}

# ceph osd tree

# id    weight  type name       up/down reweight
-1      16.8    room server-raum-keller
-2      16.8            rack rack-daywalker
-4      14.8                    storage daywalker-data
-6      7.4                             host dwssrv1
0       3.7                                     osd.0   up      1
1       3.7                                     osd.1   up      1
-7      7.4                             host dwssrv2
3       3.7                                     osd.3   up      1
4       3.7                                     osd.4   up      1
-5      2                       storage daywalker-archive
-8      2                               host dwssrv1-archive
7       1                                       osd.7   up      1
8       1                                       osd.8   up      1
-9      2                               host dwssrv2-archive
5       1                                       osd.5   up      1
6       1                                       osd.6   up      1

# rbd info dws-archive/vdi-dws-archive
rbd image 'vdi-dws-archive':
        size 1000 GB in 256000 objects
        order 22 (4096 KB objects)
        block_name_prefix: rb.0.e6d2.238e1f29
        format: 1


##
# the crushmap
#
# begin crush map
tunable choose_local_tries 0
tunable choose_local_fallback_tries 0
tunable choose_total_tries 50
tunable chooseleaf_descend_once 1

# devices
device 0 osd.0
device 1 osd.1
device 2 device2
device 3 osd.3
device 4 osd.4
device 5 osd.5
device 6 osd.6
device 7 osd.7
device 8 osd.8

# types
type 0 osd
type 1 host
type 2 storage
type 3 rack
type 4 room

# buckets
host dwssrv1 {
        id -6           # do not change unnecessarily
        # weight 7.400
        alg straw
        hash 0  # rjenkins1
        item osd.0 weight 3.700
        item osd.1 weight 3.700
}
host dwssrv2 {
        id -7           # do not change unnecessarily
        # weight 7.400
        alg straw
        hash 0  # rjenkins1
        item osd.3 weight 3.700
        item osd.4 weight 3.700
}
storage daywalker-data {
        id -4           # do not change unnecessarily
        # weight 14.800
        alg straw
        hash 0  # rjenkins1
        item dwssrv1 weight 7.400
        item dwssrv2 weight 7.400
}
host dwssrv1-archive {
        id -8           # do not change unnecessarily
        # weight 2.000
        alg straw
        hash 0  # rjenkins1
        item osd.7 weight 1.000
        item osd.8 weight 1.000
}
host dwssrv2-archive {
        id -9           # do not change unnecessarily
        # weight 2.000
        alg straw
        hash 0  # rjenkins1
        item osd.5 weight 1.000
        item osd.6 weight 1.000
}
storage daywalker-archive {
        id -5           # do not change unnecessarily
        # weight 4.000
        alg straw
        hash 0  # rjenkins1
        item dwssrv1-archive weight 2.000
        item dwssrv2-archive weight 2.000
}
rack rack-daywalker {
        id -2           # do not change unnecessarily
        # weight 16.800
        alg straw
        hash 0  # rjenkins1
        item daywalker-data weight 14.800
        item daywalker-archive weight 2.000
}
room server-raum-keller {
        id -1           # do not change unnecessarily
        # weight 16.800
        alg straw
        hash 0  # rjenkins1
        item rack-daywalker weight 16.800
}

# rules
rule data {
        ruleset 0
        type replicated
        min_size 1
        max_size 10
        step take server-raum-keller
        step chooseleaf firstn 0 type host
        step emit
}
rule metadata {
        ruleset 1
        type replicated
        min_size 1
        max_size 10
        step take server-raum-keller
        step chooseleaf firstn 0 type host
        step emit
}
rule rbd {
        ruleset 2
        type replicated
        min_size 1
        max_size 10
        step take server-raum-keller
        step chooseleaf firstn 0 type host
        step emit
}
rule dws-vdi {
        ruleset 3
        type replicated
        min_size 1
        max_size 10
        step take daywalker-data
        step chooseleaf firstn 0 type host
        step emit
}
rule dws-archive {
        ruleset 4
        type replicated
        min_size 1
        max_size 10
        step take daywalker-archive
        step chooseleaf firstn 0 type host
        step emit
}
rule dws-test {
        ruleset 5
        type replicated
        min_size 1
        max_size 10
        step take rack-daywalker
        step chooseleaf firstn 0 type host
        step emit
}

# end crush map