pg 6.f6 is stuck unclean for 68549.601723, current state active+remapped, last acting [29,25,6] $ ceph pg 6.f6 query { "state": "active+remapped", "epoch": 2345, "up": [ 29, 25], "acting": [ 29, 25, 6], "actingbackfill": [ "6", "25", "29"], "info": { "pgid": "6.f6", "last_update": "0'0", "last_complete": "0'0", "log_tail": "0'0", "last_user_version": 0, "last_backfill": "MAX", "purged_snaps": "[]", "history": { "epoch_created": 481, "last_epoch_started": 552, "last_epoch_clean": 552, "last_epoch_split": 0, "same_up_since": 547, "same_interval_since": 548, "same_primary_since": 499, "last_scrub": "0'0", "last_scrub_stamp": "2014-09-03 14:57:31.189223", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "2014-08-29 14:56:12.770231", "last_clean_scrub_stamp": "2014-09-03 14:57:31.189223"}, "stats": { "version": "0'0", "reported_seq": "1923", "reported_epoch": "2345", "state": "active+remapped", "last_fresh": "2014-09-04 09:57:50.355631", "last_change": "2014-09-04 09:21:20.357442", "last_active": "2014-09-04 09:57:50.355631", "last_clean": "2014-09-03 14:57:31.189950", "last_became_active": "0.000000", "last_unstale": "2014-09-04 09:57:50.355631", "mapping_epoch": 547, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 481, "last_epoch_clean": 552, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "2014-09-03 14:57:31.189223", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "2014-08-29 14:56:12.770231", "last_clean_scrub_stamp": "2014-09-03 14:57:31.189223", "log_size": 0, "ondisk_log_size": 0, "stats_invalid": "0", "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_degraded": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0}, "stat_cat_sum": {}, "up": [ 29, 25], "acting": [ 29, 25, 6], "up_primary": 29, "acting_primary": 29}, "empty": 1, "dne": 0, "incomplete": 0, "last_epoch_started": 552, "hit_set_history": { "current_last_update": "0'0", "current_last_stamp": "0.000000", "current_info": { "begin": "0.000000", "end": "0.000000", "version": "0'0"}, "history": []}}, "peer_info": [ { "peer": "6", "pgid": "6.f6", "last_update": "0'0", "last_complete": "0'0", "log_tail": "0'0", "last_user_version": 0, "last_backfill": "MAX", "purged_snaps": "[]", "history": { "epoch_created": 481, "last_epoch_started": 552, "last_epoch_clean": 552, "last_epoch_split": 0, "same_up_since": 547, "same_interval_since": 548, "same_primary_since": 499, "last_scrub": "0'0", "last_scrub_stamp": "2014-09-03 14:57:31.189223", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "2014-08-29 14:56:12.770231", "last_clean_scrub_stamp": "2014-09-03 14:57:31.189223"}, "stats": { "version": "0'0", "reported_seq": "91", "reported_epoch": "540", "state": "peering", "last_fresh": "2014-08-19 15:33:53.810263", "last_change": "2014-08-19 15:33:52.801947", "last_active": "2014-08-19 15:33:48.754220", "last_clean": "2014-08-19 15:30:35.387701", "last_became_active": "0.000000", "last_unstale": "2014-08-19 15:33:53.810263", "mapping_epoch": 547, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 481, "last_epoch_clean": 524, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "2014-08-19 14:55:45.492402", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "2014-08-15 14:55:30.525252", "last_clean_scrub_stamp": "2014-08-19 14:55:45.492402", "log_size": 0, "ondisk_log_size": 0, "stats_invalid": "0", "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_degraded": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0}, "stat_cat_sum": {}, "up": [ 29, 25], "acting": [ 29, 25, 6], "up_primary": 29, "acting_primary": 29}, "empty": 1, "dne": 0, "incomplete": 0, "last_epoch_started": 552, "hit_set_history": { "current_last_update": "0'0", "current_last_stamp": "0.000000", "current_info": { "begin": "0.000000", "end": "0.000000", "version": "0'0"}, "history": []}}, { "peer": "25", "pgid": "6.f6", "last_update": "0'0", "last_complete": "0'0", "log_tail": "0'0", "last_user_version": 0, "last_backfill": "MAX", "purged_snaps": "[]", "history": { "epoch_created": 481, "last_epoch_started": 552, "last_epoch_clean": 552, "last_epoch_split": 0, "same_up_since": 0, "same_interval_since": 0, "same_primary_since": 0, "last_scrub": "0'0", "last_scrub_stamp": "2014-09-03 14:57:31.189223", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "2014-08-29 14:56:12.770231", "last_clean_scrub_stamp": "2014-09-03 14:57:31.189223"}, "stats": { "version": "0'0", "reported_seq": "0", "reported_epoch": "0", "state": "inactive", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_unstale": "0.000000", "mapping_epoch": 0, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "log_size": 0, "ondisk_log_size": 0, "stats_invalid": "0", "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_degraded": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0}, "stat_cat_sum": {}, "up": [], "acting": [], "up_primary": -1, "acting_primary": -1}, "empty": 1, "dne": 0, "incomplete": 0, "last_epoch_started": 552, "hit_set_history": { "current_last_update": "0'0", "current_last_stamp": "0.000000", "current_info": { "begin": "0.000000", "end": "0.000000", "version": "0'0"}, "history": []}}], "recovery_state": [ { "name": "Started\/Primary\/Active", "enter_time": "2014-09-04 09:21:17.036190", "might_have_unfound": [], "recovery_progress": { "backfill_targets": [], "waiting_on_backfill": [], "last_backfill_started": "0\/\/0\/\/-1", "backfill_info": { "begin": "0\/\/0\/\/-1", "end": "0\/\/0\/\/-1", "objects": []}, "peer_backfill_info": [], "backfills_in_flight": [], "recovering": [], "pg_backend": { "pull_from_peer": [], "pushing": []}}, "scrub": { "scrubber.epoch_start": "539", "scrubber.active": 0, "scrubber.block_writes": 0, "scrubber.waiting_on": 0, "scrubber.waiting_on_whom": []}}, { "name": "Started", "enter_time": "2014-09-04 09:21:08.223200"}], "agent_state": {}}