UNHANDLED EXCEPTION (2015-02-19 10:48:46.318000)
BRANCH: develop TIP: 15ae966a19a4
PROCESS: ./scripts/noc-web.py
ERROR FINGERPRINT: 00a012e5-21da-5e49-ab5b-eb13ee0e9d82
WORKING DIRECTORY: /opt/noc
EXCEPTION: <type 'exceptions.KeyError'> 56
START OF TRACEBACK
------------------------------------------------------------------------
File: fm/apps/reportoutages/views.py (Line: 58)
Function: get_data
   51             mo = {}
   52             while mos:
   53                 for o in ManagedObject.objects.filter(id__in=mos[:chunk]):
   54                     mo[o.id] = o
   55                 mos = mos[chunk:]
   56             r = []
   57             for o in sorted(otime, key=lambda x: -otime[x]):
   58 ==>             m = mo[o]
   59                 dt = otime[o]
   60                 downtime = "%02d:%02d:%02d" % (
   61                     (dt // 3600) % 24,
   62                     (dt // 60) % 60,
   63                     dt % 60
   64                 )
Variables:
               otime =
defaultdict(<type 'int'>, {2: 0.665, 515: 604800.0, 1028: 125.478, 517: 13116.209, 1030: 1.491, 524: 14.937, 1026: 153.835, 529: 140.065, 1045: 29.704, 5309: 21.133, 1054: 34.886, 751: 2148.764, 1060: 93.234, 860: 40.844, 1066: 943.98, 1072: 1.727, 1074: 24.702, 565: 604800.0, 55: 21.738, 56: 604800.0, 57: 604800.0, 59: 18.416, 573: 604800.0, 5302: 22.534, 582: 604800.0, 584: 119.293, 587: 72.39, 81: 604800.0, 1108: 3049.131, 270: 75.086, 88: 152.297, 1114: 2.886, 92: 142.503, 605: 3004.731, 94: 17.547, 608: 604800.0, 668: 81.128, 103: 141.663, 621: 604800.0, 623: 604800.0, 1137: 3.622, 5395: 88.105, 1142: 15.549, 631: 604800.0, 633: 604800.0, 1148: 135.201, 1149: 168.541, 644: 604800.0, 646: 46539.000949, 1143: 29.708, 652: 604800.0, 653: 604800.0, 654: 157.669, 655: 18.162, 656: 19.177, 145: 134.768, 660: 10.175, 661: 17.897, 662: 18.465, 663: 17.102, 1176: 1612.588, 665: 14.575, 154: 604800.0, 667: 17.654, 156: 2029.32, 670: 15.621, 672: 16.134, 674: 15.353, 1189: 604800.0, 1393: 604800.0, 683: 10.231, 684: 27.883, 685: 90.565, 687: 12.12, 5296: 17.454, 1201: 54.5, 690: 108.133, 5299: 20.474, 694: 72.312, 696: 17.319, 5305: 106.665, 698: 44.121, 1311: 1.438, 701: 604800.0, 702: 87.974, 703: 23.656, 704: 15.488, 5313: 17.44, 195: 604800.0, 708: 604800.0, 709: 604800.0, 710: 604800.0, 801: 29.62, 5320: 41142.871, 5321: 20.082, 714: 85.304, 1399: 604800.0, 1229: 2729.972, 1230: 150.392, 1233: 609.593, 5331: 17.414, 212: 604800.0, 213: 128.91, 216: 604800.0, 804: 24.48, 5338: 20.499, 731: 125.989, 732: 604800.0, 805: 29.65, 224: 110.274, 228: 604800.0, 806: 34.832, 232: 155.925, 746: 162.991, 5355: 22.349, 1261: 29.879, 750: 605.231, 5359: 45.223, 1264: 123.581, 5362: 17.469, 1235: 153.924, 1268: 135.277, 248: 159.691, 5369: 20.684, 253: 122.2, 5374: 22.066, 769: 122.606, 774: 142.846, 263: 604800.0, 776: 162.051, 777: 161.977, 778: 148.29, 779: 153.805, 780: 161.99, 781: 149.301, 5390: 103.669, 783: 148.268, 1298: 154.427, 367: 604800.0, 1306: 3.999, 797: 32.749, 5407: 74.018, 5409: 100.692, 803: 29.667, 1316: 3108.455, 5413: 21.501, 5414: 22.131, 807: 0.967, 5417: 1180.116, 5418: 1184.503, 5419: 1300.285, 5420: 22972.554, 823: 29.961, 824: 2.14, 1338: 2.249, 5344: 107.785, 835: 150.15, 328: 604800.0, 848: 844.541, 1363: 84.104, 348: 604800.0, 1375: 78.105, 1376: 604800.0, 1384: 604800.0, 1391: 604800.0, 369: 601396.891051, 1395: 604800.0, 1397: 604800.0, 886: 0.773, 375: 129.778, 1402: 604800.0, 6805: 172544.756051, 388: 74.208, 833: 14.096, 908: 2.69, 911: 4532.796, 664: 18.159, 916: 153.859, 407: 15.293, 408: 604800.0, 409: 114.685, 410: 12.115, 411: 604800.0, 413: 111.04, 927: 153.811, 928: 149.348, 929: 153.851, 923: 0.328, 934: 76.091, 935: 24.477, 937: 130.088, 940: 3.664, 941: 2.745, 946: 3114.192, 440: 143.935, 456: 125.67, 459: 604800.0, 975: 34.956, 978: 29.565, 468: 138.841, 1273: 3.639, 472: 105.409, 473: 93.274, 474: 604800.0, 987: 3.988, 5352: 21.243, 480: 15.73, 994: 2.187, 1019: 3.644, 1277: 161.933, 1013: 1.61, 1227: 604800.0, 507: 604800.0, 1021: 4.211})
                   b = datetime.datetime(2015, 2, 12, 10, 48, 46, 198051)
                  td = 604800.0
                   d = datetime.timedelta(7)
             outages =
defaultdict(<type 'list'>, {2: [<Outage: 2>], 515: [<Outage: 515>], 1028: [<Outage: 1028>], 517: [<Outage: 517>], 1030: [<Outage: 1030>], 524: [<Outage: 524>], 1026: [<Outage: 1026>], 529: [<Outage: 529>], 1045: [<Outage: 1045>, <Outage: 1045>], 5309: [<Outage: 5309>], 1054: [<Outage: 1054>, <Outage: 1054>], 751: [<Outage: 751>], 1060: [<Outage: 1060>], 860: [<Outage: 860>], 1066: [<Outage: 1066>], 1072: [<Outage: 1072>], 1074: [<Outage: 1074>, <Outage: 1074>], 565: [<Outage: 565>], 55: [<Outage: 55>], 56: [<Outage: 56>], 57: [<Outage: 57>], 59: [<Outage: 59>], 573: [<Outage: 573>], 5302: [<Outage: 5302>], 582: [<Outage: 582>], 584: [<Outage: 584>], 587: [<Outage: 587>], 81: [<Outage: 81>], 1108: [<Outage: 1108>], 270: [<Outage: 270>], 88: [<Outage: 88>], 1114: [<Outage: 1114>], 92: [<Outage: 92>], 605: [<Outage: 605>], 94: [<Outage: 94>], 608: [<Outage: 608>], 668: [<Outage: 668>], 103: [<Outage: 103>], 621: [<Outage: 621>], 623: [<Outage: 623>], 1137: [<Outage: 1137>], 5395: [<Outage: 5395>], 1142: [<Outage: 1142>, <Outage: 1142>], 631: [<Outage: 631>], 633: [<Outage: 633>, <Outage: 633>], 1148: [<Outage: 1148>, <Outage: 1148>, <Outage: 1148>, <Outage: 1148>, <Outage: 1148>, <Outage: 1148>, <Outage: 1148>, <Outage: 1148>, <Outage: 1148>, <Outage: 1148>, <Outage: 1148>, <Outage: 1148>, <Outage: 1148>, <Outage: 1148>, <Outage: 1148>, <Outage: 1148>, <Outage: 1148>, <Outage: 1148>, <Outage: 1148>, <Outage: 1148>], 1149: [<Outage: 1149>, <Outage: 1149>, <Outage: 1149>, <Outage: 1149>, <Outage: 1149>, <Outage: 1149>, <Outage: 1149>, <Outage: 1149>, <Outage: 1149>, <Outage: 1149>, <Outage: 1149>, <Outage: 1149>, <Outage: 1149>, <Outage: 1149>, <Outage: 1149>, <Outage: 1149>, <Outage: 1149>, <Outage: 1149>, <Outage: 1149>, <Outage: 1149>, <Outage: 1149>], 644: [<Outage: 644>], 646: [<Outage: 646>], 1143: [<Outage: 1143>, <Outage: 1143>], 652: [<Outage: 652>], 653: [<Outage: 653>], 654: [<Outage: 654>], 655: [<Outage: 655>], 656: [<Outage: 656>], 145: [<Outage: 145>], 660: [<Outage: 660>], 661: [<Outage: 661>], 662: [<Outage: 662>], 663: [<Outage: 663>], 1176: [<Outage: 1176>, <Outage: 1176>, <Outage: 1176>, <Outage: 1176>, <Outage: 1176>], 665: [<Outage: 665>], 154: [<Outage: 154>], 667: [<Outage: 667>, <Outage: 667>, <Outage: 667>], 156: [<Outage: 156>, <Outage: 156>], 670: [<Outage: 670>], 672: [<Outage: 672>], 674: [<Outage: 674>], 1189: [<Outage: 1189>], 1393: [<Outage: 1393>], 683: [<Outage: 683>], 684: [<Outage: 684>], 685: [<Outage: 685>], 687: [<Outage: 687>], 5296: [<Outage: 5296>], 1201: [<Outage: 1201>], 690: [<Outage: 690>], 5299: [<Outage: 5299>], 694: [<Outage: 694>], 696: [<Outage: 696>], 5305: [<Outage: 5305>], 698: [<Outage: 698>], 1311: [<Outage: 1311>], 701: [<Outage: 701>], 702: [<Outage: 702>, <Outage: 702>, <Outage: 702>, <Outage: 702>, <Outage: 702>, <Outage: 702>, <Outage: 702>, <Outage: 702>, <Outage: 702>, <Outage: 702>], 703: [<Outage: 703>], 704: [<Outage: 704>], 5313: [<Outage: 5313>], 195: [<Outage: 195>], 708: [<Outage: 708>], 709: [<Outage: 709>], 710: [<Outage: 710>], 801: [<Outage: 801>, <Outage: 801>, <Outage: 801>, <Outage: 801>, <Outage: 801>], 5320: [<Outage: 5320>, <Outage: 5320>, <Outage: 5320>, <Outage: 5320>, <Outage: 5320>, <Outage: 5320>, <Outage: 5320>], 5321: [<Outage: 5321>], 714: [<Outage: 714>], 1399: [<Outage: 1399>], 1229: [<Outage: 1229>], 1230: [<Outage: 1230>, <Outage: 1230>, <Outage: 1230>, <Outage: 1230>, <Outage: 1230>, <Outage: 1230>, <Outage: 1230>, <Outage: 1230>, <Outage: 1230>, <Outage: 1230>, <Outage: 1230>, <Outage: 1230>, <Outage: 1230>, <Outage: 1230>, <Outage: 1230>, <Outage: 1230>, <Outage: 1230>, <Outage: 1230>, <Outage: 1230>, <Outage: 1230>, <Outage: 1230>, <Outage: 1230>, <Outage: 1230>], 1233: [<Outage: 1233>, <Outage: 1233>], 5331: [<Outage: 5331>], 212: [<Outage: 212>], 213: [<Outage: 213>], 216: [<Outage: 216>], 804: [<Outage: 804>, <Outage: 804>, <Outage: 804>, <Outage: 804>, <Outage: 804>, <Outage: 804>], 5338: [<Outage: 5338>], 731: [<Outage: 731>], 732: [<Outage: 732>], 805: [<Outage: 805>, <Outage: 805>, <Outage: 805>, <Outage: 805>, <Outage: 805>, <Outage: 805>], 224: [<Outage: 224>], 228: [<Outage: 228>], 806: [<Outage: 806>, <Outage: 806>, <Outage: 806>, <Outage: 806>, <Outage: 806>, <Outage: 806>], 232: [<Outage: 232>], 746: [<Outage: 746>], 5355: [<Outage: 5355>], 1261: [<Outage: 1261>, <Outage: 1261>, <Outage: 1261>, <Outage: 1261>, <Outage: 1261>], 750: [<Outage: 750>], 5359: [<Outage: 5359>], 1264: [<Outage: 1264>, <Outage: 1264>, <Outage: 1264>, <Outage: 1264>, <Outage: 1264>, <Outage: 1264>, <Outage: 1264>, <Outage: 1264>, <Outage: 1264>, <Outage: 1264>, <Outage: 1264>, <Outage: 1264>, <Outage: 1264>, <Outage: 1264>, <Outage: 1264>, <Outage: 1264>, <Outage: 1264>, <Outage: 1264>], 5362: [<Outage: 5362>], 1235: [<Outage: 1235>], 1268: [<Outage: 1268>, <Outage: 1268>, <Outage: 1268>, <Outage: 1268>, <Outage: 1268>, <Outage: 1268>, <Outage: 1268>, <Outage: 1268>, <Outage: 1268>, <Outage: 1268>, <Outage: 1268>, <Outage: 1268>, <Outage: 1268>, <Outage: 1268>, <Outage: 1268>, <Outage: 1268>, <Outage: 1268>, <Outage: 1268>, <Outage: 1268>], 248: [<Outage: 248>], 5369: [<Outage: 5369>], 253: [<Outage: 253>], 5374: [<Outage: 5374>], 769: [<Outage: 769>, <Outage: 769>, <Outage: 769>, <Outage: 769>, <Outage: 769>, <Outage: 769>, <Outage: 769>, <Outage: 769>, <Outage: 769>, <Outage: 769>, <Outage: 769>, <Outage: 769>, <Outage: 769>, <Outage: 769>, <Outage: 769>, <Outage: 769>, <Outage: 769>, <Outage: 769>, <Outage: 769>, <Outage: 769>, <Outage: 769>], 774: [<Outage: 774>, <Outage: 774>], 263: [<Outage: 263>], 776: [<Outage: 776>], 777: [<Outage: 777>, <Outage: 777>], 778: [<Outage: 778>], 779: [<Outage: 779>], 780: [<Outage: 780>], 781: [<Outage: 781>], 5390: [<Outage: 5390>], 783: [<Outage: 783>], 1298: [<Outage: 1298>, <Outage: 1298>, <Outage: 1298>, <Outage: 1298>, <Outage: 1298>, <Outage: 1298>, <Outage: 1298>, <Outage: 1298>, <Outage: 1298>, <Outage: 1298>, <Outage: 1298>, <Outage: 1298>, <Outage: 1298>, <Outage: 1298>, <Outage: 1298>, <Outage: 1298>, <Outage: 1298>, <Outage: 1298>, <Outage: 1298>, <Outage: 1298>, <Outage: 1298>], 367: [<Outage: 367>], 1306: [<Outage: 1306>], 797: [<Outage: 797>], 5407: [<Outage: 5407>], 5409: [<Outage: 5409>], 803: [<Outage: 803>, <Outage: 803>, <Outage: 803>, <Outage: 803>, <Outage: 803>], 1316: [<Outage: 1316>], 5413: [<Outage: 5413>], 5414: [<Outage: 5414>], 807: [<Outage: 807>], 5417: [<Outage: 5417>], 5418: [<Outage: 5418>], 5419: [<Outage: 5419>], 5420: [<Outage: 5420>], 823: [<Outage: 823>, <Outage: 823>, <Outage: 823>, <Outage: 823>, <Outage: 823>], 824: [<Outage: 824>], 1338: [<Outage: 1338>], 5344: [<Outage: 5344>], 835: [<Outage: 835>, <Outage: 835>], 328: [<Outage: 328>, <Outage: 328>], 848: [<Outage: 848>], 1363: [<Outage: 1363>], 348: [<Outage: 348>], 1375: [<Outage: 1375>], 1376: [<Outage: 1376>], 1384: [<Outage: 1384>], 1391: [<Outage: 1391>], 369: [<Outage: 369>], 1395: [<Outage: 1395>], 1397: [<Outage: 1397>], 886: [<Outage: 886>], 375: [<Outage: 375>], 1402: [<Outage: 1402>], 6805: [<Outage: 6805>], 388: [<Outage: 388>], 833: [<Outage: 833>], 908: [<Outage: 908>], 911: [<Outage: 911>], 664: [<Outage: 664>], 916: [<Outage: 916>], 407: [<Outage: 407>], 408: [<Outage: 408>], 409: [<Outage: 409>], 410: [<Outage: 410>], 411: [<Outage: 411>], 413: [<Outage: 413>], 927: [<Outage: 927>], 928: [<Outage: 928>], 929: [<Outage: 929>], 923: [<Outage: 923>], 934: [<Outage: 934>], 935: [<Outage: 935>, <Outage: 935>, <Outage: 935>, <Outage: 935>, <Outage: 935>], 937: [<Outage: 937>], 940: [<Outage: 940>], 941: [<Outage: 941>], 946: [<Outage: 946>], 440: [<Outage: 440>], 456: [<Outage: 456>], 459: [<Outage: 459>], 975: [<Outage: 975>, <Outage: 975>, <Outage: 975>, <Outage: 975>, <Outage: 975>], 978: [<Outage: 978>, <Outage: 978>, <Outage: 978>, <Outage: 978>, <Outage: 978>], 468: [<Outage: 468>], 1273: [<Outage: 1273>], 472: [<Outage: 472>], 473: [<Outage: 473>], 474: [<Outage: 474>], 987: [<Outage: 987>], 5352: [<Outage: 5352>], 480: [<Outage: 480>], 994: [<Outage: 994>], 1019: [<Outage: 1019>], 1277: [<Outage: 1277>], 1013: [<Outage: 1013>], 1227: [<Outage: 1227>], 507: [<Outage: 507>], 1021: [<Outage: 1021>, <Outage: 1021>, <Outage: 1021>]})
                 mos = []
                stop = datetime.datetime(2015, 2, 19, 10, 17, 47, 473000)
                  mo =