SHARE
TWEET

Scorecard validation tests

mjaniec Jan 9th, 2014 173 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. # visit: http://www.reakkt.com/2014/01/basics-of-scorecard-model-validation.html
  2.  
  3. ### validation tests examples
  4.  
  5. library(ineq)
  6.  
  7. source("validation tests.R")
  8.  
  9. ## GAUROC
  10.  
  11. n <- 100 # number of counterparties
  12. reference.ratings <- ceiling(runif(n,1,21))
  13. test.ratings      <- ceiling(runif(n,1,21))
  14.  
  15. # random test ratings
  16.  
  17. GAUROC(test.ratings, reference.ratings) # ~0.5
  18.  
  19. cor(test.ratings, reference.ratings)
  20.  
  21. #  test ratings identical with the referece
  22.  
  23. test.ratings <- reference.ratings
  24.  
  25. GAUROC(test.ratings, reference.ratings) # 1
  26.  
  27. cor(test.ratings, reference.ratings)
  28.  
  29. # some reference ratings replaced with random values
  30.  
  31. replacement.ratio <- 0.1
  32. replacement.idx   <- sample(1:n,round(n*disturbance.ratio))
  33.  
  34. replaced.ratings  <- ceiling(runif(round(n*disturbance.ratio),1,21))
  35.  
  36. disturbed.ratings                   <- reference.ratings
  37. disturbed.ratings[replacement.idx]  <- replaced.ratings
  38.  
  39. GAUROC(disturbed.ratings, reference.ratings)
  40.  
  41. cor(disturbed.ratings, reference.ratings)
  42.  
  43. ## Somers' D
  44.  
  45. # random
  46.  
  47. n         <- 10 # number of classes
  48.  
  49. PD        <- runif(n,0,40)
  50. companies <- ceiling(runif(n,1,100))
  51.  
  52. SomersD(companies,PD)
  53.  
  54. SomersD.alt(companies,PD) # verification using function from "measures2.R"
  55.  
  56. # portfolio from "A proposal for a validation methodology for the discriminatory power of a rating system over time", Oliver Blumke, The Journal of Risk Model Validation, Spring 2011
  57.  
  58. PD        <- c(0.05,0.10,0.50,1,2,5,25) # 0.05% (AAA) ... 25% (CCC)
  59. companies <- c(5,10,20,25,20,15,5)
  60.  
  61. SomersD(companies,PD)
  62.  
  63. SomersD.alt(companies,PD) # verification using function from "measures2.R"
  64.  
  65. ## Gini
  66.  
  67. # Gini for example from Somers' D
  68.  
  69. PD        <- c(0.05,0.10,0.50,1,2,5,25) # 0.05% (AAA) ... 25% (CCC)
  70. companies <- c(5,10,20,25,20,15,5)
  71.  
  72. defaults  <- PD/100*companies
  73.  
  74. ineq(defaults,type="Gini")
  75.  
  76. plot(Lc(defaults))
  77.  
  78. # Gini - random
  79.  
  80. k <- 7
  81.  
  82. defaults <- round(runif(k,0,100))
  83.  
  84. ineq(defaults,type="Gini")
  85.  
  86. plot(Lc(defaults))
  87.  
  88. ## Normal test
  89.  
  90. PD        <- 0.02
  91. defaulted <- c(1,2,3,2,3)
  92. companies <- rep(100,length(PD))
  93.  
  94. # reject? H0: realized Default Rate  <= q
  95. #         H1: realized Default Rate  > q
  96. NormalTest(PD,defaults,companies,alpha=0.01)
  97.  
  98. ## Hosmer-Lemeshow test
  99.  
  100. PD        <- c(0.01,0.05,0.10,0.17,0.25)
  101. # defaulted <- c(  10,  10,   5,   7,  5 )
  102. defaulted <- c(  0,6,12,15,30)
  103. companies <- rep(100,length(PD))
  104.  
  105. HosmerLemeshow(PD,defaulted,companies)
  106.  
  107. ## Spiegelhalter test
  108.  
  109. PD <- c(0.01,0.02,0.01,0.05,0.20,0.05,0.33,0.02,0.05,0.20)
  110. y  <- c(   0,   0,   0,   0,   0,   0,   1,   0,   0,   1)
  111. Spiegelhalter(y,PD)
  112.  
  113. ## Binomial test (for idependent defaults)
  114.  
  115. n <- 1000 # number of counterparties
  116. q <- 0.01 # PD
  117. d <- 19   # actual number of defaults
  118.  
  119. 1-pbinom(d-1,n,q) # probability of observing 'd' or more defaults
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!
 
Top