Scorecard validation tests

# visit: http://www.reakkt.com/2014/01/basics-of-scorecard-model-validation.html

### validation tests examples

library(ineq)

source("validation tests.R")

## GAUROC

n <- 100 # number of counterparties
reference.ratings <- ceiling(runif(n,1,21))
test.ratings      <- ceiling(runif(n,1,21))

# random test ratings

GAUROC(test.ratings, reference.ratings) # ~0.5

cor(test.ratings, reference.ratings)

#  test ratings identical with the referece

test.ratings <- reference.ratings

GAUROC(test.ratings, reference.ratings) # 1

cor(test.ratings, reference.ratings)

# some reference ratings replaced with random values

replacement.ratio <- 0.1
replacement.idx   <- sample(1:n,round(n*disturbance.ratio))

replaced.ratings  <- ceiling(runif(round(n*disturbance.ratio),1,21))

disturbed.ratings                   <- reference.ratings
disturbed.ratings[replacement.idx]  <- replaced.ratings

GAUROC(disturbed.ratings, reference.ratings)

cor(disturbed.ratings, reference.ratings)

## Somers' D

# random

n         <- 10 # number of classes

PD        <- runif(n,0,40)
companies <- ceiling(runif(n,1,100))

SomersD(companies,PD)

SomersD.alt(companies,PD) # verification using function from "measures2.R"

# portfolio from "A proposal for a validation methodology for the discriminatory power of a rating system over time", Oliver Blumke, The Journal of Risk Model Validation, Spring 2011

PD        <- c(0.05,0.10,0.50,1,2,5,25) # 0.05% (AAA) ... 25% (CCC)
companies <- c(5,10,20,25,20,15,5)

SomersD(companies,PD)

SomersD.alt(companies,PD) # verification using function from "measures2.R"

## Gini

# Gini for example from Somers' D

PD        <- c(0.05,0.10,0.50,1,2,5,25) # 0.05% (AAA) ... 25% (CCC)
companies <- c(5,10,20,25,20,15,5)

defaults  <- PD/100*companies

ineq(defaults,type="Gini")

plot(Lc(defaults))

# Gini - random

k <- 7

defaults <- round(runif(k,0,100))

ineq(defaults,type="Gini")

plot(Lc(defaults))

## Normal test

PD        <- 0.02
defaulted <- c(1,2,3,2,3)
companies <- rep(100,length(PD))

# reject? H0: realized Default Rate  <= q
#         H1: realized Default Rate  > q
NormalTest(PD,defaults,companies,alpha=0.01)

## Hosmer-Lemeshow test

PD        <- c(0.01,0.05,0.10,0.17,0.25)
# defaulted <- c(  10,  10,   5,   7,  5 )
defaulted <- c(  0,6,12,15,30)
companies <- rep(100,length(PD))

HosmerLemeshow(PD,defaulted,companies)

## Spiegelhalter test

PD <- c(0.01,0.02,0.01,0.05,0.20,0.05,0.33,0.02,0.05,0.20)
y  <- c(   0,   0,   0,   0,   0,   0,   1,   0,   0,   1)
Spiegelhalter(y,PD)

## Binomial test (for idependent defaults)

n <- 1000 # number of counterparties
q <- 0.01 # PD
d <- 19   # actual number of defaults

1-pbinom(d-1,n,q) # probability of observing 'd' or more defaults