a <- "{1:0987617820}{2:q312132498s7yd09f8sydf987s6df8797yds9f87098}
{3:{112:123123214321}}{4:20:asdasd3214213}"

## output should be
"0987617820" "q312132498s7yd09f8sydf987s6df8797yds9f87098", "{112:123123214321}" "20:asdasd3214213"

a = "{1:0987617820}{2:q312132498s7yd09f8sydf987s6df8797yds9f87098}{3:{112:123123214321}}{4:20:asdasd3214213}"

foohacky = function(str){
    #remove opening bracket
    pt1 = gsub('\{+[0-9]:', '@@',str)
    #remove a closing bracket that is preceded by any alphanumeric character
    pt2 = gsub('([0-9a-zA-Z])(\})', '\1',pt1, perl=TRUE)
    #split up and hack together the result
    pt3 = strsplit(pt2, "@@")[[1]][-1]
    pt3
}

> foohacky(a)
[1] "0987617820"
[2] "q312132498s7yd09f8sydf987s6df8797yds9f87098"
[3] "{112:123123214321}"
[4] "20:asdasd3214213"

> a = "{1:0987617820}{{3:{112:123123214321}}{4:{20:asdasd3214213}}"
> foohacky(a)
[1] "0987617820"         "{112:123123214321}" "{20:asdasd3214213}"

a <- "{1:0987617820}{2:q312132498s7yd09f8sydf987s6df8797yds9f87098}
{3:{112:123123214321}}{4:20:asdasd3214213}"

# split based on }{ allowing for newlines and spaces
out <- strsplit(a, "\}[[:space:]]*\{")
# Make a single vector
out <- unlist(out)
# Have an excess open bracket in first
out[1] <- substring(out[1], 2)
# Have an excess closing bracket in last
n <- length(out)
out[length(out)] <-  substring(out[n], 1, nchar(out[n])-1)
# Remove the number colon at the beginning of the string
answer <- gsub("^[0-9]*\:", "", out)

> answer
[1] "0987617820"
[2] "q312132498s7yd09f8sydf987s6df8797yds9f87098"
[3] "{112:123123214321}"
[4] "20:asdasd3214213"

regPattern <- gregexpr("(?<=\{[0-9]\:)(\{.*\}|.*?)(?=\})", a, perl=TRUE)
a_parse <- regmatches(a, regPattern)
a <- unlist(a_parse)