dput(df[1:30,-3])
structure(list(origpat = c(4247592, 4247592, 4247592, 4247592,
4247592, 4247592, 4247592, 4247592, 4247592, 4247592, 4247592,
4247592, 4247592, 4247592, 4247592, 4247592, 4247592, 4247592,
4247592, 4247592, 4247592, 4247592, 4247592, 4247592, 4247592,
4247592, 4247592, 4247592, 4247592, 4247592), ref.pat = c(4318978,
4436368, 4358181, 4478622, 4312654, 4293439, 4286061, 4363648,
4406517, 4478623, 4277285, 4375743, 4470520, 4328022, 4248614,
4297139, 4296607, 4296608, 4395271, 4321141, 4294190, 4431420,
4322467, 4285730, 4393138, 4246034, 4251278, 4339174, 4277322,
4290586), mainprim = c("442", "442", "442", "442", "442", "442",
"442", "442", "442", "442", "442", "442", "442", "442", "442",
"442", "442", "442", "442", "442", "442", "442", "442", "442",
"442", "442", "442", "442", "442", "442")), .Names = c("origpat",
"ref.pat", "mainprim"), row.names = c(NA, 30L), class = c("data.table",
"data.frame"))

dput(tmp)
structure(list(pnum = c("4318978", "4318978", "4318978", "4318978",
"4318978", "4318978", "4318978", "4318978", "4436368", "4436368",
"4436368", "4436368", "4358181", "4358181", "4358181", "4358181",
"4478622", "4312654", "4312654", "4312654", "4312654", "4312654",
"4312654", "4293439", "4293439", "4293439", "4293439", "4293439",
"4293439", "4293439", "4293439", "4293439", "4293439", "4293439",
"4293439", "4293439", "4286061", "4286061", "4286061", "4286061",
"4286061", "4286061", "4286061", "4286061", "4363648", "4363648",
"4363648", "4406517", "4478623", "4478623", "4277285", "4375743",
"4375743", "4375743", "4375743", "4470520", "4470520", "4470520",
"4328022", "4328022", "4248614", "4248614", "4248614", "4248614",
"4248614", "4248614", "4297139", "4297139", "4297139", "4297139",
"4297139", "4296607", "4296607", "4296607", "4296607", "4296607",
"4296607", "4296608", "4296608", "4296608", "4296608", "4296608",
"4395271", "4395271", "4395271", "4321141", "4321141", "4321141",
"4321141", "4294190", "4294190", "4294190", "4294190", "4294190",
"4294190", "4431420", "4431420", "4431420", "4431420", "4431420",
"4431420", "4322467", "4322467", "4322467", "4322467", "4322467",
"4322467", "4322467", "4322467", "4322467", "4322467", "4285730",
"4285730", "4393138", "4393138", "4393138", "4393138", "4393138",
"4393138", "4393138", "4246034", "4246034", "4246034", "4246034",
"4251278", "4251278", "4251278", "4339174", "4339174", "4339174",
"4339174", "4277322", "4277322", "4290586", "4290586", "4290586",
"4290586", "4290586", "4247592", "4247592", "4247592", "4247592",
"4247592", "4247592", "4247592", "4247592", "4247592"), prim = c("430",
"430", "430", "430", "430", "430", "430", "430", "340", "385",
"385", "385", "385", "385", "65", "65", "65", "118", "427", "65",
"65", "65", "65", "106", "106", "106", "501", "501", "501", "501",
"501", "516", "516", "516", "516", "516", "435", "435", "435",
"435", "435", "435", "435", "435", "156", "428", "65", "385",
"65", "65", "501", "422", "53", "53", "53", "222", "422", "604",
"65", "65", "385", "385", "65", "65", "65", "65", "106", "106",
"501", "501", "501", "252", "423", "423", "501", "505", "62",
"423", "501", "501", "505", "62", "65", "65", "65", "210", "210",
"210", "435", "118", "118", "118", "118", "118", "118", "106",
"433", "433", "433", "433", "501", "156", "427", "427", "428",
"428", "428", "428", "428", "428", "428", "501", "501", "426",
"426", "426", "435", "435", "435", "435", "428", "501", "501",
"501", "501", "501", "65", "385", "385", "385", "65", "204",
"204", "204", "266", "266", "432", "73", "427", "427", "428",
"442", "442", "442", "442", "8", "8")), .Names = c("pnum", "prim"
), class = c("data.table", "data.frame"), row.names = c(NA, -147L
), .internal.selfref = <pointer: 0x0000000000100788>)

library(data.table)
df <- data.table(df) ; setkey(df, refpat, origpat)
refs <- unique(df$refpat) # Capture all unique refpat in df (71,000 in entire data.table)
startrow <- 0 # Set loop
overlap <- function(a,b) sum (a %chin% b) / length(b)
df$compare <- NA # overlap values will be inserted here

for (h in 1:length(refs)) {
  refclass <- tmp$prim[tmp$pnum == refs[h]] #subgroup of relevant 'prim'
    x <- length(df$refpat[df$refpat == refs[h]])
    prims <- df$mainprim[startrow:(startrow + x)] # isolate subset from large `df` data.table to reduce memory needed in second loop
      for (i in 1:x) {
      df$compare[startrow + i] <- overlap(prims[i], refclass)
        }
    startrow <- startrow + x
    print(h)
  }