SHARE
TWEET

Untitled

a guest Sep 15th, 2019 90 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. local function reverse_resolve(ip, ns, retrans, timeout)
  2.     local r, err = resolver:new{
  3.         nameservers = ns,
  4.         retrans = retrans,  -- 5 retransmissions on receive timeout
  5.         timeout = timeout,  -- 1 sec
  6.     }
  7.     if not r then
  8.         ngx.log(ngx.ERR, "failed to instantiate the resolver: ".. err .. ", client_ip: " .. ip)
  9.         return
  10.     end
  11.     local answers, err = r:reverse_query(ip)
  12.     if not answers then
  13.         ngx.log(ngx.ERR, "failed to query the DNS server: " .. err .. ", client_ip: " .. ip)
  14.         return
  15.     end
  16.     if answers.errcode then
  17.         ngx.log(ngx.ERR, "server returned error code: " .. answers.errcode ..
  18.                 ": " .. answers.errstr .. ", client_ip: " .. ip)
  19.         return
  20.     end
  21.     if not answers[1] or not answers[1].ptrdname then
  22.         ngx.log(ngx.ERR, "ptrdname entry not exist, client_ip: " .. ip)
  23.         return
  24.     end
  25.     local ip_host = answers[1].ptrdname
  26.     ip_host = string.gsub(ip_host, "google.com", "googlebot.com") -- bring to single form
  27.     local suff = string.sub(ip_host, -13)
  28.     if suff == "googlebot.com" then
  29.         return true
  30.     end
  31.     return nil, "not googlebot"
  32. end
  33.  
  34. local function is_googlebot(ip, ns, retrans, timeout)
  35.     local res = string.find(string.lower(ngx.var.http_user_agent or ""), "googlebot")
  36.     if not res then
  37.         -- not found googlebot in user agent
  38.         return nil, "not googlebot"
  39.     end
  40.     return reverse_resolve(ip, ns, retrans, timeout)
  41. end
  42.  
  43. local function has_referer()
  44.     local referer = ngx.var.http_referer or ""
  45.     if referer == "" then
  46.         return
  47.     end
  48.     return referer
  49. end
  50.  
  51. local function diff_ms(start)
  52.     return math.ceil(ngx.now() - start)
  53. end
  54.  
  55. --local remote_addr = ngx.var.proxy_add_x_forwarded_for
  56. local remote_addr = ngx.var.http_cf_connecting_ip or "127.0.0.1"
  57. --local remote_addr = ngx.var.arg_ip or "127.0.0.1" --TEST
  58.  
  59. local diff = -1
  60. local wl, err = redis_get_wl(red, remote_addr)
  61. if not wl or wl == ngx.null then
  62.     -- redis error or not found in WL
  63.     local bl, err = redis_get_bl(red, remote_addr)
  64.     if not bl or bl == ngx.null then
  65.         --redis error or not found in BL
  66.     local start_at = ngx.now()
  67.         local ip2loc = ip2location.new(ip2loc_db, ip2location.IP2LOCATION_FILE_IO)
  68.  
  69.     local proxy, iso = is_proxy(ip2loc, remote_addr)
  70.         ip2loc:close()
  71.     local diff = diff_ms(start_at)
  72.     --local proxy = "1"
  73.         if not proxy or proxy == "DCH" then
  74.             -- seems to be not proxy or DCH(googlebot fall here)
  75.             local googlebot, err = is_googlebot(remote_addr, resolver_ns, resolver_retrans, resolver_timeout)
  76.  
  77.         if not googlebot and proxy == "DCH" then
  78.         redis_add_bl(red, remote_addr, bl_proxy_ttl)
  79.         redis_set_keepalive(red, redis_pool_size, redis_pool_timeout)
  80.         ngx.log(ngx.NOTICE, "BL: proxy: " .. proxy .. ", iso_code: " .. iso .. ", ttl: " .. bl_proxy_ttl .. ", client_ip: " .. remote_addr .. ", p.t: " .. diff)
  81.         return ngx.exit(ngx.HTTP_NOT_FOUND)
  82.         end
  83.  
  84.             if not googlebot then
  85.                 -- not googlebot
  86.                 local referer = has_referer()  
  87.                 if not referer then
  88.                     -- no referer 404 add to BL
  89.                     redis_add_bl(red, remote_addr, bl_ttl)
  90.                     redis_set_keepalive(red, redis_pool_size, redis_pool_timeout)
  91.                     ngx.log(ngx.NOTICE, "BL: no referer, ttl: " .. bl_ttl .. ", client_ip: " .. remote_addr  .. ", p.t: " .. diff)
  92.                     return ngx.exit(ngx.HTTP_NOT_FOUND)
  93.                 end
  94.  
  95.                 -- has referer add to WL
  96.                 redis_add_wl(red, remote_addr, wl_ttl)
  97.                 redis_set_keepalive(red, redis_pool_size, redis_pool_timeout)
  98.                 ngx.log(ngx.NOTICE, "WL: has referer, ttl: " .. wl_ttl .. ", client_ip: " .. remote_addr  .. ", p.t: " .. diff)
  99.                 return
  100.             end
  101.  
  102.             -- googlebot here add to WL
  103.             redis_add_wl(red, remote_addr, wl_googlebot_ttl)
  104.             redis_set_keepalive(red, redis_pool_size, redis_pool_timeout)
  105.             ngx.log(ngx.NOTICE, "WL: googlebot detected, ttl: " .. wl_googlebot_ttl .. ", client_ip: " .. remote_addr  .. ", p.t: " .. diff)
  106.             return
  107.         end
  108.  
  109.         -- seems to be proxy 404 add to BL
  110.         redis_add_bl(red, remote_addr, bl_proxy_ttl)
  111.         redis_set_keepalive(red, redis_pool_size, redis_pool_timeout)
  112.         ngx.log(ngx.NOTICE, "BL: proxy: " .. proxy .. ", iso_code: " .. iso .. ", ttl: " .. bl_proxy_ttl .. ", client_ip: " .. remote_addr  .. ", p.t: " .. diff)
  113.         return ngx.exit(ngx.HTTP_NOT_FOUND)
  114.     end
  115.  
  116.     -- exist in BL send 404
  117.     redis_set_keepalive(red, redis_pool_size, redis_pool_timeout)
  118.     ngx.log(ngx.NOTICE, "BL" .. ", client_ip: " .. remote_addr  .. ", p.t: " .. diff)
  119.     return ngx.exit(ngx.HTTP_NOT_FOUND)
  120. end
  121. ngx.log(ngx.NOTICE, "WL" .. ", client_ip: " .. remote_addr  .. ", p.t: " .. diff)
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!
 
Top