jmunsch

urllib2 jnumb python example

Mar 11th, 2014
120
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 1.24 KB | None | 0 0
  1.     def download_item_info():
  2.  
  3.  
  4.         errors = open('download_error_log','a+')
  5.         with open('item_number_list_3-5-14.txt','r') as item_list:
  6.             # make the web page directory if it doesn't exist
  7.             path = 'XXXXXXXX_pages'
  8.             try:
  9.                 os.mkdir(path)
  10.             except Exception,e:print e        
  11.  
  12.             for item_number in item_list:
  13.                 # format item_number strip \r and \n
  14.                 item_number = item_number.rstrip("\r\n")
  15.                 # create filepath
  16.                 fp = os.path.join(path,item_number)
  17.                 print "### Current:"+fp
  18.                
  19.                 # Check if file exists if it does skip it? Maybe some diff checking is needed?
  20.                 if os.path.isfile(fp):
  21.                     print "Skipped:"+fp
  22.                     continue
  23.                
  24.                 # try to open the page with urllib2 and read the page
  25.                 try:
  26.                     f = urllib2.urlopen('http://www.XXXXXXXXX.com/product/?&familyid='+j)
  27.                     # Read the entire page and store into data
  28.                     data = f.read()
  29.                 except Exception,e:
  30.                     error = "Error opening/reading page:\n"+str(e)
  31.                     print error
  32.                     errors.write(error+"\n"+str(fp))
  33.                
  34.                 try:
  35.                    with open(fp,'w+') as jpage:
  36.                             jpage.write(data)
  37.                    print "Got page for:"+str(j)              
  38.                 except Exception,e:
  39.                     print e
  40.                     errors.write(str(e)+"\n"+str(fp))
  41.                 time.sleep(4)
  42.         errors.close()
Advertisement
Add Comment
Please, Sign In to add comment