Guest User

PHP Multi Curl - solution to curl misbehaving

a guest
Jun 25th, 2012
168
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. $urls = array(
  2.     "http://www.google.com",
  3.     "http://www.yahoo.com",
  4. );
  5.  
  6. $mh = curl_multi_init();
  7. $activeDownloads = array();
  8.  
  9. foreach ($urls as $i => $url) {
  10.     $curlHandle = curl_init();
  11.     $key = (int) $curlHandle;
  12.  
  13.     curl_setopt($curlHandle, CURLOPT_FOLLOWLOCATION, true);
  14. //    curl_setopt($curlHandle, CURLOPT_RETURNTRANSFER, true);
  15.     curl_setopt($curlHandle, CURLOPT_HEADER, 0);
  16.     curl_setopt($curlHandle, CURLOPT_URL, $url);
  17.  
  18.     $destinationFilepath = '/tmp/download_' . $key;
  19.     $file = fopen($destinationFilepath, "w");
  20.     curl_setopt($curlHandle, CURLOPT_FILE, $file);
  21.  
  22.     curl_multi_add_handle($mh, $curlHandle);
  23.  
  24.     $activeDownloads[$key] = array(
  25.         'url' => $url,
  26.         'filepath' => $destinationFilepath,
  27.         'filehandle' => $file,
  28.     );
  29. }
  30.  
  31. while (true) {
  32.  
  33.     usleep(50000);
  34.  
  35.     curl_multi_exec($mh, $running);
  36.  
  37.     // Call select to see if anything is waiting for us
  38.     $select = curl_multi_select($mh);
  39.     if ($select < 1) {
  40. //        echo 'No download is ready yet' . PHP_EOL;
  41.         continue;
  42.     }
  43.  
  44.     // Since something's waiting, give curl a chance to process it
  45.     do {
  46.         $status = curl_multi_exec($mh, $running);
  47.         usleep(10000);
  48.     } while ($status == CURLM_CALL_MULTI_PERFORM || $running);
  49.  
  50.  
  51.     // Now grab the information about the completed requests
  52.     while ($info = curl_multi_info_read($mh)) {
  53.  
  54.         $ch = $info['handle'];
  55.         $key = (int) $ch;
  56.  
  57.         // This outputs all the content correctly when CURLOPT_RETURNTRANSFER is true and the CURLOPT_FILE flag is not set
  58. //        $content = curl_multi_getcontent($ch);
  59. //        var_dump($content);
  60. //        die;
  61.  
  62.         $download = $activeDownloads[$key];
  63.  
  64.         // This is needed!!! Without this the file_get_contents() below only reads the first 40960 characters!
  65.         fclose($download['filehandle']);
  66.  
  67.         $info = curl_getinfo($ch);
  68.  
  69.         curl_close($ch);
  70.  
  71. //        print_r($info); die;
  72.  
  73.         $content = file_get_contents($download['filepath']);
  74.  
  75.         // This output is cut off at 40960 characters if the original filehandle is not closed first, why?
  76.         echo $content . PHP_EOL;
  77.  
  78.         echo strlen($content) . PHP_EOL;
  79.  
  80.         die;
  81.     }
  82. }
RAW Paste Data