retesere20

php-multi-curl-old

Jul 10th, 2021 (edited)
11
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. /* ######### RollingCurl_old : https://github.com/LionsAd/rolling-curl
  2. $this->urlsArray = [
  3. 'x'=>['url'=>'http://www.google.com', 'post'=>[] ],
  4. 'b'=>['url'=>'http://www.facebook.com', 'post'=>[] ],
  5. 'l'=>['url'=>'http://www.yahoo.com', 'post'=>[] ]
  6. ];
  7. public function request_callback($response, $info, $request) {
  8. $this->var_dump($response);
  9. $original_url =$request->url;
  10. $original_key =$request->headers['url_key'];
  11. $this->var_dump($info['url']);
  12. if ($info['http_code'] ==200)...
  13. echo '<hr>';
  14. }
  15. $helpers->multi_curl_rolling_old( $this->urlsArray, [$this,'request_callback']);
  16.  
  17. $info:
  18. 'url' => string 'http://www.google.com/' (length=22)
  19. 'content_type' => string 'text/html; charset=ISO-8859-1' (length=29)
  20. 'http_code' => int 200
  21. 'total_time' => float 0.212069
  22. 'redirect_url' => string '' (length=0)
  23. */
  24. public $multi_curl_rolling_old_arrs=[];
  25. public function multi_curl_rolling_old($urlsArray, $opts=[], $callback=null, $decode=true)
  26. {
  27. $uniqId = uniqid('k');
  28. $this->multi_curl_rolling_old_arrs[$uniqId]=[];
  29. $callback_final = function($response, $info, $request) use($callback, $uniqId, $decode){
  30. $original_url =$request->url;
  31. $original_key =$request->headers['url_key'];
  32. if ($info['http_code'] ==200){
  33. }
  34. try{ $this->multi_curl_rolling_old_arrs[$uniqId][$original_key]= $decode ? json_decode($response) : $response; }
  35. catch(\Exception $ex) { $this->multi_curl_rolling_old_arrs[$uniqId][$original_key]= $response; }
  36.  
  37. call_user_func( ($func = $callback==null ? function(){} : $callback), $response, $info, $request);
  38. };
  39. $rc = new \Puvox\RollingCurl_old($callback_final);
  40. // $rc->options = array(CURLOPT_HEADER => true, CURLOPT_NOBODY => true);
  41. $simultaneous_urls = empty($opts['simultaneous_urls']) ? 10 : $opts['simultaneous_urls'];
  42.  
  43. if (!empty($opts['headers'])) { $headers=[]; foreach($opts['headers'] as $key=>$value) { $headers[]="$key:$value"; } $rc->__set('headers',$headers ); }
  44.  
  45. $rc->window_size = $simultaneous_urls; // the window size determines how many simultaneous requests to allow.
  46. //if bare urls array or multi-dimensional
  47. if (is_string($urlsArray[ array_keys($urlsArray)[0] ])){
  48. foreach($urlsArray as $key=>$each){
  49. $final_array[$key]=['url'=>$each];
  50. }
  51. }
  52. else{
  53. $final_array =$urlsArray;
  54. }
  55. foreach ($final_array as $key=>$urlBlock) {
  56. $url = $urlBlock['url'];
  57. $postArray = empty($urlBlock['post'])? [] : $urlBlock['post'];
  58. $request = empty($postArray) ? new \Puvox\RollingCurlRequest_old($url) : new \Puvox\RollingCurlRequest_old($url, 'POST', $postArray);
  59. $request->headers['url_key']=$key;
  60. // $request->options = array(CURLOPT_HEADER => true, CURLOPT_NOBODY => true);
  61. $rc->add($request); // add each request to the RollingCurl object
  62. }
  63. $rc->execute();
  64. $ress = $this->multi_curl_rolling_old_arrs[$uniqId];
  65. unset($this->multi_curl_rolling_old_arrs[$uniqId]);
  66. return $ress;
  67. }
  68.  
  69.  
  70.  
  71.  
  72.  
  73.  
  74.  
  75.  
  76.  
  77.  
  78.  
  79.  
  80.  
  81.  
  82.  
  83.  
  84.  
  85.  
  86.  
  87.  
  88.  
  89.  
  90.  
  91.  
  92. // https://github.com/joshfraser/rolling-curl
  93. class RollingCurlRequest_old {
  94. public $url = false;
  95. public $method = 'GET';
  96. public $post_data = null;
  97. public $headers = null;
  98. public $options = null;
  99. function __construct($url, $method = "GET", $post_data = null, $headers = null, $options = null) {
  100. $this->url = $url;
  101. $this->method = $method;
  102. $this->post_data = $post_data;
  103. $this->headers = $headers;
  104. $this->options = $options;
  105. }
  106. public function __destruct() {
  107. unset($this->url, $this->method, $this->post_data, $this->headers, $this->options);
  108. }
  109. }
  110. class RollingCurlException_old extends \Exception {}
  111. class RollingCurl_old {
  112. private $window_size = 5;
  113. private $timeout = 10;
  114. private $callback;
  115. protected $options = array(CURLOPT_SSL_VERIFYPEER => 0, CURLOPT_SSL_VERIFYHOST => 0, CURLOPT_RETURNTRANSFER => 1, CURLOPT_CONNECTTIMEOUT => 30, CURLOPT_AUTOREFERER => true, CURLOPT_ENCODING => '', CURLOPT_TIMEOUT => 30);
  116. private $headers = array();
  117. private $requests = array();
  118. private $requestMap = array();
  119. function __construct($callback = null) {
  120. $this->callback = $callback;
  121. }
  122. public function __get($name) {
  123. return (isset($this->{
  124. $name
  125. })) ? $this->{
  126. $name
  127. }
  128. : null;
  129. }
  130. public function __set($name, $value) {
  131. if ($name == "options" || $name == "headers") {
  132. $this->{
  133. $name
  134. } = $value + $this->{
  135. $name
  136. };
  137. } else {
  138. $this->{
  139. $name
  140. } = $value;
  141. }
  142. return true;
  143. }
  144. public function add($request) {
  145. $this->requests[] = $request;
  146. return true;
  147. }
  148. public function request($url, $method = "GET", $post_data = null, $headers = null, $options = null) {
  149. $this->requests[] = new RollingCurlRequest_old($url, $method, $post_data, $headers, $options);
  150. return true;
  151. }
  152. public function get($url, $headers = null, $options = null) {
  153. return $this->request($url, "GET", null, $headers, $options);
  154. }
  155. public function post($url, $post_data = null, $headers = null, $options = null) {
  156. return $this->request($url, "POST", $post_data, $headers, $options);
  157. }
  158. public function IsNewPhp() {
  159. return PHP_MAJOR_VERSION >= 8;
  160. }
  161. public function HandleKey($curlHandle) {
  162. return $this->IsNewPhp() ? curl_getinfo($curlHandle, CURLINFO_PRIVATE) : (string)$curlHandle;
  163. }
  164. public function execute($window_size = null) {
  165. if (sizeof($this->requests) == 1) {
  166. return $this->single_curl();
  167. } else {
  168. return $this->rolling_curl($window_size);
  169. }
  170. }
  171. private function single_curl() {
  172. $ch = curl_init();
  173. $request = array_shift($this->requests);
  174. $options = $this->get_options($request);
  175. curl_setopt_array($ch, $options);
  176. $output = curl_exec($ch);
  177. $info = curl_getinfo($ch);
  178. if ($this->callback) {
  179. $callback = $this->callback;
  180. if (is_callable($this->callback)) {
  181. call_user_func($callback, $output, $info, $request);
  182. }
  183. } else
  184. return $output;
  185. return true;
  186. }
  187. private function rolling_curl($window_size = null) {
  188. if ($window_size)
  189. $this->window_size = $window_size;
  190. if (sizeof($this->requests) < $this->window_size)
  191. $this->window_size = sizeof($this->requests);
  192. if ($this->window_size < 2) {
  193. throw new RollingCurlException_old("Window size must be greater than 1");
  194. }
  195. $master = curl_multi_init();
  196. for ($i = 0; $i < $this->window_size; $i++) {
  197. $ch = curl_init();
  198. $options = $this->get_options($this->requests[$i]);
  199. if ($this->IsNewPhp())
  200. $options[CURLOPT_PRIVATE] = "req_$i";
  201. curl_setopt_array($ch, $options);
  202. curl_multi_add_handle($master, $ch);
  203. $key = $this->HandleKey($ch);
  204. $this->requestMap[$key] = $i;
  205. }
  206. do {
  207. while (($execrun = curl_multi_exec($master, $running)) == CURLM_CALL_MULTI_PERFORM);
  208. if ($execrun != CURLM_OK) {
  209. break;
  210. }
  211. while ($done = curl_multi_info_read($master)) {
  212. $info = curl_getinfo($done['handle']);
  213. $output = curl_multi_getcontent($done['handle']);
  214. $callback = $this->callback;
  215. if (is_callable($callback)) {
  216. $key = $this->HandleKey($done['handle']);
  217. $request = $this->requests[$this->requestMap[$key]];
  218. unset($this->requestMap[$key]);
  219. call_user_func($callback, $output, $info, $request);
  220. }
  221. if ($i < sizeof($this->requests) && isset($this->requests[$i]) && $i < count($this->requests)) {
  222. $ch = curl_init();
  223. $options = $this->get_options($this->requests[$i]);
  224. if ($this->IsNewPhp())
  225. $options[CURLOPT_PRIVATE] = "req_$i";
  226. curl_setopt_array($ch, $options);
  227. curl_multi_add_handle($master, $ch);
  228. $key = $this->HandleKey($ch);
  229. $this->requestMap[$key] = $i;
  230. $i++;
  231. }
  232. curl_multi_remove_handle($master, $done['handle']);
  233. }
  234. if ($running) {
  235. curl_multi_select($master, $this->timeout);
  236. }
  237. } while ($running);
  238. curl_multi_close($master);
  239. return true;
  240. }
  241. private function get_options($request) {
  242. $options = $this->__get('options');
  243. if ((ini_get('safe_mode') == 'Off' || !ini_get('safe_mode')) && ini_get('open_basedir') == '') {
  244. $options[CURLOPT_FOLLOWLOCATION] = 1;
  245. $options[CURLOPT_MAXREDIRS] = 5;
  246. }
  247. $headers = $this->__get('headers');
  248. if ($request->options) {
  249. $options = $request->options + $options;
  250. }
  251. $options[CURLOPT_URL] = $request->url;
  252. if ($request->post_data) {
  253. $options[CURLOPT_POST] = 1;
  254. $options[CURLOPT_POSTFIELDS] = $request->post_data;
  255. }
  256. if ($headers) {
  257. $options[CURLOPT_HEADER] = 0;
  258. $options[CURLOPT_HTTPHEADER] = $headers;
  259. }
  260. if (!empty($options[CURLOPT_WRITEFUNCTION])) {
  261. $writeCallback = $options[CURLOPT_WRITEFUNCTION];
  262. unset($options[CURLOPT_WRITEFUNCTION]);
  263. $options[CURLOPT_WRITEFUNCTION] = $writeCallback;
  264. }
  265. return $options;
  266. }
  267. public function __destruct() {
  268. unset($this->window_size, $this->callback, $this->options, $this->headers, $this->requests);
  269. }
  270. }
RAW Paste Data