swoole_process 實現了多進程的管理,多個進程同時進行采集任務,
公司的框架比較low,用的tp框架,結合tp框架實現多進程的采集
這是swoole好的學習資源
https://segmentfault.com/a/1190000002946586
1 /** 2 * 用swoole多進程方法進行采集 3 * 采集網貸天眼下各個平台的數據,由於網貸天眼的平台的每個參數指標對應一個ajax請求, 4 * 所有設計每個參數指標對應一張表,這樣看起來雖然比較繁瑣,但有助於數據的更新和維護. 5 * 先循環遍歷出url表中所有的屬於網貸天眼的url,和cid,調用各個抓取參數方法,進行 6 * 入庫操作 7 */ 8 public function addDataSwoole() 9 { 10 11 for ($i = 0; $i < 14; $i++) { 12 13 $process = new \swoole_process([$this, 'swooleGetContents'], true); 14 $start = $process->start(); 15 $process->write($i); 16 $workers[$start] = $process; 17 } 18 19 //防止產生僵屍進程. 20 while (true) { 21 $res = \swoole_process::wait(); 22 23 if ($res) { 24 $pid = $res['pid']; 25 write_logs('exit '.$pid); 26 }else{ 27 break; 28 } 29 } 30 31 } 32 33 /** 34 * 這是swoole回調函數用的 35 * @param swoole_process $worker 36 */ 37 public function swooleGetContents(swoole_process $worker) 38 { 39 $i = $worker->read(); 40 41 $urls = $this->allUrls(); 42 43 switch ($i) { 44 case 0: 45 $method = 'add_new_collect_money'; 46 break; 47 case 1: 48 $method = 'add_pure_money'; 49 break; 50 case 2: 51 $method = 'add_rate'; 52 break; 53 case 3: 54 $method = 'add_remainder'; 55 break; 56 case 4: 57 $method = 'add_paid_invest_people'; 58 break; 59 case 5: 60 $method = 'add_plat_paid'; 61 break; 62 case 6: 63 $method = 'add_top_ten_borrower'; 64 break; 65 case 7: 66 $method = 'add_top_ten_invest'; 67 break; 68 case 8: 69 $method = 'add_full_scale_time'; 70 break; 71 case 9: 72 $method = 'add_average_borrowing_time'; 73 break; 74 case 10: 75 $method = 'add_borrowing_to_invest'; 76 break; 77 case 11: 78 $method = 'add_invest_vs'; 79 break; 80 case 12: 81 $method = 'add_invest_total_vs'; 82 break; 83 case 13: 84 $method = 'add_new_borrowing_distribution'; 85 break; 86 } 87 88 foreach ($urls as $v) { 89 $res = $this->$method($v['url'], $v['cid']); 90 } 91 92 var_dump('this process' . $method); 93 94 }