Teknik is a suite of services with attractive and functional interfaces. https://www.teknik.io/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

CliMulti.php 8.2KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281
  1. <?php
  2. /**
  3. * Piwik - free/libre analytics platform
  4. *
  5. * @link http://piwik.org
  6. * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
  7. */
  8. namespace Piwik;
  9. use Piwik\CliMulti\CliPhp;
  10. use Piwik\CliMulti\Output;
  11. use Piwik\CliMulti\Process;
  12. /**
  13. * Class CliMulti.
  14. */
  15. class CliMulti {
  16. /**
  17. * If set to true or false it will overwrite whether async is supported or not.
  18. *
  19. * @var null|bool
  20. */
  21. public $supportsAsync = null;
  22. /**
  23. * @var \Piwik\CliMulti\Process[]
  24. */
  25. private $processes = array();
  26. /**
  27. * If set it will issue at most concurrentProcessesLimit requests
  28. * @var int
  29. */
  30. private $concurrentProcessesLimit = null;
  31. /**
  32. * @var \Piwik\CliMulti\Output[]
  33. */
  34. private $outputs = array();
  35. private $acceptInvalidSSLCertificate = false;
  36. public function __construct()
  37. {
  38. $this->supportsAsync = $this->supportsAsync();
  39. }
  40. /**
  41. * It will request all given URLs in parallel (async) using the CLI and wait until all requests are finished.
  42. * If multi cli is not supported (eg windows) it will initiate an HTTP request instead (not async).
  43. *
  44. * @param string[] $piwikUrls An array of urls, for instance:
  45. * array('http://www.example.com/piwik?module=API...')
  46. * @return array The response of each URL in the same order as the URLs. The array can contain null values in case
  47. * there was a problem with a request, for instance if the process died unexpected.
  48. */
  49. public function request(array $piwikUrls)
  50. {
  51. $chunks = array($piwikUrls);
  52. if($this->concurrentProcessesLimit) {
  53. $chunks = array_chunk( $piwikUrls, $this->concurrentProcessesLimit);
  54. }
  55. $results = array();
  56. foreach($chunks as $urlsChunk) {
  57. $results = array_merge($results, $this->requestUrls($urlsChunk));
  58. }
  59. return $results;
  60. }
  61. /**
  62. * Ok, this sounds weird. Why should we care about ssl certificates when we are in CLI mode? It is needed for
  63. * our simple fallback mode for Windows where we initiate HTTP requests instead of CLI.
  64. * @param $acceptInvalidSSLCertificate
  65. */
  66. public function setAcceptInvalidSSLCertificate($acceptInvalidSSLCertificate)
  67. {
  68. $this->acceptInvalidSSLCertificate = $acceptInvalidSSLCertificate;
  69. }
  70. /**
  71. * @param $limit int Maximum count of requests to issue in parallel
  72. */
  73. public function setConcurrentProcessesLimit($limit)
  74. {
  75. $this->concurrentProcessesLimit = $limit;
  76. }
  77. private function start($piwikUrls)
  78. {
  79. foreach ($piwikUrls as $index => $url) {
  80. $cmdId = $this->generateCommandId($url) . $index;
  81. $output = new Output($cmdId);
  82. if ($this->supportsAsync) {
  83. $this->executeAsyncCli($url, $output, $cmdId);
  84. } else {
  85. $this->executeNotAsyncHttp($url, $output);
  86. }
  87. $this->outputs[] = $output;
  88. }
  89. }
  90. private function buildCommand($hostname, $query, $outputFile)
  91. {
  92. $bin = $this->findPhpBinary();
  93. return sprintf('%s %s/console climulti:request --piwik-domain=%s %s > %s 2>&1 &',
  94. $bin, PIWIK_INCLUDE_PATH, escapeshellarg($hostname), escapeshellarg($query), $outputFile);
  95. }
  96. private function getResponse()
  97. {
  98. $response = array();
  99. foreach ($this->outputs as $output) {
  100. $response[] = $output->get();
  101. }
  102. return $response;
  103. }
  104. private function hasFinished()
  105. {
  106. foreach ($this->processes as $index => $process) {
  107. $hasStarted = $process->hasStarted();
  108. if (!$hasStarted && 8 <= $process->getSecondsSinceCreation()) {
  109. // if process was created more than 8 seconds ago but still not started there must be something wrong.
  110. // ==> declare the process as finished
  111. $process->finishProcess();
  112. continue;
  113. } elseif (!$hasStarted) {
  114. return false;
  115. }
  116. if ($process->isRunning()) {
  117. return false;
  118. }
  119. if ($process->hasFinished()) {
  120. // prevent from checking this process over and over again
  121. unset($this->processes[$index]);
  122. }
  123. }
  124. return true;
  125. }
  126. private function generateCommandId($command)
  127. {
  128. return substr(Common::hash($command . microtime(true) . rand(0, 99999)), 0, 100);
  129. }
  130. /**
  131. * What is missing under windows? Detection whether a process is still running in Process::isProcessStillRunning
  132. * and how to send a process into background in start()
  133. */
  134. public function supportsAsync()
  135. {
  136. return Process::isSupported() && !Common::isPhpCgiType() && $this->findPhpBinary();
  137. }
  138. private function findPhpBinary()
  139. {
  140. $cliPhp = new CliPhp();
  141. return $cliPhp->findPhpBinary();
  142. }
  143. private function cleanup()
  144. {
  145. foreach ($this->processes as $pid) {
  146. $pid->finishProcess();
  147. }
  148. foreach ($this->outputs as $output) {
  149. $output->destroy();
  150. }
  151. $this->processes = array();
  152. $this->outputs = array();
  153. }
  154. /**
  155. * Remove files older than one week. They should be cleaned up automatically after each request but for whatever
  156. * reason there can be always some files left.
  157. */
  158. public static function cleanupNotRemovedFiles()
  159. {
  160. $timeOneWeekAgo = strtotime('-1 week');
  161. $files = _glob(self::getTmpPath() . '/*');
  162. if(empty($files)) {
  163. return;
  164. }
  165. foreach ($files as $file) {
  166. $timeLastModified = filemtime($file);
  167. if ($timeOneWeekAgo > $timeLastModified) {
  168. unlink($file);
  169. }
  170. }
  171. }
  172. public static function getTmpPath()
  173. {
  174. $dir = PIWIK_INCLUDE_PATH . '/tmp/climulti';
  175. return SettingsPiwik::rewriteTmpPathWithInstanceId($dir);
  176. }
  177. private function executeAsyncCli($url, Output $output, $cmdId)
  178. {
  179. $this->processes[] = new Process($cmdId);
  180. $url = $this->appendTestmodeParamToUrlIfNeeded($url);
  181. $query = UrlHelper::getQueryFromUrl($url, array('pid' => $cmdId));
  182. $hostname = UrlHelper::getHostFromUrl($url);
  183. $command = $this->buildCommand($hostname, $query, $output->getPathToFile());
  184. Log::debug($command);
  185. shell_exec($command);
  186. }
  187. private function executeNotAsyncHttp($url, Output $output)
  188. {
  189. try {
  190. Log::debug("Execute HTTP API request: " . $url);
  191. $response = Http::sendHttpRequestBy('curl', $url, $timeout = 0, $userAgent = null, $destinationPath = null, $file = null, $followDepth = 0, $acceptLanguage = false, $this->acceptInvalidSSLCertificate);
  192. $output->write($response);
  193. } catch (\Exception $e) {
  194. $message = "Got invalid response from API request: $url. ";
  195. if (isset($response) && empty($response)) {
  196. $message .= "The response was empty. This usually means a server error. This solution to this error is generally to increase the value of 'memory_limit' in your php.ini file. Please check your Web server Error Log file for more details.";
  197. } else {
  198. $message .= "Response was '" . $e->getMessage() . "'";
  199. }
  200. $output->write($message);
  201. Log::debug($e);
  202. }
  203. }
  204. private function appendTestmodeParamToUrlIfNeeded($url)
  205. {
  206. $isTestMode = $url && false !== strpos($url, 'tests/PHPUnit/proxy');
  207. if ($isTestMode && false === strpos($url, '?')) {
  208. $url .= "?testmode=1";
  209. } elseif ($isTestMode) {
  210. $url .= "&testmode=1";
  211. }
  212. return $url;
  213. }
  214. /**
  215. * @param array $piwikUrls
  216. * @return array
  217. */
  218. private function requestUrls(array $piwikUrls)
  219. {
  220. $this->start($piwikUrls);
  221. do {
  222. usleep(100000); // 100 * 1000 = 100ms
  223. } while (!$this->hasFinished());
  224. $results = $this->getResponse($piwikUrls);
  225. $this->cleanup();
  226. self::cleanupNotRemovedFiles();
  227. return $results;
  228. }
  229. }