Teknik is a suite of services with attractive and functional interfaces. https://www.teknik.io/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

CronArchive.php 49KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360
  1. <?php
  2. /**
  3. * Piwik - free/libre analytics platform
  4. *
  5. * @link http://piwik.org
  6. * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
  7. *
  8. */
  9. namespace Piwik;
  10. use Exception;
  11. use Piwik\ArchiveProcessor\Rules;
  12. use Piwik\CronArchive\FixedSiteIds;
  13. use Piwik\CronArchive\SharedSiteIds;
  14. use Piwik\Period\Factory as PeriodFactory;
  15. use Piwik\Plugins\CoreAdminHome\API as APICoreAdminHome;
  16. use Piwik\Plugins\SitesManager\API as APISitesManager;
  17. /**
  18. * ./console core:archive runs as a cron and is a useful tool for general maintenance,
  19. * and pre-process reports for a Fast dashboard rendering.
  20. */
  21. class CronArchive
  22. {
  23. // the url can be set here before the init, and it will be used instead of --url=
  24. public static $url = false;
  25. // Max parallel requests for a same site's segments
  26. const MAX_CONCURRENT_API_REQUESTS = 3;
  27. // force-timeout-for-periods default (1 hour)
  28. const SECONDS_DELAY_BETWEEN_PERIOD_ARCHIVES = 3600;
  29. // force-all-periods default (7 days)
  30. const ARCHIVE_SITES_WITH_TRAFFIC_SINCE = 604800;
  31. // By default, will process last 52 days and months
  32. // It will be overwritten by the number of days since last archiving ran until completion.
  33. const DEFAULT_DATE_LAST = 52;
  34. // Since weeks are not used in yearly archives, we make sure that all possible weeks are processed
  35. const DEFAULT_DATE_LAST_WEEKS = 260;
  36. const DEFAULT_DATE_LAST_YEARS = 7;
  37. // Flag to know when the archive cron is calling the API
  38. const APPEND_TO_API_REQUEST = '&trigger=archivephp';
  39. // Flag used to record timestamp in Option::
  40. const OPTION_ARCHIVING_FINISHED_TS = "LastCompletedFullArchiving";
  41. // Name of option used to store starting timestamp
  42. const OPTION_ARCHIVING_STARTED_TS = "LastFullArchivingStartTime";
  43. // Show only first N characters from Piwik API output in case of errors
  44. const TRUNCATE_ERROR_MESSAGE_SUMMARY = 6000;
  45. // archiving will be triggered on all websites with traffic in the last $shouldArchiveOnlySitesWithTrafficSince seconds
  46. private $shouldArchiveOnlySitesWithTrafficSince;
  47. // By default, we only process the current week/month/year at most once an hour
  48. private $processPeriodsMaximumEverySeconds;
  49. private $todayArchiveTimeToLive;
  50. private $websiteDayHasFinishedSinceLastRun = array();
  51. private $idSitesInvalidatedOldReports = array();
  52. private $shouldArchiveOnlySpecificPeriods = array();
  53. /**
  54. * @var SharedSiteIds|FixedSiteIds
  55. */
  56. private $websites = array();
  57. private $allWebsites = array();
  58. private $segments = array();
  59. private $piwikUrl = false;
  60. private $token_auth = false;
  61. private $visitsToday = 0;
  62. private $requests = 0;
  63. private $output = '';
  64. private $archiveAndRespectTTL = true;
  65. private $lastSuccessRunTimestamp = false;
  66. private $errors = array();
  67. private $isCoreInited = false;
  68. const NO_ERROR = "no error";
  69. public $testmode = false;
  70. /**
  71. * The list of IDs for sites for whom archiving should be initiated. If supplied, only these
  72. * sites will be archived.
  73. *
  74. * @var int[]
  75. */
  76. public $shouldArchiveSpecifiedSites = array();
  77. /**
  78. * The list of IDs of sites to ignore when launching archiving. Archiving will not be launched
  79. * for any site whose ID is in this list (even if the ID is supplied in {@link $shouldArchiveSpecifiedSites}
  80. * or if {@link $shouldArchiveAllSites} is true).
  81. *
  82. * @var int[]
  83. */
  84. public $shouldSkipSpecifiedSites = array();
  85. /**
  86. * If true, archiving will be launched for every site.
  87. *
  88. * @var bool
  89. */
  90. public $shouldArchiveAllSites = false;
  91. /**
  92. * If true, xhprof will be initiated for the archiving run. Only for development/testing.
  93. *
  94. * @var bool
  95. */
  96. public $shouldStartProfiler = false;
  97. /**
  98. * If HTTP requests are used to initiate archiving, this controls whether invalid SSL certificates should
  99. * be accepted or not by each request.
  100. *
  101. * @var bool
  102. */
  103. public $acceptInvalidSSLCertificate = false;
  104. /**
  105. * If set to true, scheduled tasks will not be run.
  106. *
  107. * @var bool
  108. */
  109. public $disableScheduledTasks = false;
  110. /**
  111. * The amount of seconds between non-day period archiving. That is, if archiving has been launched within
  112. * the past [$forceTimeoutPeriod] seconds, Piwik will not initiate archiving for week, month and year periods.
  113. *
  114. * @var int|false
  115. */
  116. public $forceTimeoutPeriod = false;
  117. /**
  118. * If supplied, archiving will be launched for sites that have had visits within the last [$shouldArchiveAllPeriodsSince]
  119. * seconds. If set to `true`, the value defaults to {@link ARCHIVE_SITES_WITH_TRAFFIC_SINCE}.
  120. *
  121. * @var int|bool
  122. */
  123. public $shouldArchiveAllPeriodsSince = false;
  124. /**
  125. * If supplied, archiving will be launched only for periods that fall within this date range. For example,
  126. * `"2012-01-01,2012-03-15"` would result in January 2012, February 2012 being archived but not April 2012.
  127. *
  128. * @var string|false eg, `"2012-01-01,2012-03-15"`
  129. */
  130. public $restrictToDateRange = false;
  131. /**
  132. * A list of periods to launch archiving for. By default, day, week, month and year periods
  133. * are considered. This variable can limit the periods to, for example, week & month only.
  134. *
  135. * @var string[] eg, `array("day","week","month","year")`
  136. */
  137. public $restrictToPeriods = array();
  138. /**
  139. * Forces CronArchive to retrieve data for the last [$dateLastForced] periods when initiating archiving.
  140. * When archiving weeks, for example, if 10 is supplied, the API will be called w/ last10. This will potentially
  141. * initiate archiving for the last 10 weeks.
  142. *
  143. * @var int|false
  144. */
  145. public $dateLastForced = false;
  146. /**
  147. * The number of concurrent requests to issue per website. Defaults to {@link MAX_CONCURRENT_API_REQUESTS}.
  148. *
  149. * Used when archiving a site's segments concurrently.
  150. *
  151. * @var int|false
  152. */
  153. public $concurrentRequestsPerWebsite = false;
  154. /**
  155. * Returns the option name of the option that stores the time core:archive was last executed.
  156. *
  157. * @param int $idSite
  158. * @param string $period
  159. * @return string
  160. */
  161. public static function lastRunKey($idSite, $period)
  162. {
  163. return "lastRunArchive" . $period . "_" . $idSite;
  164. }
  165. /**
  166. * Constructor.
  167. *
  168. * @param string|false $piwikUrl The URL to the Piwik installation to initiate archiving for. If `false`,
  169. * we determine it using the current request information.
  170. *
  171. * If invoked via the command line, $piwikUrl cannot be false.
  172. */
  173. public function __construct($piwikUrl = false)
  174. {
  175. $this->initLog();
  176. $this->initPiwikHost($piwikUrl);
  177. }
  178. /**
  179. * Initializes and runs the cron archiver.
  180. */
  181. public function main()
  182. {
  183. $this->init();
  184. $this->run();
  185. $this->runScheduledTasks();
  186. $this->end();
  187. }
  188. public function init()
  189. {
  190. // Note: the order of methods call matters here.
  191. $this->initCore();
  192. $this->initTokenAuth();
  193. $this->initCheckCli();
  194. $this->initStateFromParameters();
  195. Piwik::setUserHasSuperUserAccess(true);
  196. $this->logInitInfo();
  197. $this->checkPiwikUrlIsValid();
  198. $this->logArchiveTimeoutInfo();
  199. // record archiving start time
  200. Option::set(self::OPTION_ARCHIVING_STARTED_TS, time());
  201. $this->segments = $this->initSegmentsToArchive();
  202. $this->allWebsites = APISitesManager::getInstance()->getAllSitesId();
  203. if(!empty($this->shouldArchiveOnlySpecificPeriods)) {
  204. $this->log("- Will process the following periods: " . implode(", ", $this->shouldArchiveOnlySpecificPeriods) . " (--force-periods)");
  205. }
  206. $websitesIds = $this->initWebsiteIds();
  207. $this->filterWebsiteIds($websitesIds);
  208. if (!empty($this->shouldArchiveSpecifiedSites)
  209. || !empty($this->shouldArchiveAllSites)
  210. || !SharedSiteIds::isSupported()) {
  211. $this->websites = new FixedSiteIds($websitesIds);
  212. } else {
  213. $this->websites = new SharedSiteIds($websitesIds);
  214. if ($this->websites->getInitialSiteIds() != $websitesIds) {
  215. $this->log('Will ignore websites and help finish a previous started queue instead. IDs: ' . implode(', ', $this->websites->getInitialSiteIds()));
  216. }
  217. }
  218. if ($this->shouldStartProfiler) {
  219. \Piwik\Profiler::setupProfilerXHProf($mainRun = true);
  220. $this->log("XHProf profiling is enabled.");
  221. }
  222. /**
  223. * This event is triggered after a CronArchive instance is initialized.
  224. *
  225. * @param array $websiteIds The list of website IDs this CronArchive instance is processing.
  226. * This will be the entire list of IDs regardless of whether some have
  227. * already been processed.
  228. */
  229. Piwik::postEvent('CronArchive.init.finish', array($this->websites->getInitialSiteIds()));
  230. }
  231. public function runScheduledTasksInTrackerMode()
  232. {
  233. $this->initCore();
  234. $this->initTokenAuth();
  235. $this->logInitInfo();
  236. $this->checkPiwikUrlIsValid();
  237. $this->runScheduledTasks();
  238. }
  239. private $websitesWithVisitsSinceLastRun = 0;
  240. private $skippedPeriodsArchivesWebsite = 0;
  241. private $skippedDayArchivesWebsites = 0;
  242. private $skipped = 0;
  243. private $processed = 0;
  244. private $archivedPeriodsArchivesWebsite = 0;
  245. /**
  246. * Main function, runs archiving on all websites with new activity
  247. */
  248. public function run()
  249. {
  250. $timer = new Timer;
  251. $this->logSection("START");
  252. $this->log("Starting Piwik reports archiving...");
  253. do {
  254. $idSite = $this->websites->getNextSiteId();
  255. if (null === $idSite) {
  256. break;
  257. }
  258. flush();
  259. $requestsBefore = $this->requests;
  260. if ($idSite <= 0) {
  261. continue;
  262. }
  263. $skipWebsiteForced = in_array($idSite, $this->shouldSkipSpecifiedSites);
  264. if($skipWebsiteForced) {
  265. $this->log("Skipped website id $idSite, found in --skip-idsites ");
  266. $this->skipped++;
  267. continue;
  268. }
  269. /**
  270. * This event is triggered before the cron archiving process starts archiving data for a single
  271. * site.
  272. *
  273. * @param int $idSite The ID of the site we're archiving data for.
  274. */
  275. Piwik::postEvent('CronArchive.archiveSingleSite.start', array($idSite));
  276. $completed = $this->archiveSingleSite($idSite, $requestsBefore);
  277. /**
  278. * This event is triggered immediately after the cron archiving process starts archiving data for a single
  279. * site.
  280. *
  281. * @param int $idSite The ID of the site we're archiving data for.
  282. */
  283. Piwik::postEvent('CronArchive.archiveSingleSite.finish', array($idSite, $completed));
  284. } while (!empty($idSite));
  285. $this->log("Done archiving!");
  286. $this->logSection("SUMMARY");
  287. $this->log("Total visits for today across archived websites: " . $this->visitsToday);
  288. $totalWebsites = count($this->allWebsites);
  289. $this->skipped = $totalWebsites - $this->websitesWithVisitsSinceLastRun;
  290. $this->log("Archived today's reports for {$this->websitesWithVisitsSinceLastRun} websites");
  291. $this->log("Archived week/month/year for {$this->archivedPeriodsArchivesWebsite} websites");
  292. $this->log("Skipped {$this->skipped} websites: no new visit since the last script execution");
  293. $this->log("Skipped {$this->skippedDayArchivesWebsites} websites day archiving: existing daily reports are less than {$this->todayArchiveTimeToLive} seconds old");
  294. $this->log("Skipped {$this->skippedPeriodsArchivesWebsite} websites week/month/year archiving: existing periods reports are less than {$this->processPeriodsMaximumEverySeconds} seconds old");
  295. $this->log("Total API requests: {$this->requests}");
  296. //DONE: done/total, visits, wtoday, wperiods, reqs, time, errors[count]: first eg.
  297. $percent = $this->websites->getNumSites() == 0
  298. ? ""
  299. : " " . round($this->processed * 100 / $this->websites->getNumSites(), 0) . "%";
  300. $this->log("done: " .
  301. $this->processed . "/" . $this->websites->getNumSites() . "" . $percent . ", " .
  302. $this->visitsToday . " vtoday, $this->websitesWithVisitsSinceLastRun wtoday, {$this->archivedPeriodsArchivesWebsite} wperiods, " .
  303. $this->requests . " req, " . round($timer->getTimeMs()) . " ms, " .
  304. (empty($this->errors)
  305. ? self::NO_ERROR
  306. : (count($this->errors) . " errors."))
  307. );
  308. $this->log($timer->__toString());
  309. }
  310. /**
  311. * End of the script
  312. */
  313. public function end()
  314. {
  315. if (empty($this->errors)) {
  316. // No error -> Logs the successful script execution until completion
  317. Option::set(self::OPTION_ARCHIVING_FINISHED_TS, time());
  318. return;
  319. }
  320. $this->logSection("SUMMARY OF ERRORS");
  321. foreach ($this->errors as $error) {
  322. // do not logError since errors are already in stderr
  323. $this->log("Error: " . $error);
  324. }
  325. $summary = count($this->errors) . " total errors during this script execution, please investigate and try and fix these errors.";
  326. $this->logFatalError($summary);
  327. }
  328. public function logFatalError($m)
  329. {
  330. $this->logError($m);
  331. exit(1);
  332. }
  333. public function runScheduledTasks()
  334. {
  335. $this->logSection("SCHEDULED TASKS");
  336. if ($this->disableScheduledTasks) {
  337. $this->log("Scheduled tasks are disabled with --disable-scheduled-tasks");
  338. return;
  339. }
  340. $this->log("Starting Scheduled tasks... ");
  341. $tasksOutput = $this->request("?module=API&method=CoreAdminHome.runScheduledTasks&format=csv&convertToUnicode=0&token_auth=" . $this->token_auth);
  342. if ($tasksOutput == \Piwik\DataTable\Renderer\Csv::NO_DATA_AVAILABLE) {
  343. $tasksOutput = " No task to run";
  344. }
  345. $this->log($tasksOutput);
  346. $this->log("done");
  347. $this->logSection("");
  348. }
  349. private function archiveSingleSite($idSite, $requestsBefore)
  350. {
  351. $timerWebsite = new Timer;
  352. $lastTimestampWebsiteProcessedPeriods = $lastTimestampWebsiteProcessedDay = false;
  353. if ($this->archiveAndRespectTTL) {
  354. Option::clearCachedOption($this->lastRunKey($idSite, "periods"));
  355. $lastTimestampWebsiteProcessedPeriods = Option::get($this->lastRunKey($idSite, "periods"));
  356. Option::clearCachedOption($this->lastRunKey($idSite, "day"));
  357. $lastTimestampWebsiteProcessedDay = Option::get($this->lastRunKey($idSite, "day"));
  358. }
  359. $this->updateIdSitesInvalidatedOldReports();
  360. // For period other than days, we only re-process the reports at most
  361. // 1) every $processPeriodsMaximumEverySeconds
  362. $secondsSinceLastExecution = time() - $lastTimestampWebsiteProcessedPeriods;
  363. // if timeout is more than 10 min, we account for a 5 min processing time, and allow trigger 1 min earlier
  364. if ($this->processPeriodsMaximumEverySeconds > 10 * 60) {
  365. $secondsSinceLastExecution += 5 * 60;
  366. }
  367. $shouldArchivePeriods = $secondsSinceLastExecution > $this->processPeriodsMaximumEverySeconds;
  368. if (empty($lastTimestampWebsiteProcessedPeriods)) {
  369. // 2) OR always if script never executed for this website before
  370. $shouldArchivePeriods = true;
  371. }
  372. // (*) If the website is archived because it is a new day in its timezone
  373. // We make sure all periods are archived, even if there is 0 visit today
  374. $dayHasEndedMustReprocess = in_array($idSite, $this->websiteDayHasFinishedSinceLastRun);
  375. if ($dayHasEndedMustReprocess) {
  376. $shouldArchivePeriods = true;
  377. }
  378. // (*) If there was some old reports invalidated for this website
  379. // we make sure all these old reports are triggered at least once
  380. $websiteIsOldDataInvalidate = $this->isOldReportInvalidatedForWebsite($idSite);
  381. if ($websiteIsOldDataInvalidate) {
  382. $shouldArchivePeriods = true;
  383. }
  384. $websiteIdIsForced = in_array($idSite, $this->shouldArchiveSpecifiedSites);
  385. if($websiteIdIsForced) {
  386. $shouldArchivePeriods = true;
  387. }
  388. // Test if we should process this website at all
  389. $elapsedSinceLastArchiving = time() - $lastTimestampWebsiteProcessedDay;
  390. // Skip this day archive if last archive was older than TTL
  391. $existingArchiveIsValid = ($elapsedSinceLastArchiving < $this->todayArchiveTimeToLive);
  392. $skipDayArchive = $existingArchiveIsValid;
  393. // Invalidate old website forces the archiving for this site
  394. $skipDayArchive = $skipDayArchive && !$websiteIsOldDataInvalidate;
  395. // Also reprocess when day has ended since last run
  396. if ($dayHasEndedMustReprocess
  397. // it might have reprocessed for that day by another cron
  398. && !$this->hasBeenProcessedSinceMidnight($idSite, $lastTimestampWebsiteProcessedDay)
  399. && !$existingArchiveIsValid) {
  400. $skipDayArchive = false;
  401. }
  402. if ($websiteIdIsForced) {
  403. $skipDayArchive = false;
  404. }
  405. if ($skipDayArchive) {
  406. $this->log("Skipped website id $idSite, already done "
  407. . \Piwik\MetricsFormatter::getPrettyTimeFromSeconds($elapsedSinceLastArchiving, true, $isHtml = false)
  408. . " ago, " . $timerWebsite->__toString());
  409. $this->skippedDayArchivesWebsites++;
  410. $this->skipped++;
  411. return false;
  412. }
  413. $shouldProceed = $this->processArchiveDays($idSite, $lastTimestampWebsiteProcessedDay, $shouldArchivePeriods, $timerWebsite);
  414. if(!$shouldProceed) {
  415. return false;
  416. }
  417. if (!$shouldArchivePeriods) {
  418. $this->log("Skipped website id $idSite periods processing, already done "
  419. . \Piwik\MetricsFormatter::getPrettyTimeFromSeconds($elapsedSinceLastArchiving, true, $isHtml = false)
  420. . " ago, " . $timerWebsite->__toString());
  421. $this->skippedDayArchivesWebsites++;
  422. $this->skipped++;
  423. return false;
  424. }
  425. $success = true;
  426. foreach (array('week', 'month', 'year') as $period) {
  427. if(!$this->shouldProcessPeriod($period)) {
  428. // if any period was skipped, we do not mark the Periods archiving as successful
  429. $success = false;
  430. continue;
  431. }
  432. $success = $this->archiveVisitsAndSegments($idSite, $period, $lastTimestampWebsiteProcessedPeriods)
  433. && $success;
  434. }
  435. // Record succesful run of this website's periods archiving
  436. if ($success) {
  437. Option::set($this->lastRunKey($idSite, "periods"), time());
  438. }
  439. $this->archivedPeriodsArchivesWebsite++;
  440. $requestsWebsite = $this->requests - $requestsBefore;
  441. Log::info("Archived website id = $idSite, "
  442. . $requestsWebsite . " API requests, "
  443. . $timerWebsite->__toString()
  444. . " [" . $this->websites->getNumProcessedWebsites() . "/"
  445. . $this->websites->getNumSites()
  446. . " done]");
  447. return true;
  448. }
  449. /**
  450. * Checks the config file is found.
  451. *
  452. * @param $piwikUrl
  453. * @throws Exception
  454. */
  455. protected function initConfigObject($piwikUrl)
  456. {
  457. // HOST is required for the Config object
  458. $parsed = parse_url($piwikUrl);
  459. Url::setHost($parsed['host']);
  460. Config::getInstance()->clear();
  461. try {
  462. Config::getInstance()->checkLocalConfigFound();
  463. } catch (Exception $e) {
  464. throw new Exception("The configuration file for Piwik could not be found. " .
  465. "Please check that config/config.ini.php is readable by the user " .
  466. get_current_user());
  467. }
  468. }
  469. /**
  470. * Returns base URL to process reports for the $idSite on a given $period
  471. */
  472. private function getVisitsRequestUrl($idSite, $period, $date)
  473. {
  474. return "?module=API&method=API.get&idSite=$idSite&period=$period&date=" . $date . "&format=php&token_auth=" . $this->token_auth;
  475. }
  476. private function initSegmentsToArchive()
  477. {
  478. $segments = \Piwik\SettingsPiwik::getKnownSegmentsToArchive();
  479. if (empty($segments)) {
  480. return array();
  481. }
  482. $this->log("- Will pre-process " . count($segments) . " Segments for each website and each period: " . implode(", ", $segments));
  483. return $segments;
  484. }
  485. /**
  486. * @param $idSite
  487. * @param $lastTimestampWebsiteProcessedDay
  488. * @param $shouldArchivePeriods
  489. * @param $timerWebsite
  490. * @return bool
  491. */
  492. protected function processArchiveDays($idSite, $lastTimestampWebsiteProcessedDay, $shouldArchivePeriods, Timer $timerWebsite)
  493. {
  494. if (!$this->shouldProcessPeriod("day")) {
  495. // skip day archiving and proceed to period processing
  496. return true;
  497. }
  498. // Fake that the request is already done, so that other core:archive commands
  499. // running do not grab the same website from the queue
  500. Option::set($this->lastRunKey($idSite, "day"), time());
  501. // Remove this website from the list of websites to be invalidated
  502. // since it's now just about to being re-processed, makes sure another running cron archiving process
  503. // does not archive the same idSite
  504. if ($this->isOldReportInvalidatedForWebsite($idSite)) {
  505. $this->removeWebsiteFromInvalidatedWebsites($idSite);
  506. }
  507. // when some data was purged from this website
  508. // we make sure we query all previous days/weeks/months
  509. $processDaysSince = $lastTimestampWebsiteProcessedDay;
  510. if($this->isOldReportInvalidatedForWebsite($idSite)
  511. // when --force-all-websites option,
  512. // also forces to archive last52 days to be safe
  513. || $this->shouldArchiveAllSites) {
  514. $processDaysSince = false;
  515. }
  516. $date = $this->getApiDateParameter($idSite, "day", $processDaysSince);
  517. $url = $this->getVisitsRequestUrl($idSite, "day", $date);
  518. $content = $this->request($url);
  519. $daysResponse = @unserialize($content);
  520. if (empty($content)
  521. || !is_array($daysResponse)
  522. || count($daysResponse) == 0
  523. ) {
  524. // cancel the succesful run flag
  525. Option::set($this->lastRunKey($idSite, "day"), 0);
  526. $this->logError("Empty or invalid response '$content' for website id $idSite, " . $timerWebsite->__toString() . ", skipping");
  527. $this->skipped++;
  528. return false;
  529. }
  530. $visitsToday = $this->getVisitsLastPeriodFromApiResponse($daysResponse);
  531. $visitsLastDays = $this->getVisitsFromApiResponse($daysResponse);
  532. $this->requests++;
  533. $this->processed++;
  534. // If there is no visit today and we don't need to process this website, we can skip remaining archives
  535. if ($visitsToday == 0
  536. && !$shouldArchivePeriods
  537. ) {
  538. $this->log("Skipped website id $idSite, no visit today, " . $timerWebsite->__toString());
  539. $this->skipped++;
  540. return false;
  541. }
  542. if ($visitsLastDays == 0
  543. && !$shouldArchivePeriods
  544. && $this->shouldArchiveAllSites
  545. ) {
  546. $this->log("Skipped website id $idSite, no visits in the last " . $date . " days, " . $timerWebsite->__toString());
  547. $this->skipped++;
  548. return false;
  549. }
  550. $this->visitsToday += $visitsToday;
  551. $this->websitesWithVisitsSinceLastRun++;
  552. $this->archiveVisitsAndSegments($idSite, "day", $processDaysSince);
  553. $this->logArchivedWebsite($idSite, "day", $date, $visitsLastDays, $visitsToday, $timerWebsite);
  554. return true;
  555. }
  556. private function getSegmentsForSite($idSite)
  557. {
  558. $segmentsAllSites = $this->segments;
  559. $segmentsThisSite = \Piwik\SettingsPiwik::getKnownSegmentsToArchiveForSite($idSite);
  560. if (!empty($segmentsThisSite)) {
  561. $this->log("Will pre-process the following " . count($segmentsThisSite) . " Segments for this website (id = $idSite): " . implode(", ", $segmentsThisSite));
  562. }
  563. $segments = array_unique(array_merge($segmentsAllSites, $segmentsThisSite));
  564. return $segments;
  565. }
  566. /**
  567. * Will trigger API requests for the specified Website $idSite,
  568. * for the specified $period, for all segments that are pre-processed for this website.
  569. * Requests are triggered using cURL multi handle
  570. *
  571. * @param $idSite int
  572. * @param $period
  573. * @param $lastTimestampWebsiteProcessed
  574. * @return bool True on success, false if some request failed
  575. */
  576. private function archiveVisitsAndSegments($idSite, $period, $lastTimestampWebsiteProcessed)
  577. {
  578. $timer = new Timer();
  579. $url = $this->piwikUrl;
  580. $date = $this->getApiDateParameter($idSite, $period, $lastTimestampWebsiteProcessed);
  581. $url .= $this->getVisitsRequestUrl($idSite, $period, $date);
  582. $url .= self::APPEND_TO_API_REQUEST;
  583. $visitsInLastPeriods = $visitsLastPeriod = 0;
  584. $success = true;
  585. $urls = array();
  586. $noSegmentUrl = $url;
  587. // already processed above for "day"
  588. if ($period != "day") {
  589. $urls[] = $url;
  590. $this->requests++;
  591. }
  592. foreach ($this->getSegmentsForSite($idSite) as $segment) {
  593. $urlWithSegment = $url . '&segment=' . urlencode($segment);
  594. $urls[] = $urlWithSegment;
  595. $this->requests++;
  596. }
  597. $cliMulti = new CliMulti();
  598. $cliMulti->setAcceptInvalidSSLCertificate($this->acceptInvalidSSLCertificate);
  599. $cliMulti->setConcurrentProcessesLimit($this->getConcurrentRequestsPerWebsite());
  600. $response = $cliMulti->request($urls);
  601. foreach ($urls as $index => $url) {
  602. $content = array_key_exists($index, $response) ? $response[$index] : null;
  603. $success = $success && $this->checkResponse($content, $url);
  604. if ($noSegmentUrl === $url && $success) {
  605. $stats = @unserialize($content);
  606. if (!is_array($stats)) {
  607. $this->logError("Error unserializing the following response from $url: " . $content);
  608. }
  609. $visitsInLastPeriods = $this->getVisitsFromApiResponse($stats);
  610. $visitsLastPeriod = $this->getVisitsLastPeriodFromApiResponse($stats);
  611. }
  612. }
  613. // we have already logged the daily archive above
  614. if($period != "day") {
  615. $this->logArchivedWebsite($idSite, $period, $date, $visitsInLastPeriods, $visitsLastPeriod, $timer);
  616. }
  617. return $success;
  618. }
  619. /**
  620. * Logs a section in the output
  621. */
  622. private function logSection($title = "")
  623. {
  624. $this->log("---------------------------");
  625. if(!empty($title)) {
  626. $this->log($title);
  627. }
  628. }
  629. public function log($m)
  630. {
  631. $this->output .= $m . "\n";
  632. try {
  633. Log::info($m);
  634. } catch(Exception $e) {
  635. print($m . "\n");
  636. }
  637. }
  638. public function logError($m)
  639. {
  640. if (!defined('PIWIK_ARCHIVE_NO_TRUNCATE')) {
  641. $m = substr($m, 0, self::TRUNCATE_ERROR_MESSAGE_SUMMARY);
  642. }
  643. $m = str_replace(array("\n", "\t"), " ", $m);
  644. $this->errors[] = $m;
  645. Log::error($m);
  646. }
  647. private function logNetworkError($url, $response)
  648. {
  649. $message = "Got invalid response from API request: $url. ";
  650. if (empty($response)) {
  651. $message .= "The response was empty. This usually means a server error. This solution to this error is generally to increase the value of 'memory_limit' in your php.ini file. Please check your Web server Error Log file for more details.";
  652. } else {
  653. $message .= "Response was '$response'";
  654. }
  655. $this->logError($message);
  656. return false;
  657. }
  658. /**
  659. * Issues a request to $url
  660. */
  661. private function request($url)
  662. {
  663. $url = $this->piwikUrl . $url . self::APPEND_TO_API_REQUEST;
  664. if($this->shouldStartProfiler) {
  665. $url .= "&xhprof=2";
  666. }
  667. if ($this->testmode) {
  668. $url .= "&testmode=1";
  669. }
  670. try {
  671. $cliMulti = new CliMulti();
  672. $cliMulti->setAcceptInvalidSSLCertificate($this->acceptInvalidSSLCertificate);
  673. $responses = $cliMulti->request(array($url));
  674. $response = !empty($responses) ? array_shift($responses) : null;
  675. } catch (Exception $e) {
  676. return $this->logNetworkError($url, $e->getMessage());
  677. }
  678. if ($this->checkResponse($response, $url)) {
  679. return $response;
  680. }
  681. return false;
  682. }
  683. private function checkResponse($response, $url)
  684. {
  685. if (empty($response)
  686. || stripos($response, 'error')
  687. ) {
  688. return $this->logNetworkError($url, $response);
  689. }
  690. return true;
  691. }
  692. /**
  693. * Configures Piwik\Log so messages are written in output
  694. */
  695. private function initLog()
  696. {
  697. $config = Config::getInstance();
  698. $log = $config->log;
  699. $log['log_only_when_debug_parameter'] = 0;
  700. $log[Log::LOG_WRITERS_CONFIG_OPTION][] = "screen";
  701. $config->log = $log;
  702. // Make sure we log at least INFO (if logger is set to DEBUG then keep it)
  703. $logLevel = Log::getInstance()->getLogLevel();
  704. if ($logLevel < Log::INFO) {
  705. Log::getInstance()->setLogLevel(Log::INFO);
  706. }
  707. }
  708. /**
  709. * Script does run on http:// ONLY if the SU token is specified
  710. */
  711. private function initCheckCli()
  712. {
  713. if (Common::isPhpCliMode()) {
  714. return;
  715. }
  716. $token_auth = Common::getRequestVar('token_auth', '', 'string');
  717. if ($token_auth !== $this->token_auth
  718. || strlen($token_auth) != 32
  719. ) {
  720. die('<b>You must specify the Super User token_auth as a parameter to this script, eg. <code>?token_auth=XYZ</code> if you wish to run this script through the browser. </b><br>
  721. However it is recommended to run it <a href="http://piwik.org/docs/setup-auto-archiving/">via cron in the command line</a>, since it can take a long time to run.<br/>
  722. In a shell, execute for example the following to trigger archiving on the local Piwik server:<br/>
  723. <code>$ /path/to/php /path/to/piwik/console core:archive --url=http://your-website.org/path/to/piwik/</code>');
  724. }
  725. }
  726. /**
  727. * Init Piwik, connect DB, create log & config objects, etc.
  728. */
  729. private function initCore()
  730. {
  731. try {
  732. FrontController::getInstance()->init();
  733. $this->isCoreInited = true;
  734. } catch (Exception $e) {
  735. throw new Exception("ERROR: During Piwik init, Message: " . $e->getMessage());
  736. }
  737. }
  738. public function isCoreInited()
  739. {
  740. return $this->isCoreInited;
  741. }
  742. /**
  743. * Initializes the various parameters to the script, based on input parameters.
  744. *
  745. */
  746. private function initStateFromParameters()
  747. {
  748. $this->todayArchiveTimeToLive = Rules::getTodayArchiveTimeToLive();
  749. $this->processPeriodsMaximumEverySeconds = $this->getDelayBetweenPeriodsArchives();
  750. $this->lastSuccessRunTimestamp = Option::get(self::OPTION_ARCHIVING_FINISHED_TS);
  751. $this->shouldArchiveOnlySitesWithTrafficSince = $this->isShouldArchiveAllSitesWithTrafficSince();
  752. $this->shouldArchiveOnlySpecificPeriods = $this->getPeriodsToProcess();
  753. if($this->shouldArchiveOnlySitesWithTrafficSince === false) {
  754. // force-all-periods is not set here
  755. if (empty($this->lastSuccessRunTimestamp)) {
  756. // First time we run the script
  757. $this->shouldArchiveOnlySitesWithTrafficSince = self::ARCHIVE_SITES_WITH_TRAFFIC_SINCE;
  758. } else {
  759. // there was a previous successful run
  760. $this->shouldArchiveOnlySitesWithTrafficSince = time() - $this->lastSuccessRunTimestamp;
  761. }
  762. } else {
  763. // force-all-periods is set here
  764. $this->archiveAndRespectTTL = false;
  765. if($this->shouldArchiveOnlySitesWithTrafficSince === true) {
  766. // force-all-periods without value
  767. $this->shouldArchiveOnlySitesWithTrafficSince = self::ARCHIVE_SITES_WITH_TRAFFIC_SINCE;
  768. }
  769. }
  770. }
  771. public function filterWebsiteIds(&$websiteIds)
  772. {
  773. // Keep only the websites that do exist
  774. $websiteIds = array_intersect($websiteIds, $this->allWebsites);
  775. /**
  776. * Triggered by the **core:archive** console command so plugins can modify the list of
  777. * websites that the archiving process will be launched for.
  778. *
  779. * Plugins can use this hook to add websites to archive, remove websites to archive, or change
  780. * the order in which websites will be archived.
  781. *
  782. * @param array $websiteIds The list of website IDs to launch the archiving process for.
  783. */
  784. Piwik::postEvent('CronArchive.filterWebsiteIds', array(&$websiteIds));
  785. }
  786. /**
  787. * Returns the list of sites to loop over and archive.
  788. * @return array
  789. */
  790. public function initWebsiteIds()
  791. {
  792. if(count($this->shouldArchiveSpecifiedSites) > 0) {
  793. $this->log("- Will process " . count($this->shouldArchiveSpecifiedSites) . " websites (--force-idsites)");
  794. return $this->shouldArchiveSpecifiedSites;
  795. }
  796. if ($this->shouldArchiveAllSites) {
  797. $this->log("- Will process all " . count($this->allWebsites) . " websites");
  798. return $this->allWebsites;
  799. }
  800. $websiteIds = array_merge(
  801. $this->addWebsiteIdsWithVisitsSinceLastRun(),
  802. $this->getWebsiteIdsToInvalidate()
  803. );
  804. $websiteIds = array_merge($websiteIds, $this->addWebsiteIdsInTimezoneWithNewDay($websiteIds));
  805. return array_unique($websiteIds);
  806. }
  807. private function initTokenAuth()
  808. {
  809. $superUser = Db::get()->fetchRow("SELECT login, token_auth
  810. FROM " . Common::prefixTable("user") . "
  811. WHERE superuser_access = 1
  812. ORDER BY date_registered ASC");
  813. $this->token_auth = $superUser['token_auth'];
  814. }
  815. private function initPiwikHost($piwikUrl = false)
  816. {
  817. // If core:archive command run as a web cron, we use the current hostname+path
  818. if (empty($piwikUrl)) {
  819. if (!empty(self::$url)) {
  820. $piwikUrl = self::$url;
  821. } else {
  822. // example.org/piwik/
  823. $piwikUrl = SettingsPiwik::getPiwikUrl();
  824. }
  825. }
  826. if (!$piwikUrl) {
  827. $this->logFatalErrorUrlExpected();
  828. }
  829. if(!\Piwik\UrlHelper::isLookLikeUrl($piwikUrl)) {
  830. // try adding http:// in case it's missing
  831. $piwikUrl = "http://" . $piwikUrl;
  832. }
  833. if(!\Piwik\UrlHelper::isLookLikeUrl($piwikUrl)) {
  834. $this->logFatalErrorUrlExpected();
  835. }
  836. // ensure there is a trailing slash
  837. if ($piwikUrl[strlen($piwikUrl) - 1] != '/' && !Common::stringEndsWith($piwikUrl, 'index.php')) {
  838. $piwikUrl .= '/';
  839. }
  840. $this->initConfigObject($piwikUrl);
  841. if (Config::getInstance()->General['force_ssl'] == 1) {
  842. $piwikUrl = str_replace('http://', 'https://', $piwikUrl);
  843. }
  844. if (!Common::stringEndsWith($piwikUrl, 'index.php')) {
  845. $piwikUrl .= 'index.php';
  846. }
  847. $this->piwikUrl = $piwikUrl;
  848. }
  849. private function updateIdSitesInvalidatedOldReports()
  850. {
  851. $this->idSitesInvalidatedOldReports = APICoreAdminHome::getWebsiteIdsToInvalidate();
  852. }
  853. /**
  854. * Return All websites that had reports in the past which were invalidated recently
  855. * (see API CoreAdminHome.invalidateArchivedReports)
  856. * eg. when using Python log import script
  857. *
  858. * @return array
  859. */
  860. private function getWebsiteIdsToInvalidate()
  861. {
  862. $this->updateIdSitesInvalidatedOldReports();
  863. if (count($this->idSitesInvalidatedOldReports) > 0) {
  864. $ids = ", IDs: " . implode(", ", $this->idSitesInvalidatedOldReports);
  865. $this->log("- Will process " . count($this->idSitesInvalidatedOldReports)
  866. . " other websites because some old data reports have been invalidated (eg. using the Log Import script) "
  867. . $ids);
  868. }
  869. return $this->idSitesInvalidatedOldReports;
  870. }
  871. /**
  872. * Returns all sites that had visits since specified time
  873. *
  874. * @return string
  875. */
  876. private function addWebsiteIdsWithVisitsSinceLastRun()
  877. {
  878. $sitesIdWithVisits = APISitesManager::getInstance()->getSitesIdWithVisits(time() - $this->shouldArchiveOnlySitesWithTrafficSince);
  879. $websiteIds = !empty($sitesIdWithVisits) ? ", IDs: " . implode(", ", $sitesIdWithVisits) : "";
  880. $prettySeconds = \Piwik\MetricsFormatter::getPrettyTimeFromSeconds( $this->shouldArchiveOnlySitesWithTrafficSince, true, false);
  881. $this->log("- Will process " . count($sitesIdWithVisits) . " websites with new visits since "
  882. . $prettySeconds
  883. . " "
  884. . $websiteIds);
  885. return $sitesIdWithVisits;
  886. }
  887. /**
  888. * Returns the list of timezones where the specified timestamp in that timezone
  889. * is on a different day than today in that timezone.
  890. *
  891. * @return array
  892. */
  893. private function getTimezonesHavingNewDay()
  894. {
  895. $timestamp = $this->lastSuccessRunTimestamp;
  896. $uniqueTimezones = APISitesManager::getInstance()->getUniqueSiteTimezones();
  897. $timezoneToProcess = array();
  898. foreach ($uniqueTimezones as &$timezone) {
  899. $processedDateInTz = Date::factory((int)$timestamp, $timezone);
  900. $currentDateInTz = Date::factory('now', $timezone);
  901. if ($processedDateInTz->toString() != $currentDateInTz->toString()) {
  902. $timezoneToProcess[] = $timezone;
  903. }
  904. }
  905. return $timezoneToProcess;
  906. }
  907. private function hasBeenProcessedSinceMidnight($idSite, $lastTimestampWebsiteProcessedDay)
  908. {
  909. if (false === $lastTimestampWebsiteProcessedDay) {
  910. return true;
  911. }
  912. $timezone = Site::getTimezoneFor($idSite);
  913. $dateInTimezone = Date::factory('now', $timezone);
  914. $midnightInTimezone = $dateInTimezone->setTime('00:00:00');
  915. $lastProcessedDateInTimezone = Date::factory((int) $lastTimestampWebsiteProcessedDay, $timezone);
  916. return $lastProcessedDateInTimezone->getTimestamp() >= $midnightInTimezone->getTimestamp();
  917. }
  918. /**
  919. * Returns the list of websites in which timezones today is a new day
  920. * (compared to the last time archiving was executed)
  921. *
  922. * @param $websiteIds
  923. * @return array Website IDs
  924. */
  925. private function addWebsiteIdsInTimezoneWithNewDay($websiteIds)
  926. {
  927. $timezones = $this->getTimezonesHavingNewDay();
  928. $websiteDayHasFinishedSinceLastRun = APISitesManager::getInstance()->getSitesIdFromTimezones($timezones);
  929. $websiteDayHasFinishedSinceLastRun = array_diff($websiteDayHasFinishedSinceLastRun, $websiteIds);
  930. $this->websiteDayHasFinishedSinceLastRun = $websiteDayHasFinishedSinceLastRun;
  931. if (count($websiteDayHasFinishedSinceLastRun) > 0) {
  932. $ids = !empty($websiteDayHasFinishedSinceLastRun) ? ", IDs: " . implode(", ", $websiteDayHasFinishedSinceLastRun) : "";
  933. $this->log("- Will process " . count($websiteDayHasFinishedSinceLastRun)
  934. . " other websites because the last time they were archived was on a different day (in the website's timezone) "
  935. . $ids);
  936. }
  937. return $websiteDayHasFinishedSinceLastRun;
  938. }
  939. /**
  940. * Test that the specified piwik URL is a valid Piwik endpoint.
  941. */
  942. private function checkPiwikUrlIsValid()
  943. {
  944. $response = $this->request("?module=API&method=API.getDefaultMetricTranslations&format=original&serialize=1");
  945. $responseUnserialized = @unserialize($response);
  946. if ($response === false
  947. || !is_array($responseUnserialized)
  948. ) {
  949. $this->logFatalError("The Piwik URL {$this->piwikUrl} does not seem to be pointing to a Piwik server. Response was '$response'.");
  950. }
  951. }
  952. private function logInitInfo()
  953. {
  954. $this->logSection("INIT");
  955. $this->log("Piwik is installed at: {$this->piwikUrl}");
  956. $this->log("Running Piwik " . Version::VERSION . " as Super User");
  957. }
  958. private function logArchiveTimeoutInfo()
  959. {
  960. $this->logSection("NOTES");
  961. // Recommend to disable browser archiving when using this script
  962. if (Rules::isBrowserTriggerEnabled()) {
  963. $this->log("- If you execute this script at least once per hour (or more often) in a crontab, you may disable 'Browser trigger archiving' in Piwik UI > Settings > General Settings. ");
  964. $this->log(" See the doc at: http://piwik.org/docs/setup-auto-archiving/");
  965. }
  966. $this->log("- Reports for today will be processed at most every " . $this->todayArchiveTimeToLive
  967. . " seconds. You can change this value in Piwik UI > Settings > General Settings.");
  968. $this->log("- Reports for the current week/month/year will be refreshed at most every "
  969. . $this->processPeriodsMaximumEverySeconds . " seconds.");
  970. // Try and not request older data we know is already archived
  971. if ($this->lastSuccessRunTimestamp !== false) {
  972. $dateLast = time() - $this->lastSuccessRunTimestamp;
  973. $this->log("- Archiving was last executed without error " . \Piwik\MetricsFormatter::getPrettyTimeFromSeconds($dateLast, true, $isHtml = false) . " ago");
  974. }
  975. }
  976. /**
  977. * Returns the delay in seconds, that should be enforced, between calling archiving for Periods Archives.
  978. * It can be set by --force-timeout-for-periods=X
  979. *
  980. * @return int
  981. */
  982. private function getDelayBetweenPeriodsArchives()
  983. {
  984. if (empty($this->forceTimeoutPeriod)) {
  985. return self::SECONDS_DELAY_BETWEEN_PERIOD_ARCHIVES;
  986. }
  987. // Ensure the cache for periods is at least as high as cache for today
  988. if ($this->forceTimeoutPeriod > $this->todayArchiveTimeToLive) {
  989. return $this->forceTimeoutPeriod;
  990. }
  991. $this->log("WARNING: Automatically increasing --force-timeout-for-periods from {$this->forceTimeoutPeriod} to "
  992. . $this->todayArchiveTimeToLive
  993. . " to match the cache timeout for Today's report specified in Piwik UI > Settings > General Settings");
  994. return $this->todayArchiveTimeToLive;
  995. }
  996. private function isShouldArchiveAllSitesWithTrafficSince()
  997. {
  998. if (empty($this->shouldArchiveAllPeriodsSince)) {
  999. return false;
  1000. }
  1001. if (is_numeric($this->shouldArchiveAllPeriodsSince)
  1002. && $this->shouldArchiveAllPeriodsSince > 1
  1003. ) {
  1004. return (int)$this->shouldArchiveAllPeriodsSince;
  1005. }
  1006. return true;
  1007. }
  1008. /**
  1009. * @param $idSite
  1010. */
  1011. protected function removeWebsiteFromInvalidatedWebsites($idSite)
  1012. {
  1013. $websiteIdsInvalidated = APICoreAdminHome::getWebsiteIdsToInvalidate();
  1014. if (count($websiteIdsInvalidated)) {
  1015. $found = array_search($idSite, $websiteIdsInvalidated);
  1016. if ($found !== false) {
  1017. unset($websiteIdsInvalidated[$found]);
  1018. Option::set(APICoreAdminHome::OPTION_INVALIDATED_IDSITES, serialize($websiteIdsInvalidated));
  1019. }
  1020. }
  1021. }
  1022. private function logFatalErrorUrlExpected()
  1023. {
  1024. $this->logFatalError("./console core:archive expects the argument 'url' to be set to your Piwik URL, for example: --url=http://example.org/piwik/ "
  1025. . "\n--help for more information");
  1026. }
  1027. private function getVisitsLastPeriodFromApiResponse($stats)
  1028. {
  1029. if(empty($stats)) {
  1030. return 0;
  1031. }
  1032. $today = end($stats);
  1033. return $today['nb_visits'];
  1034. }
  1035. private function getVisitsFromApiResponse($stats)
  1036. {
  1037. if(empty($stats)) {
  1038. return 0;
  1039. }
  1040. $visits = 0;
  1041. foreach($stats as $metrics) {
  1042. if(empty($metrics['nb_visits'])) {
  1043. continue;
  1044. }
  1045. $visits += $metrics['nb_visits'];
  1046. }
  1047. return $visits;
  1048. }
  1049. /**
  1050. * @param $idSite
  1051. * @param $period
  1052. * @param $lastTimestampWebsiteProcessed
  1053. * @return float|int|true
  1054. */
  1055. private function getApiDateParameter($idSite, $period, $lastTimestampWebsiteProcessed = false)
  1056. {
  1057. $dateRangeForced = $this->getDateRangeToProcess();
  1058. if(!empty($dateRangeForced)) {
  1059. return $dateRangeForced;
  1060. }
  1061. return $this->getDateLastN($idSite, $period, $lastTimestampWebsiteProcessed);
  1062. }
  1063. /**
  1064. * @param $idSite
  1065. * @param $period
  1066. * @param $date
  1067. * @param $visitsInLastPeriods
  1068. * @param $visitsToday
  1069. * @param $timer
  1070. */
  1071. private function logArchivedWebsite($idSite, $period, $date, $visitsInLastPeriods, $visitsToday, Timer $timer)
  1072. {
  1073. if(substr($date, 0, 4) === 'last') {
  1074. $visitsInLastPeriods = (int)$visitsInLastPeriods . " visits in last " . $date . " " . $period . "s, ";
  1075. $thisPeriod = $period == "day" ? "today" : "this " . $period;
  1076. $visitsInLastPeriod = (int)$visitsToday . " visits " . $thisPeriod . ", ";
  1077. } else {
  1078. $visitsInLastPeriods = (int)$visitsInLastPeriods . " visits in " . $period . "s included in: $date, ";
  1079. $visitsInLastPeriod = '';
  1080. }
  1081. $this->log("Archived website id = $idSite, period = $period, "
  1082. . $visitsInLastPeriods
  1083. . $visitsInLastPeriod
  1084. . $timer->__toString());
  1085. }
  1086. private function getDateRangeToProcess()
  1087. {
  1088. if (empty($this->restrictToDateRange)) {
  1089. return false;
  1090. }
  1091. if (strpos($this->restrictToDateRange, ',') === false) {
  1092. throw new Exception("--force-date-range expects a date range ie. YYYY-MM-DD,YYYY-MM-DD");
  1093. }
  1094. return $this->restrictToDateRange;
  1095. }
  1096. /**
  1097. * @return array
  1098. */
  1099. private function getPeriodsToProcess()
  1100. {
  1101. $this->restrictToPeriods = array_intersect($this->restrictToPeriods, $this->getDefaultPeriodsToProcess());
  1102. $this->restrictToPeriods = array_intersect($this->restrictToPeriods, PeriodFactory::getPeriodsEnabledForAPI());
  1103. return $this->restrictToPeriods;
  1104. }
  1105. /**
  1106. * @return array
  1107. */
  1108. private function getDefaultPeriodsToProcess()
  1109. {
  1110. return array('day', 'week', 'month', 'year');
  1111. }
  1112. /**
  1113. * @param $idSite
  1114. * @return bool
  1115. */
  1116. private function isOldReportInvalidatedForWebsite($idSite)
  1117. {
  1118. return in_array($idSite, $this->idSitesInvalidatedOldReports);
  1119. }
  1120. private function shouldProcessPeriod($period)
  1121. {
  1122. if(empty($this->shouldArchiveOnlySpecificPeriods)) {
  1123. return true;
  1124. }
  1125. return in_array($period, $this->shouldArchiveOnlySpecificPeriods);
  1126. }
  1127. /**
  1128. * @param $idSite
  1129. * @param $period
  1130. * @param $lastTimestampWebsiteProcessed
  1131. * @return string
  1132. */
  1133. private function getDateLastN($idSite, $period, $lastTimestampWebsiteProcessed)
  1134. {
  1135. $dateLastMax = self::DEFAULT_DATE_LAST;
  1136. if ($period == 'year') {
  1137. $dateLastMax = self::DEFAULT_DATE_LAST_YEARS;
  1138. } elseif ($period == 'week') {
  1139. $dateLastMax = self::DEFAULT_DATE_LAST_WEEKS;
  1140. }
  1141. if (empty($lastTimestampWebsiteProcessed)) {
  1142. $lastTimestampWebsiteProcessed = strtotime(\Piwik\Site::getCreationDateFor($idSite));
  1143. }
  1144. // Enforcing last2 at minimum to work around timing issues and ensure we make most archives available
  1145. $dateLast = floor((time() - $lastTimestampWebsiteProcessed) / 86400) + 2;
  1146. if ($dateLast > $dateLastMax) {
  1147. $dateLast = $dateLastMax;
  1148. }
  1149. if (!empty($this->dateLastForced)) {
  1150. $dateLast = $this->dateLastForced;
  1151. }
  1152. return "last" . $dateLast;
  1153. }
  1154. /**
  1155. * @return int
  1156. */
  1157. private function getConcurrentRequestsPerWebsite()
  1158. {
  1159. if ($this->concurrentRequestsPerWebsite !== false) {
  1160. return $this->concurrentRequestsPerWebsite;
  1161. }
  1162. return self::MAX_CONCURRENT_API_REQUESTS;
  1163. }
  1164. }