<?php
/**
 * Nightly traffic aggregator. Walks every site under /docker/users/, parses
 * yesterday's Apache combined log, writes daily totals + top URLs + top
 * bandwidth into per-user monthly SQLite files.
 *
 * Invoked from cron at 1am wrapped with flock + timeout. Per-site failures
 * are logged and skipped; one bad site never kills the run.
 */

// Batch worker over potentially large logs — bump memory beyond the
// default 128M so a single huge line/buffer doesn't OOM the run. Lines
// are streamed, so this is just headroom.
ini_set('memory_limit', '512M');

if (file_exists('/docker/whp/web/libs/traffic_analytics.php')) {
    require_once('/docker/whp/web/libs/traffic_analytics.php');
} else {
    require_once(dirname(__DIR__) . '/libs/traffic_analytics.php');
}

const TA_BOT_REGEX = '/Googlebot|Bingbot|YandexBot|Baiduspider|DuckDuckBot|AhrefsBot|SemrushBot|MJ12bot|DotBot|PetalBot|Sogou|facebookexternalhit|Twitterbot|LinkedInBot|WhatsApp|Slackbot|TelegramBot|Discordbot|Applebot|PerplexityBot|GPTBot|ClaudeBot|bot|crawler|spider|slurp/i';

const TA_ASSET_EXTENSIONS = [
    'css', 'js', 'png', 'jpg', 'jpeg', 'gif', 'svg', 'webp', 'ico',
    'woff', 'woff2', 'ttf', 'map', 'otf', 'eot',
];

// CLI flags. --date and --backfill-days are mutually exclusive: --date
// targets one specific day, --backfill-days=N processes the last N days
// (yesterday, day-before, ...) in one invocation. --verbose prints
// per-site progress.
$cliDate = null;
$backfillDays = 0;
$verbose = false;
foreach ($argv ?? [] as $arg) {
    if (preg_match('/^--date=(\d{4}-\d{2}-\d{2})$/', $arg, $m)) {
        $cliDate = $m[1];
    } elseif (preg_match('/^--backfill-days=(\d+)$/', $arg, $m)) {
        $backfillDays = max(1, min(60, (int)$m[1]));
    } elseif ($arg === '--verbose' || $arg === '-v') {
        $verbose = true;
    } elseif ($arg === '--help' || $arg === '-h') {
        echo "Usage: whp-traffic-aggregator.php [--date=YYYY-MM-DD | --backfill-days=N] [--verbose]\n";
        echo "  --date=YYYY-MM-DD     Target a specific date (default: yesterday).\n";
        echo "  --backfill-days=N     Process the last N days (1..60) in order.\n";
        echo "  --verbose, -v         Print per-site progress.\n";
        exit(0);
    }
}
if ($cliDate !== null && $backfillDays > 0) {
    fwrite(STDERR, "--date and --backfill-days are mutually exclusive\n");
    exit(2);
}

$lockFile = fopen('/tmp/whp-traffic-aggregator.lock', 'c');
if (!$lockFile || !flock($lockFile, LOCK_EX | LOCK_NB)) {
    exit(0);
}

// Per-run temp dir for docker-cp'd shared-httpd logs. Cleaned at the end
// (and on shutdown so a fatal error doesn't leak files).
// getmypid() not posix_getpid(): WHP's CLI PHP has no posix extension, so
// posix_getpid() fatals with "undefined function" and kills the whole run.
$tmpDir = '/tmp/whp-traffic-aggregator-' . getmypid();
@mkdir($tmpDir, 0700, true);
register_shutdown_function(function() use ($tmpDir) {
    if (is_dir($tmpDir)) {
        foreach (glob($tmpDir . '/*') ?: [] as $f) { @unlink($f); }
        @rmdir($tmpDir);
    }
});

$startTime = microtime(true);
$stats = ['users' => 0, 'sites' => 0, 'errors' => 0];

try {
    $password = trim(shell_exec('cat /root/.my.cnf | grep password | cut -f2- -d='));
    $db = new PDO('mysql:host=127.0.0.1;dbname=whp;charset=utf8mb4', 'root', $password);
    $db->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);

    $analytics = new TrafficAnalytics('/docker/users');

    // Build a chronological list of dates to process. For --backfill-days
    // we walk yesterday back N days; --date overrides; otherwise the
    // single-day "yesterday" cron default.
    $datesToProcess = [];
    if ($backfillDays > 0) {
        for ($i = $backfillDays; $i >= 1; $i--) {
            $datesToProcess[] = new DateTimeImmutable('-' . $i . ' days');
        }
    } elseif ($cliDate !== null) {
        $datesToProcess[] = new DateTimeImmutable($cliDate);
    } else {
        $datesToProcess[] = new DateTimeImmutable('yesterday');
    }

    // Build a (username, fqdn) -> [site_id] map. Wildcards stored on disk
    // as "wildcard.foo.com" are joined as "*.foo.com" via translation
    // when scanning user dirs below.
    $sitesStmt = $db->query(
        "SELECT s.id AS site_id,
                s.username AS username,
                s.container_count AS container_count,
                COALESCE(sd.fqdn, d.domain_name) AS fqdn
         FROM whp.sites s
         JOIN whp.domains d ON s.primary_domain_id = d.id
         LEFT JOIN whp.site_domains sd ON sd.site_id = s.id AND sd.is_primary = 1
         WHERE s.username IS NOT NULL"
    );

    $siteByUserDomain = [];
    foreach ($sitesStmt->fetchAll(PDO::FETCH_ASSOC) as $row) {
        $siteByUserDomain[$row['username']][$row['fqdn']] = [
            'site_id' => (int)$row['site_id'],
            'fqdn' => $row['fqdn'],
            'container_count' => max(1, (int)($row['container_count'] ?? 1)),
        ];
    }

    $usersBase = '/docker/users';
    if (!is_dir($usersBase)) {
        echo "No /docker/users/ directory; nothing to do.\n";
        flock($lockFile, LOCK_UN);
        fclose($lockFile);
        exit(0);
    }

    foreach ($datesToProcess as $targetDate) {
        $yesterday = $targetDate->format('Y-m-d');
        $yesterdayApache = $targetDate->format('d/M/Y');
        if ($verbose) {
            echo "Target date: {$yesterday} (Apache filter: {$yesterdayApache})\n";
        }

        foreach (scandir($usersBase) ?: [] as $username) {
            if ($username === '.' || $username === '..') {
                continue;
            }
            $userDir = $usersBase . '/' . $username;
            if (!is_dir($userDir)) {
                continue;
            }
            if (!isset($siteByUserDomain[$username])) {
                continue;
            }
            // users counter only on first iteration so backfill totals are sane
            if ($targetDate === $datesToProcess[0]) {
                $stats['users']++;
            }

            foreach (scandir($userDir) ?: [] as $siteDirName) {
                if ($siteDirName === '.' || $siteDirName === '..' || $siteDirName === 'whp-analytics') {
                    continue;
                }
                $sitePath = $userDir . '/' . $siteDirName;
                if (!is_dir($sitePath)) {
                    continue;
                }

                $fqdn = (strpos($siteDirName, 'wildcard.') === 0)
                    ? ('*.' . substr($siteDirName, strlen('wildcard.')))
                    : $siteDirName;

                $siteInfo = $siteByUserDomain[$username][$fqdn] ?? null;
                if (!$siteInfo) {
                    continue;
                }

                try {
                    $result = whp_ta_processSite(
                        $analytics,
                        $username,
                        $siteInfo['site_id'],
                        $fqdn,
                        $sitePath,
                        $yesterday,
                        $yesterdayApache,
                        $siteInfo['container_count'],
                        $tmpDir
                    );
                    $stats['sites']++;
                    if ($verbose) {
                        printf(
                            "  %s/%s [%s]: requests=%d  human=%d  bytes=%d\n",
                            $username, $siteDirName, $result['source'],
                            $result['total_requests'], $result['human_views'],
                            $result['bytes_sent']
                        );
                    }
                } catch (Throwable $e) {
                    $stats['errors']++;
                    error_log(sprintf(
                        "whp-traffic-aggregator: site %s/%s [%s] failed: %s",
                        $username,
                        $siteDirName,
                        $yesterday,
                        $e->getMessage()
                    ));
                }
            }
        }
    }

    whp_ta_runRetentionSweep('/docker/users', new DateTimeImmutable('today'));
} catch (Throwable $e) {
    error_log("whp-traffic-aggregator fatal: " . $e->getMessage());
    $stats['errors']++;
}

$elapsed = round(microtime(true) - $startTime, 2);
echo sprintf(
    "Processed %d sites across %d users in %ss, %d errors\n",
    $stats['sites'],
    $stats['users'],
    $elapsed,
    $stats['errors']
);

flock($lockFile, LOCK_UN);
fclose($lockFile);

function whp_ta_processSite(
    TrafficAnalytics $analytics,
    string $username,
    int $siteId,
    string $domain,
    string $sitePath,
    string $yesterday,
    string $yesterdayApache,
    int $containerCount = 1,
    string $tmpDir = '/tmp'
): array {
    $totalRequests = 0;
    $humanViews = 0;
    $bytesSent = 0;
    $hitsByUrl = [];
    $bytesByUrl = [];
    $sources = [];

    // A site can legitimately have multiple log sources active at once:
    // a shared-httpd vhost AND a per-site cloud-apache container, or a
    // per-site folder + a default-httpd-image container during migration.
    // We probe every source and feed lines through the same per-line
    // accumulator. The yesterday-timestamp filter inside processLine
    // drops stale lines, so sources that aren't currently writing
    // contribute zero. URL hit counts merge naturally on the same path.

    // 1) Shared-httpd container — vhost configs at
    //    /docker/shared-httpd/vhosts/<domain-with-underscores>.conf;
    //    logs are docker-cp'd from /var/log/httpd/<fqdn>-access.log
    //    into temp files and streamed (busy sites have multi-MB logs
    //    that don't fit in PHP's default memory limit).
    $sharedTmpFiles = whp_ta_collectFromSharedHttpd($domain, $tmpDir);
    if ($sharedTmpFiles !== null) {
        $sources[] = 'shared-httpd';
        foreach ($sharedTmpFiles as $stf) {
            $fh = @fopen($stf, 'r');
            if ($fh) {
                while (($line = fgets($fh)) !== false) {
                    whp_ta_processLine(
                        rtrim($line, "\r\n"), $yesterdayApache,
                        $totalRequests, $humanViews, $bytesSent,
                        $hitsByUrl, $bytesByUrl
                    );
                }
                fclose($fh);
            }
            @unlink($stf);
        }
    }

    // 2) Per-site folder logs — both cloud-apache (symlink trick) and
    //    nginx/Node sites land here.
    $fileCandidates = [
        $sitePath . '/logs/apache/access_log',
        $sitePath . '/logs/apache/access.log',
        $sitePath . '/logs/nginx/access.log',
        $sitePath . '/logs/nginx/access_log',
        $sitePath . '/logs/access_log',
        $sitePath . '/logs/access.log',
    ];
    $logBase = null;
    foreach ($fileCandidates as $c) {
        if (file_exists($c)) {
            $logBase = $c;
            $sources[] = str_replace($sitePath . '/', '', $c);
            break;
        }
    }
    if ($logBase !== null) {
        $logFiles = [$logBase];
        if (file_exists($logBase . '.1')) {
            $logFiles[] = $logBase . '.1';
        }
        foreach ($logFiles as $lf) {
            $fh = @fopen($lf, 'r');
            if (!$fh) {
                error_log("whp-traffic-aggregator: cannot open $lf");
                continue;
            }
            while (($line = fgets($fh)) !== false) {
                whp_ta_processLine(
                    rtrim($line, "\r\n"), $yesterdayApache,
                    $totalRequests, $humanViews, $bytesSent,
                    $hitsByUrl, $bytesByUrl
                );
            }
            fclose($fh);
        }
    }

    // 3) Per-site `docker logs` — only when no other source was found.
    //    Cloud-apache containers already log to per-site files (their
    //    stdout is just init noise), so we don't run docker logs for them.
    //    Default-httpd-image sites whose CustomLog goes to /proc/self/fd/1
    //    are the case this fallback exists for.
    if (empty($sources)) {
        $safeDomain = str_replace('*.', 'wildcard.', $domain);
        $sinceTs = strtotime($yesterday . ' 00:00:00');
        $untilTs = $sinceTs + 86400;
        $producedAny = false;

        for ($i = 1; $i <= $containerCount; $i++) {
            $containerName = $safeDomain . '-' . str_pad((string)$i, 2, '0', STR_PAD_LEFT);
            if (!preg_match('/^[a-zA-Z0-9._-]+$/', $containerName)) {
                continue;
            }
            $cmd = sprintf(
                'docker logs --since=%s --until=%s %s 2>/dev/null',
                escapeshellarg((string)$sinceTs),
                escapeshellarg((string)$untilTs),
                escapeshellarg($containerName)
            );
            $output = @shell_exec($cmd);
            if ($output === null || $output === '') {
                continue;
            }
            $producedAny = true;
            foreach (preg_split("/\r?\n/", $output) as $line) {
                if ($line === '') continue;
                whp_ta_processLine(
                    $line, $yesterdayApache,
                    $totalRequests, $humanViews, $bytesSent,
                    $hitsByUrl, $bytesByUrl
                );
            }
        }

        if ($producedAny) {
            $sources[] = 'docker logs';
        } else {
            throw new RuntimeException(
                "no access log: checked shared-httpd, logs/apache, logs/nginx, " .
                "logs/, and docker logs for {$containerCount} container(s)"
            );
        }
    }

    $sourceLabel = implode(' + ', $sources);

    arsort($hitsByUrl, SORT_NUMERIC);
    $topUrls = [];
    foreach (array_slice($hitsByUrl, 0, 10, true) as $url => $hits) {
        $topUrls[] = ['url' => $url, 'hits' => $hits];
    }

    arsort($bytesByUrl, SORT_NUMERIC);
    $topBandwidth = [];
    foreach (array_slice($bytesByUrl, 0, 10, true) as $url => $bytes) {
        $topBandwidth[] = [
            'url' => $url,
            'bytes_total' => $bytes,
            'hits' => $hitsByUrl[$url] ?? 0,
        ];
    }

    $analytics->recordDailySnapshot(
        $username,
        $siteId,
        $domain,
        $yesterday,
        [
            'total_requests' => $totalRequests,
            'human_views' => $humanViews,
            'bytes_sent' => $bytesSent,
        ],
        $topUrls,
        $topBandwidth
    );

    return [
        'source' => $sourceLabel,
        'total_requests' => $totalRequests,
        'human_views' => $humanViews,
        'bytes_sent' => $bytesSent,
    ];
}

/**
 * Tokenize an Apache combined-format line. Returns null if the line
 * doesn't look like a valid combined entry.
 *
 * Per-line accumulator. Mutates the running counters and the URL maps in
 * place. Used by both the file-based and docker-logs collection paths.
 */
function whp_ta_processLine(
    string $line,
    string $yesterdayApache,
    int &$totalRequests,
    int &$humanViews,
    int &$bytesSent,
    array &$hitsByUrl,
    array &$bytesByUrl
): void {
    $parsed = whp_ta_parseApacheLine($line);
    if (!$parsed) return;
    if (strpos($parsed['timestamp'], $yesterdayApache) === false) return;

    $path = whp_ta_stripQueryString($parsed['path']);
    $status = $parsed['status'];
    $bytes = $parsed['bytes'];

    // total_requests and bytes_sent are the honest raw totals — every line
    // in the access log counts, including 4xx probes and bot traffic. The
    // top-URL maps below need to be more selective: a site full of bot
    // probes against /login.php and /bbs/viewtopic.php (common phpBB/WP
    // exploit paths) was making those URLs dominate the report even
    // though every request returned 404 and no human ever saw them.
    $totalRequests++;
    $bytesSent += $bytes;

    $isHumanView = whp_ta_isHumanView($status, $path, $parsed['user_agent']);
    if ($isHumanView) {
        $humanViews++;
        // Top URLs by hits = what HUMANS actually viewed. Status 2xx/3xx,
        // non-asset path, non-bot UA. This is what the AI traffic
        // analysis reads to talk about "entry-page intent signals".
        $hitsByUrl[$path] = ($hitsByUrl[$path] ?? 0) + 1;
    }

    // Top URLs by bandwidth = what consumed transfer. Filter to 2xx/3xx
    // only (a 4xx response is the default ~250-byte error page, not a
    // real consumer); keep assets and bots in scope so a bot hammering
    // a big media file or a CDN-uncached download still surfaces here.
    if ($status >= 200 && $status < 400) {
        $bytesByUrl[$path] = ($bytesByUrl[$path] ?? 0) + $bytes;
    }
}

/**
 * @return array{timestamp:string,path:string,status:int,bytes:int,user_agent:string}|null
 */
function whp_ta_parseApacheLine(string $line): ?array
{
    $tokens = whp_ta_tokenizeLogLine($line);
    // Common format = 7 tokens (host, ident, user, time, request, status,
    // bytes). Combined adds referer + user-agent for 9. The default httpd
    // image emits common, so accept either — User-Agent is empty when the
    // line doesn't carry one (bot detection still works; the regex just
    // doesn't match).
    if (count($tokens) < 7) {
        return null;
    }

    $timestamp = trim($tokens[3], '[]');
    $request = $tokens[4];
    $statusRaw = $tokens[5];
    $bytesRaw = $tokens[6];
    $userAgent = (count($tokens) >= 9) ? (string)end($tokens) : '';

    if (!ctype_digit($statusRaw)) {
        return null;
    }
    $status = (int)$statusRaw;
    $bytes = ($bytesRaw === '-') ? 0 : (ctype_digit($bytesRaw) ? (int)$bytesRaw : 0);

    $path = '';
    if ($request !== '') {
        $parts = explode(' ', $request);
        $path = $parts[1] ?? '';
    }
    if ($path === '') {
        return null;
    }

    return [
        'timestamp' => $timestamp,
        'path' => $path,
        'status' => $status,
        'bytes' => $bytes,
        'user_agent' => $userAgent,
    ];
}

/**
 * Splits a log line into space-delimited tokens, respecting quoted strings
 * ("...") and bracketed strings ([...]). Honors backslash-escaped quotes
 * inside quoted fields.
 *
 * @return string[]
 */
function whp_ta_tokenizeLogLine(string $line): array
{
    $tokens = [];
    $len = strlen($line);
    $i = 0;
    while ($i < $len) {
        while ($i < $len && $line[$i] === ' ') {
            $i++;
        }
        if ($i >= $len) {
            break;
        }

        $ch = $line[$i];
        if ($ch === '"') {
            $i++;
            $buf = '';
            while ($i < $len) {
                if ($line[$i] === '\\' && $i + 1 < $len) {
                    $buf .= $line[$i + 1];
                    $i += 2;
                    continue;
                }
                if ($line[$i] === '"') {
                    break;
                }
                $buf .= $line[$i];
                $i++;
            }
            $tokens[] = $buf;
            if ($i < $len && $line[$i] === '"') {
                $i++;
            }
        } elseif ($ch === '[') {
            $start = $i;
            $i++;
            while ($i < $len && $line[$i] !== ']') {
                $i++;
            }
            if ($i < $len) {
                $i++;
            }
            $tokens[] = substr($line, $start, $i - $start);
        } else {
            $start = $i;
            while ($i < $len && $line[$i] !== ' ') {
                $i++;
            }
            $tokens[] = substr($line, $start, $i - $start);
        }
    }
    return $tokens;
}

function whp_ta_stripQueryString(string $path): string
{
    $q = strpos($path, '?');
    return $q === false ? $path : substr($path, 0, $q);
}

function whp_ta_isHumanView(int $status, string $path, string $userAgent): bool
{
    if ($status < 200 || $status >= 400) {
        return false;
    }
    $dot = strrpos($path, '.');
    if ($dot !== false) {
        $ext = strtolower(substr($path, $dot + 1));
        $ext = preg_replace('~[^a-z0-9].*$~', '', $ext) ?? $ext;
        if (in_array($ext, TA_ASSET_EXTENSIONS, true)) {
            return false;
        }
    }
    if ($userAgent !== '' && preg_match(TA_BOT_REGEX, $userAgent)) {
        return false;
    }
    return true;
}

/**
 * Pull a domain's access log out of the shared-httpd container if the site
 * has a vhost config there. Returns an array of temp file paths the caller
 * should stream and then delete; an empty array means the site IS on
 * shared-httpd but no log was retrievable (zero-traffic day or missing
 * file); null means the site isn't migrated to shared-httpd.
 *
 * Returning paths (not content) keeps memory bounded — busy sites have
 * multi-MB-to-GB per-domain logs that don't fit in PHP's default limit.
 */
function whp_ta_collectFromSharedHttpd(string $domain, string $tmpDir): ?array
{
    // Vhost config files use underscores in place of dots:
    //   anhonesthost.com           → anhonesthost_com.conf
    //   blog.anti-social.online    → blog_anti-social_online.conf
    // Hyphens are preserved.
    $safeDomain = str_replace('*.', 'wildcard.', $domain);
    $vhostCandidates = [
        "/docker/shared-httpd/vhosts/" . str_replace('.', '_', $domain) . ".conf",
        "/docker/shared-httpd/vhosts/" . str_replace('.', '_', $safeDomain) . ".conf",
    ];
    $hasVhost = false;
    foreach ($vhostCandidates as $v) {
        if (file_exists($v)) { $hasVhost = true; break; }
    }
    if (!$hasVhost) return null;

    $listing = (string)@shell_exec(
        "docker ps --format '{{.Names}}' --filter 'name=shared-httpd' 2>/dev/null"
    );
    $containers = array_filter(
        array_map('trim', preg_split("/\r?\n/", $listing) ?: []),
        function($n) { return $n !== '' && preg_match('/^[a-zA-Z0-9._-]+$/', $n); }
    );
    if (empty($containers)) return [];

    $tmpFiles = [];
    foreach ($containers as $cn) {
        foreach ([$domain, $safeDomain] as $logName) {
            $tmpFile = $tmpDir . '/' . $cn . '-' . str_replace(['/', '*'], '_', $logName) . '-access.log';
            $cmd = sprintf(
                'docker cp %s:%s %s 2>/dev/null',
                escapeshellarg($cn),
                escapeshellarg("/var/log/httpd/{$logName}-access.log"),
                escapeshellarg($tmpFile)
            );
            @shell_exec($cmd);
            if (file_exists($tmpFile) && filesize($tmpFile) > 0) {
                $tmpFiles[] = $tmpFile;
            } elseif (file_exists($tmpFile)) {
                @unlink($tmpFile);
            }
        }
    }
    return $tmpFiles;
}

function whp_ta_runRetentionSweep(string $usersBase, DateTimeImmutable $now): void
{
    if (!is_dir($usersBase)) {
        return;
    }
    $cutoff = $now->modify('-12 months')->format('Y-m');
    foreach (scandir($usersBase) ?: [] as $username) {
        if ($username === '.' || $username === '..') {
            continue;
        }
        $analyticsDir = $usersBase . '/' . $username . '/whp-analytics';
        if (!is_dir($analyticsDir)) {
            continue;
        }
        foreach (scandir($analyticsDir) ?: [] as $f) {
            if (preg_match('/^(\d{4}-\d{2})\.sqlite3$/', $f, $m)) {
                if (strcmp($m[1], $cutoff) < 0) {
                    @unlink($analyticsDir . '/' . $f);
                    @unlink($analyticsDir . '/' . $f . '-wal');
                    @unlink($analyticsDir . '/' . $f . '-shm');
                }
            }
        }
    }
}
