From 07aa53689e4d33cf549421eec9b53091cc5eb4cf Mon Sep 17 00:00:00 2001
From: hbrain <hbrain@piagent.local>
Date: Sat, 20 Jun 2026 17:05:04 +0200
Subject: [PATCH] Updated robots.txt and sitemap added

---
 robots.txt  | 38 +++++++++++++++++++++
 sitemap.php | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 sitemap.xml | 51 ++++++++++++++++++++++++++++
 3 files changed, 186 insertions(+)
 create mode 100644 robots.txt
 create mode 100644 sitemap.php
 create mode 100644 sitemap.xml

diff --git a/robots.txt b/robots.txt
new file mode 100644
index 0000000..f0f2c9b
--- /dev/null
+++ b/robots.txt
@@ -0,0 +1,38 @@
+User-agent: facebookexternalhit
+Allow: /
+
+User-agent: Facebot
+Allow: /
+
+User-agent: meta-externalagent
+Allow: /
+
+User-agent: *
+Content-Signal: search=yes,ai-train=no
+Allow: /
+
+User-agent: Amazonbot
+Disallow: /
+
+User-agent: Applebot-Extended
+Disallow: /
+
+User-agent: Bytespider
+Disallow: /
+
+User-agent: CCBot
+Disallow: /
+
+User-agent: ClaudeBot
+Disallow: /
+
+User-agent: CloudflareBrowserRenderingCrawler
+Disallow: /
+
+User-agent: Google-Extended
+Disallow: /
+
+User-agent: GPTBot
+Disallow: /
+
+Sitemap: https://bubulescu.org/sitemap.xml
diff --git a/sitemap.php b/sitemap.php
new file mode 100644
index 0000000..431f53e
--- /dev/null
+++ b/sitemap.php
@@ -0,0 +1,97 @@
+<?php
+// Dynamic XML sitemap for public MVLog pages and enabled article permalinks.
+declare(strict_types=1);
+
+date_default_timezone_set('Europe/Copenhagen');
+
+$root = __DIR__;
+$config = is_file($root . '/config.php') ? require $root . '/config.php' : [];
+$uploadsDir = (string)($config['uploads_dir'] ?? ($root . '/in-dir'));
+$videosDir = (string)($config['videos_dir'] ?? ($root . '/out-dir'));
+
+function sitemap_xml_escape(string $value): string {
+    return htmlspecialchars($value, ENT_XML1 | ENT_COMPAT, 'UTF-8');
+}
+
+function sitemap_lastmod_from_ts(int|float $ts): string {
+    return gmdate('Y-m-d', (int)$ts);
+}
+
+function sitemap_article_id_is_valid(string $id): bool {
+    return (bool)preg_match('/^[0-9]{14}[a-f0-9]{16}$/', $id);
+}
+
+function sitemap_add_url(array &$urls, string $loc, string $lastmod, string $changefreq, string $priority): void {
+    $urls[$loc] = [
+        'loc' => $loc,
+        'lastmod' => $lastmod,
+        'changefreq' => $changefreq,
+        'priority' => $priority,
+    ];
+}
+
+$urls = [];
+
+$staticPages = [
+    ['https://bubulescu.org/', $root . '/index.php', 'daily', '1.0'],
+    ['https://bubulescu.org/contact.php', $root . '/contact.php', 'monthly', '0.5'],
+    ['https://bubulescu.org/feed.php', $root . '/feed.php', 'monthly', '0.3'],
+];
+foreach ($staticPages as [$loc, $path, $changefreq, $priority]) {
+    if (is_file($path)) {
+        sitemap_add_url($urls, $loc, sitemap_lastmod_from_ts((int)filemtime($path)), $changefreq, $priority);
+    }
+}
+
+foreach (glob(rtrim($uploadsDir, '/') . '/*', GLOB_ONLYDIR) ?: [] as $dir) {
+    if (!is_file($dir . '/.mvlog-permalink')) continue;
+
+    $idFile = $dir . '/.mvlog-id';
+    $articleId = is_file($idFile) ? strtolower(trim((string)@file_get_contents($idFile))) : '';
+    if (!sitemap_article_id_is_valid($articleId)) continue;
+
+    $state = [];
+    $stateFile = $dir . '/.movmaker-state.json';
+    if (is_file($stateFile)) {
+        $decoded = json_decode((string)@file_get_contents($stateFile), true);
+        if (is_array($decoded)) $state = $decoded;
+    }
+
+    $lastmodCandidates = [];
+    foreach ([$dir . '/data.txt', $stateFile, $idFile, $dir . '/.mvlog-permalink'] as $candidate) {
+        if (is_file($candidate)) $lastmodCandidates[] = (int)filemtime($candidate);
+    }
+
+    $output = isset($state['output']) && is_string($state['output']) ? basename($state['output']) : '';
+    if ($output !== '') {
+        $videoPath = rtrim($videosDir, '/') . '/' . $output;
+        if (is_file($videoPath)) $lastmodCandidates[] = (int)filemtime($videoPath);
+    }
+
+    $lastmodTs = $lastmodCandidates ? max($lastmodCandidates) : (int)filemtime($dir);
+    sitemap_add_url(
+        $urls,
+        'https://bubulescu.org/?id=' . rawurlencode($articleId),
+        sitemap_lastmod_from_ts($lastmodTs),
+        'monthly',
+        '0.8'
+    );
+}
+
+ksort($urls, SORT_STRING);
+
+header('Content-Type: application/xml; charset=UTF-8');
+header('X-MVLog-Sitemap: dynamic');
+header('Cache-Control: public, max-age=300');
+
+echo '<?xml version="1.0" encoding="UTF-8"?>' . "\n";
+echo '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">' . "\n";
+foreach ($urls as $url) {
+    echo "  <url>\n";
+    echo '    <loc>' . sitemap_xml_escape($url['loc']) . "</loc>\n";
+    echo '    <lastmod>' . sitemap_xml_escape($url['lastmod']) . "</lastmod>\n";
+    echo '    <changefreq>' . sitemap_xml_escape($url['changefreq']) . "</changefreq>\n";
+    echo '    <priority>' . sitemap_xml_escape($url['priority']) . "</priority>\n";
+    echo "  </url>\n";
+}
+echo "</urlset>\n";
diff --git a/sitemap.xml b/sitemap.xml
new file mode 100644
index 0000000..9ed8161
--- /dev/null
+++ b/sitemap.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url>
+    <loc>https://bubulescu.org/</loc>
+    <lastmod>2026-06-20</lastmod>
+    <changefreq>daily</changefreq>
+    <priority>1.0</priority>
+  </url>
+  <url>
+    <loc>https://bubulescu.org/contact.php</loc>
+    <lastmod>2026-06-07</lastmod>
+    <changefreq>monthly</changefreq>
+    <priority>0.5</priority>
+  </url>
+  <url>
+    <loc>https://bubulescu.org/feed.php</loc>
+    <lastmod>2026-06-05</lastmod>
+    <changefreq>monthly</changefreq>
+    <priority>0.3</priority>
+  </url>
+  <url>
+    <loc>https://bubulescu.org/?id=202605311436027136dfbfe683edb2</loc>
+    <lastmod>2026-06-05</lastmod>
+    <changefreq>monthly</changefreq>
+    <priority>0.8</priority>
+  </url>
+  <url>
+    <loc>https://bubulescu.org/?id=2026053114360277f7ac3306fe8287</loc>
+    <lastmod>2026-06-15</lastmod>
+    <changefreq>monthly</changefreq>
+    <priority>0.8</priority>
+  </url>
+  <url>
+    <loc>https://bubulescu.org/?id=202605311436143f593bdf63a42a1b</loc>
+    <lastmod>2026-06-15</lastmod>
+    <changefreq>monthly</changefreq>
+    <priority>0.8</priority>
+  </url>
+  <url>
+    <loc>https://bubulescu.org/?id=202606041612488d1c9612336a71c3</loc>
+    <lastmod>2026-06-14</lastmod>
+    <changefreq>monthly</changefreq>
+    <priority>0.8</priority>
+  </url>
+  <url>
+    <loc>https://bubulescu.org/?id=20260613064744a37e9ccc4442a143</loc>
+    <lastmod>2026-06-20</lastmod>
+    <changefreq>monthly</changefreq>
+    <priority>0.8</priority>
+  </url>
+</urlset>