feat: Add IndexNow submission and sitemap updates

- Add IndexNow submit script + state tracking
- Update deploy script to notify IndexNow after healthy deploy
- Publish IndexNow verification file in public
- Update sitemaps and add env placeholders
- Pass analytics/ads/IndexNow env vars into frontend build

frontend/scripts/submit-indexnow.mjs

@@ -0,0 +1,311 @@
+import { access, mkdir, readdir, readFile, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const frontendRoot = path.resolve(__dirname, '..');
+const publicDir = path.join(frontendRoot, 'public');
+const distDir = path.join(frontendRoot, 'dist');
+const stateDir = path.resolve(process.env.INDEXNOW_STATE_DIR || path.join(frontendRoot, '.indexnow'));
+const stateFile = path.join(stateDir, 'last-submission.json');
+const defaultEndpoint = 'https://www.bing.com/indexnow';
+const keyPattern = /^[A-Za-z0-9-]{8,128}$/;
+const dryRun = process.argv.includes('--dry-run') || process.env.INDEXNOW_DRY_RUN === 'true';
+const forceFullSubmit = process.argv.includes('--full') || process.env.INDEXNOW_FULL_SUBMIT === 'true';
+const strictMode = process.env.INDEXNOW_STRICT === 'true';
+const isDirectRun = process.argv[1] ? path.resolve(process.argv[1]) === __filename : false;
+
+function normalizeOrigin(rawOrigin) {
+  const normalized = String(rawOrigin || 'https://dociva.io').trim().replace(/\/$/, '');
+  return new URL(normalized);
+}
+
+function normalizeEndpoint(rawEndpoint) {
+  const endpointUrl = new URL(String(rawEndpoint || defaultEndpoint).trim());
+
+  if (endpointUrl.pathname === '/' || !endpointUrl.pathname) {
+    endpointUrl.pathname = '/indexnow';
+  }
+
+  return endpointUrl.toString();
+}
+
+async function pathExists(filePath) {
+  try {
+    await access(filePath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function writeJsonFile(filePath, value) {
+  await mkdir(path.dirname(filePath), { recursive: true });
+  await writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`, 'utf8');
+}
+
+async function ensureKeyFile(key) {
+  const keyFileName = `${key}.txt`;
+  const targets = [publicDir, distDir];
+
+  for (const targetDir of targets) {
+    if (!(await pathExists(targetDir))) {
+      continue;
+    }
+
+    await mkdir(targetDir, { recursive: true });
+    await writeFile(path.join(targetDir, keyFileName), key, 'utf8');
+  }
+}
+
+async function resolveKey() {
+  const envKey = String(process.env.INDEXNOW_KEY || '').trim();
+  if (envKey) {
+    if (!keyPattern.test(envKey)) {
+      throw new Error('INDEXNOW_KEY is not a valid IndexNow key.');
+    }
+
+    await ensureKeyFile(envKey);
+    return envKey;
+  }
+
+  for (const baseDir of [distDir, publicDir]) {
+    if (!(await pathExists(baseDir))) {
+      continue;
+    }
+
+    const entries = await readdir(baseDir);
+    for (const entry of entries.sort()) {
+      if (!entry.endsWith('.txt')) {
+        continue;
+      }
+
+      const candidateKey = entry.slice(0, -4);
+      if (!keyPattern.test(candidateKey)) {
+        continue;
+      }
+
+      const contents = (await readFile(path.join(baseDir, entry), 'utf8')).trim();
+      if (contents === candidateKey) {
+        return candidateKey;
+      }
+    }
+  }
+
+  return '';
+}
+
+function extractLocs(xml) {
+  return [...xml.matchAll(/<loc>([^<]+)<\/loc>/g)].map((match) => match[1].trim());
+}
+
+async function collectSitemapFiles() {
+  const sitemapFiles = [];
+
+  for (const baseDir of [distDir, publicDir]) {
+    const nestedSitemapDir = path.join(baseDir, 'sitemaps');
+    if (await pathExists(nestedSitemapDir)) {
+      const entries = await readdir(nestedSitemapDir);
+      for (const entry of entries.sort()) {
+        if (entry.endsWith('.xml')) {
+          sitemapFiles.push(path.join(nestedSitemapDir, entry));
+        }
+      }
+    }
+
+    const rootSitemap = path.join(baseDir, 'sitemap.xml');
+    if (await pathExists(rootSitemap)) {
+      sitemapFiles.push(rootSitemap);
+    }
+
+    if (sitemapFiles.length > 0) {
+      break;
+    }
+  }
+
+  return sitemapFiles;
+}
+
+async function collectUrls(siteOrigin) {
+  const urls = new Set();
+  const sitemapFiles = await collectSitemapFiles();
+
+  for (const sitemapFile of sitemapFiles) {
+    const xml = await readFile(sitemapFile, 'utf8');
+
+    for (const loc of extractLocs(xml)) {
+      let parsedUrl;
+
+      try {
+        parsedUrl = new URL(loc);
+      } catch {
+        continue;
+      }
+
+      if (parsedUrl.host !== siteOrigin.host) {
+        continue;
+      }
+
+      if (parsedUrl.pathname.endsWith('.xml')) {
+        continue;
+      }
+
+      urls.add(parsedUrl.toString());
+    }
+  }
+
+  return [...urls].sort();
+}
+
+function chunkUrls(urlList, chunkSize) {
+  const chunks = [];
+
+  for (let index = 0; index < urlList.length; index += chunkSize) {
+    chunks.push(urlList.slice(index, index + chunkSize));
+  }
+
+  return chunks;
+}
+
+async function loadPreviousUrls() {
+  if (!(await pathExists(stateFile))) {
+    return [];
+  }
+
+  try {
+    const payload = JSON.parse(await readFile(stateFile, 'utf8'));
+    if (!Array.isArray(payload.urls)) {
+      return [];
+    }
+
+    return payload.urls.filter((url) => typeof url === 'string');
+  } catch {
+    return [];
+  }
+}
+
+function diffUrlLists(currentUrls, previousUrls, submitAll = false) {
+  if (submitAll || previousUrls.length === 0) {
+    return [...new Set(currentUrls)].sort();
+  }
+
+  const currentSet = new Set(currentUrls);
+  const previousSet = new Set(previousUrls);
+  const changedUrls = new Set();
+
+  for (const url of currentSet) {
+    if (!previousSet.has(url)) {
+      changedUrls.add(url);
+    }
+  }
+
+  for (const url of previousSet) {
+    if (!currentSet.has(url)) {
+      changedUrls.add(url);
+    }
+  }
+
+  return [...changedUrls].sort();
+}
+
+async function persistSubmittedUrls(currentUrls) {
+  if (dryRun) {
+    return false;
+  }
+
+  await writeJsonFile(stateFile, {
+    updatedAt: new Date().toISOString(),
+    urls: [...new Set(currentUrls)].sort(),
+  });
+
+  return true;
+}
+
+async function submitBatch(endpoint, payload, batchIndex, totalBatches) {
+  if (dryRun) {
+    console.log(`Dry run: batch ${batchIndex}/${totalBatches} -> ${payload.urlList.length} URLs`);
+    console.log(JSON.stringify(payload, null, 2));
+    return;
+  }
+
+  const response = await fetch(endpoint, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json; charset=utf-8',
+    },
+    body: JSON.stringify(payload),
+  });
+
+  const responseText = (await response.text()).trim();
+  if (!response.ok) {
+    throw new Error(
+      `IndexNow request failed with ${response.status}${responseText ? `: ${responseText}` : ''}`,
+    );
+  }
+
+  console.log(
+    `Submitted IndexNow batch ${batchIndex}/${totalBatches} with ${payload.urlList.length} URLs (${response.status}).`,
+  );
+}
+
+async function main() {
+  const siteOrigin = normalizeOrigin(process.env.VITE_SITE_DOMAIN || process.env.SITE_DOMAIN);
+  const endpoint = normalizeEndpoint(process.env.INDEXNOW_ENDPOINT);
+  const key = await resolveKey();
+
+  if (!key) {
+    console.log('Skipping IndexNow submission: no verification key was found.');
+    return;
+  }
+
+  const currentUrls = await collectUrls(siteOrigin);
+  if (currentUrls.length === 0) {
+    console.log('Skipping IndexNow submission: no sitemap URLs were found.');
+    return;
+  }
+
+  const previousUrls = await loadPreviousUrls();
+  const urlList = diffUrlLists(currentUrls, previousUrls, forceFullSubmit);
+  if (urlList.length === 0) {
+    console.log('Skipping IndexNow submission: no changed URLs were detected since the previous successful submission.');
+    return;
+  }
+
+  const keyLocation = `${siteOrigin.origin}/${key}.txt`;
+  const payloads = chunkUrls(urlList, 10000).map((chunk) => ({
+    host: siteOrigin.host,
+    key,
+    keyLocation,
+    urlList: chunk,
+  }));
+
+  console.log(
+    `${dryRun ? 'Preparing' : 'Submitting'} ${urlList.length} URLs to ${endpoint} using ${keyLocation}.`,
+  );
+
+  for (const [index, payload] of payloads.entries()) {
+    await submitBatch(endpoint, payload, index + 1, payloads.length);
+  }
+
+  if (await persistSubmittedUrls(currentUrls)) {
+    console.log(`Saved IndexNow state snapshot to ${stateFile}.`);
+  }
+}
+
+if (isDirectRun) {
+  main().catch((error) => {
+    console.error(`IndexNow submission failed: ${error.message}`);
+    if (strictMode) {
+      process.exitCode = 1;
+    }
+  });
+}
+
+export {
+  diffUrlLists,
+  extractLocs,
+  main,
+  normalizeEndpoint,
+  normalizeOrigin,
+};