ahmed-bakr1988/SaaS-PDF
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add IndexNow submission and sitemap updates

Browse Source 
- Add IndexNow submit script + state tracking
- Update deploy script to notify IndexNow after healthy deploy
- Publish IndexNow verification file in public
- Update sitemaps and add env placeholders
- Pass analytics/ads/IndexNow env vars into frontend build
This commit is contained in:
Your Name
2026-04-04 00:03:46 +02:00
parent f55d726df2
commit 700941a24c
15 changed files with 697 additions and 480 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
.env.example
View File

@@ -74,6 +74,11 @@ POSTGRES_PASSWORD=
# Frontend
VITE_SITE_DOMAIN=https://dociva.io
VITE_SENTRY_DSN=
INDEXNOW_KEY=
INDEXNOW_ENDPOINT=https://www.bing.com/indexnow
INDEXNOW_AUTO_SUBMIT=true
INDEXNOW_STRICT=false
INDEXNOW_FULL_SUBMIT=false
# Frontend Analytics / Ads (Vite)
VITE_GA_MEASUREMENT_ID=G-XXXXXXXXXX
@@ -85,10 +90,9 @@ VITE_ADSENSE_SLOT_HOME_TOP=1234567890
VITE_ADSENSE_SLOT_HOME_BOTTOM=1234567891
VITE_ADSENSE_SLOT_TOP_BANNER=1234567892
VITE_ADSENSE_SLOT_BOTTOM_BANNER=1234567893
VITE_CLARITY_PROJECT_ID=vzw2jb2ipq
# Feature Flags (set to "false" to disable a specific tool)
FEATURE_EDITOR=true
FEATURE_OCR=true
FEATURE_REMOVEBG=true
VITE_CLARITY_PROJECT_ID=vzw2jb2ipq

1
.gitignore vendored
View File

@@ -16,6 +16,7 @@ docs/
node_modules/
frontend/dist/
frontend/build/
frontend/.indexnow/
.npm
*.tsbuildinfo

10
README.md
View File

@@ -79,6 +79,16 @@ docker-compose up --build
- `VITE_ADSENSE_SLOT_BOTTOM_BANNER`
- `DATABASE_PATH`
## 🔎 IndexNow
- Verification file is published at `frontend/public/718dc0aa7c7d4d3ebe71e3f97dacef9c.txt` and copied into the production build.
- Dry-run the payload locally with `cd frontend && npm run indexnow:dry-run`.
- Submit the current sitemap URLs manually with `cd frontend && npm run indexnow:submit`.
- Production deploys call `scripts/deploy.sh`, which runs the same submission step after the health check when `INDEXNOW_AUTO_SUBMIT` is not disabled.
- Successful submissions persist a local state snapshot so later deploys only notify changed or removed URLs instead of re-sending the full sitemap every time.
- `INDEXNOW_STRICT=true` now fails the deployment when the IndexNow request fails.
- Optional env overrides: `INDEXNOW_KEY`, `INDEXNOW_ENDPOINT`, `INDEXNOW_AUTO_SUBMIT`, `INDEXNOW_STRICT`, and `INDEXNOW_FULL_SUBMIT`.
## 📁 Project Structure
```

27
docker-compose.prod.yml
View File

@@ -137,6 +137,25 @@ services:
context: ./frontend
dockerfile: Dockerfile
target: build
args:
VITE_GA_MEASUREMENT_ID: ${VITE_GA_MEASUREMENT_ID:-}
VITE_PLAUSIBLE_DOMAIN: ${VITE_PLAUSIBLE_DOMAIN:-}
VITE_PLAUSIBLE_SRC: ${VITE_PLAUSIBLE_SRC:-https://plausible.io/js/script.js}
VITE_GOOGLE_SITE_VERIFICATION: ${VITE_GOOGLE_SITE_VERIFICATION:-}
VITE_ADSENSE_CLIENT_ID: ${VITE_ADSENSE_CLIENT_ID:-}
VITE_ADSENSE_SLOT_HOME_TOP: ${VITE_ADSENSE_SLOT_HOME_TOP:-}
VITE_ADSENSE_SLOT_HOME_BOTTOM: ${VITE_ADSENSE_SLOT_HOME_BOTTOM:-}
VITE_ADSENSE_SLOT_TOP_BANNER: ${VITE_ADSENSE_SLOT_TOP_BANNER:-}
VITE_ADSENSE_SLOT_BOTTOM_BANNER: ${VITE_ADSENSE_SLOT_BOTTOM_BANNER:-}
VITE_FEATURE_EDITOR: ${VITE_FEATURE_EDITOR:-true}
VITE_FEATURE_OCR: ${VITE_FEATURE_OCR:-true}
VITE_FEATURE_REMOVEBG: ${VITE_FEATURE_REMOVEBG:-true}
VITE_SITE_DOMAIN: ${VITE_SITE_DOMAIN:-}
VITE_SENTRY_DSN: ${VITE_SENTRY_DSN:-}
VITE_CLARITY_PROJECT_ID: ${VITE_CLARITY_PROJECT_ID:-}
INDEXNOW_KEY: ${INDEXNOW_KEY:-}
INDEXNOW_ENDPOINT: ${INDEXNOW_ENDPOINT:-https://www.bing.com/indexnow}
INDEXNOW_STRICT: ${INDEXNOW_STRICT:-false}
environment:
- VITE_GA_MEASUREMENT_ID=${VITE_GA_MEASUREMENT_ID:-}
- VITE_PLAUSIBLE_DOMAIN=${VITE_PLAUSIBLE_DOMAIN:-}
@@ -152,8 +171,15 @@ services:
- VITE_FEATURE_REMOVEBG=${VITE_FEATURE_REMOVEBG:-true}
- VITE_SITE_DOMAIN=${VITE_SITE_DOMAIN:-}
- VITE_SENTRY_DSN=${VITE_SENTRY_DSN:-}
- VITE_CLARITY_PROJECT_ID=${VITE_CLARITY_PROJECT_ID:-}
- INDEXNOW_KEY=${INDEXNOW_KEY:-}
- INDEXNOW_ENDPOINT=${INDEXNOW_ENDPOINT:-https://www.bing.com/indexnow}
- INDEXNOW_STRICT=${INDEXNOW_STRICT:-false}
- INDEXNOW_STATE_DIR=/app/.indexnow
- INDEXNOW_FULL_SUBMIT=${INDEXNOW_FULL_SUBMIT:-false}
volumes:
- frontend_build:/app/dist
- indexnow_state:/app/.indexnow
volumes:
postgres_data:
@@ -162,3 +188,4 @@ volumes:
output_data:
db_data:
frontend_build:
indexnow_state:

38
frontend/Dockerfile
View File

@@ -3,6 +3,44 @@ FROM node:20-alpine AS build
WORKDIR /app
ARG VITE_GA_MEASUREMENT_ID
ARG VITE_PLAUSIBLE_DOMAIN
ARG VITE_PLAUSIBLE_SRC
ARG VITE_GOOGLE_SITE_VERIFICATION
ARG VITE_ADSENSE_CLIENT_ID
ARG VITE_ADSENSE_SLOT_HOME_TOP
ARG VITE_ADSENSE_SLOT_HOME_BOTTOM
ARG VITE_ADSENSE_SLOT_TOP_BANNER
ARG VITE_ADSENSE_SLOT_BOTTOM_BANNER
ARG VITE_FEATURE_EDITOR
ARG VITE_FEATURE_OCR
ARG VITE_FEATURE_REMOVEBG
ARG VITE_SITE_DOMAIN
ARG VITE_SENTRY_DSN
ARG VITE_CLARITY_PROJECT_ID
ARG INDEXNOW_KEY
ARG INDEXNOW_ENDPOINT
ARG INDEXNOW_STRICT
ENV VITE_GA_MEASUREMENT_ID=$VITE_GA_MEASUREMENT_ID \
VITE_PLAUSIBLE_DOMAIN=$VITE_PLAUSIBLE_DOMAIN \
VITE_PLAUSIBLE_SRC=$VITE_PLAUSIBLE_SRC \
VITE_GOOGLE_SITE_VERIFICATION=$VITE_GOOGLE_SITE_VERIFICATION \
VITE_ADSENSE_CLIENT_ID=$VITE_ADSENSE_CLIENT_ID \
VITE_ADSENSE_SLOT_HOME_TOP=$VITE_ADSENSE_SLOT_HOME_TOP \
VITE_ADSENSE_SLOT_HOME_BOTTOM=$VITE_ADSENSE_SLOT_HOME_BOTTOM \
VITE_ADSENSE_SLOT_TOP_BANNER=$VITE_ADSENSE_SLOT_TOP_BANNER \
VITE_ADSENSE_SLOT_BOTTOM_BANNER=$VITE_ADSENSE_SLOT_BOTTOM_BANNER \
VITE_FEATURE_EDITOR=$VITE_FEATURE_EDITOR \
VITE_FEATURE_OCR=$VITE_FEATURE_OCR \
VITE_FEATURE_REMOVEBG=$VITE_FEATURE_REMOVEBG \
VITE_SITE_DOMAIN=$VITE_SITE_DOMAIN \
VITE_SENTRY_DSN=$VITE_SENTRY_DSN \
VITE_CLARITY_PROJECT_ID=$VITE_CLARITY_PROJECT_ID \
INDEXNOW_KEY=$INDEXNOW_KEY \
INDEXNOW_ENDPOINT=$INDEXNOW_ENDPOINT \
INDEXNOW_STRICT=$INDEXNOW_STRICT
# Install dependencies
COPY package.json ./
RUN npm install

2
frontend/package.json
View File

@@ -7,6 +7,8 @@
"dev": "vite",
"prebuild": "node scripts/merge-keywords.mjs && node scripts/generate-seo-assets.mjs",
"build": "tsc --noEmit && vite build && node scripts/render-seo-shells.mjs",
"indexnow:submit": "node scripts/submit-indexnow.mjs",
"indexnow:dry-run": "node scripts/submit-indexnow.mjs --dry-run",
"preview": "vite preview",
"lint": "eslint .",
"test": "vitest run",

1
frontend/public/718dc0aa7c7d4d3ebe71e3f97dacef9c.txt Normal file
View File

@@ -0,0 +1 @@
718dc0aa7c7d4d3ebe71e3f97dacef9c

8
frontend/public/sitemap.xml
View File

@@ -2,18 +2,18 @@
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<sitemap>
<loc>https://dociva.io/sitemaps/static.xml</loc>
<lastmod>2026-04-01</lastmod>
<lastmod>2026-04-03</lastmod>
</sitemap>
<sitemap>
<loc>https://dociva.io/sitemaps/blog.xml</loc>
<lastmod>2026-04-01</lastmod>
<lastmod>2026-04-03</lastmod>
</sitemap>
<sitemap>
<loc>https://dociva.io/sitemaps/tools.xml</loc>
<lastmod>2026-04-01</lastmod>
<lastmod>2026-04-03</lastmod>
</sitemap>
<sitemap>
<loc>https://dociva.io/sitemaps/seo.xml</loc>
<lastmod>2026-04-01</lastmod>
<lastmod>2026-04-03</lastmod>
</sitemap>
</sitemapindex>

10
frontend/public/sitemaps/blog.xml
View File

@@ -2,31 +2,31 @@
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url>
<loc>https://dociva.io/blog/how-to-compress-pdf-online</loc>
<lastmod>2026-04-01</lastmod>
<lastmod>2026-04-03</lastmod>
<changefreq>monthly</changefreq>
<priority>0.6</priority>
</url>
<url>
<loc>https://dociva.io/blog/convert-images-without-losing-quality</loc>
<lastmod>2026-04-01</lastmod>
<lastmod>2026-04-03</lastmod>
<changefreq>monthly</changefreq>
<priority>0.6</priority>
</url>
<url>
<loc>https://dociva.io/blog/ocr-extract-text-from-images</loc>
<lastmod>2026-04-01</lastmod>
<lastmod>2026-04-03</lastmod>
<changefreq>monthly</changefreq>
<priority>0.6</priority>
</url>
<url>
<loc>https://dociva.io/blog/merge-split-pdf-files</loc>
<lastmod>2026-04-01</lastmod>
<lastmod>2026-04-03</lastmod>
<changefreq>monthly</changefreq>
<priority>0.6</priority>
</url>
<url>
<loc>https://dociva.io/blog/ai-chat-with-pdf-documents</loc>
<lastmod>2026-04-01</lastmod>
<lastmod>2026-04-03</lastmod>
<changefreq>monthly</changefreq>
<priority>0.6</priority>
</url>

376
frontend/public/sitemaps/seo.xml
View File

File diff suppressed because it is too large Load Diff

24
frontend/public/sitemaps/static.xml
View File

@@ -2,55 +2,61 @@
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url>
<loc>https://dociva.io/</loc>
<lastmod>2026-04-01</lastmod>
<lastmod>2026-04-03</lastmod>
<changefreq>daily</changefreq>
<priority>1.0</priority>
</url>
<url>
<loc>https://dociva.io/tools</loc>
<lastmod>2026-04-01</lastmod>
<lastmod>2026-04-03</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
<url>
<loc>https://dociva.io/about</loc>
<lastmod>2026-04-01</lastmod>
<lastmod>2026-04-03</lastmod>
<changefreq>monthly</changefreq>
<priority>0.4</priority>
</url>
<url>
<loc>https://dociva.io/contact</loc>
<lastmod>2026-04-01</lastmod>
<lastmod>2026-04-03</lastmod>
<changefreq>monthly</changefreq>
<priority>0.4</priority>
</url>
<url>
<loc>https://dociva.io/privacy</loc>
<lastmod>2026-04-01</lastmod>
<lastmod>2026-04-03</lastmod>
<changefreq>yearly</changefreq>
<priority>0.3</priority>
</url>
<url>
<loc>https://dociva.io/terms</loc>
<lastmod>2026-04-01</lastmod>
<lastmod>2026-04-03</lastmod>
<changefreq>yearly</changefreq>
<priority>0.3</priority>
</url>
<url>
<loc>https://dociva.io/pricing</loc>
<lastmod>2026-04-01</lastmod>
<lastmod>2026-04-03</lastmod>
<changefreq>monthly</changefreq>
<priority>0.7</priority>
</url>
<url>
<loc>https://dociva.io/pricing-transparency</loc>
<lastmod>2026-04-03</lastmod>
<changefreq>monthly</changefreq>
<priority>0.7</priority>
</url>
<url>
<loc>https://dociva.io/blog</loc>
<lastmod>2026-04-01</lastmod>
<lastmod>2026-04-03</lastmod>
<changefreq>weekly</changefreq>
<priority>0.6</priority>
</url>
<url>
<loc>https://dociva.io/developers</loc>
<lastmod>2026-04-01</lastmod>
<lastmod>2026-04-03</lastmod>
<changefreq>monthly</changefreq>
<priority>0.5</priority>
</url>

265
frontend/public/sitemaps/tools.xml
View File

@@ -1,267 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url>
<loc>https://dociva.io/tools/pdf-to-word</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.9</priority>
</url>
<url>
<loc>https://dociva.io/tools/word-to-pdf</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.9</priority>
</url>
<url>
<loc>https://dociva.io/tools/compress-pdf</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.9</priority>
</url>
<url>
<loc>https://dociva.io/tools/merge-pdf</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.9</priority>
</url>
<url>
<loc>https://dociva.io/tools/split-pdf</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
<url>
<loc>https://dociva.io/tools/rotate-pdf</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.7</priority>
</url>
<url>
<loc>https://dociva.io/tools/pdf-to-images</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
<url>
<loc>https://dociva.io/tools/images-to-pdf</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
<url>
<loc>https://dociva.io/tools/watermark-pdf</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.7</priority>
</url>
<url>
<loc>https://dociva.io/tools/protect-pdf</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
<url>
<loc>https://dociva.io/tools/unlock-pdf</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
<url>
<loc>https://dociva.io/tools/page-numbers</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.7</priority>
</url>
<url>
<loc>https://dociva.io/tools/pdf-editor</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
<url>
<loc>https://dociva.io/tools/pdf-flowchart</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.7</priority>
</url>
<url>
<loc>https://dociva.io/tools/pdf-to-excel</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
<url>
<loc>https://dociva.io/tools/remove-watermark-pdf</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.7</priority>
</url>
<url>
<loc>https://dociva.io/tools/reorder-pdf</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.7</priority>
</url>
<url>
<loc>https://dociva.io/tools/extract-pages</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.7</priority>
</url>
<url>
<loc>https://dociva.io/tools/image-converter</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
<url>
<loc>https://dociva.io/tools/image-resize</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
<url>
<loc>https://dociva.io/tools/compress-image</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
<url>
<loc>https://dociva.io/tools/ocr</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
<url>
<loc>https://dociva.io/tools/remove-background</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
<url>
<loc>https://dociva.io/tools/image-to-svg</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
<url>
<loc>https://dociva.io/tools/html-to-pdf</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.7</priority>
</url>
<url>
<loc>https://dociva.io/tools/chat-pdf</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
<url>
<loc>https://dociva.io/tools/summarize-pdf</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
<url>
<loc>https://dociva.io/tools/translate-pdf</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
<url>
<loc>https://dociva.io/tools/extract-tables</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
<url>
<loc>https://dociva.io/tools/qr-code</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.7</priority>
</url>
<url>
<loc>https://dociva.io/tools/video-to-gif</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.7</priority>
</url>
<url>
<loc>https://dociva.io/tools/word-counter</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.6</priority>
</url>
<url>
<loc>https://dociva.io/tools/text-cleaner</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.6</priority>
</url>
<url>
<loc>https://dociva.io/tools/pdf-to-pptx</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
<url>
<loc>https://dociva.io/tools/excel-to-pdf</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
<url>
<loc>https://dociva.io/tools/pptx-to-pdf</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
<url>
<loc>https://dociva.io/tools/sign-pdf</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
<url>
<loc>https://dociva.io/tools/crop-pdf</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.7</priority>
</url>
<url>
<loc>https://dociva.io/tools/flatten-pdf</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.7</priority>
</url>
<url>
<loc>https://dociva.io/tools/repair-pdf</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.7</priority>
</url>
<url>
<loc>https://dociva.io/tools/pdf-metadata</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.6</priority>
</url>
<url>
<loc>https://dociva.io/tools/image-crop</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.7</priority>
</url>
<url>
<loc>https://dociva.io/tools/image-rotate-flip</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.7</priority>
</url>
<url>
<loc>https://dociva.io/tools/barcode-generator</loc>
<lastmod>2026-04-01</lastmod>
<changefreq>weekly</changefreq>
<priority>0.7</priority>
</url>
</urlset>

311
frontend/scripts/submit-indexnow.mjs Normal file
View File

@@ -0,0 +1,311 @@
import { access, mkdir, readdir, readFile, writeFile } from 'node:fs/promises';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const frontendRoot = path.resolve(__dirname, '..');
const publicDir = path.join(frontendRoot, 'public');
const distDir = path.join(frontendRoot, 'dist');
const stateDir = path.resolve(process.env.INDEXNOW_STATE_DIR || path.join(frontendRoot, '.indexnow'));
const stateFile = path.join(stateDir, 'last-submission.json');
const defaultEndpoint = 'https://www.bing.com/indexnow';
const keyPattern = /^[A-Za-z0-9-]{8,128}$/;
const dryRun = process.argv.includes('--dry-run') || process.env.INDEXNOW_DRY_RUN === 'true';
const forceFullSubmit = process.argv.includes('--full') || process.env.INDEXNOW_FULL_SUBMIT === 'true';
const strictMode = process.env.INDEXNOW_STRICT === 'true';
const isDirectRun = process.argv[1] ? path.resolve(process.argv[1]) === __filename : false;
function normalizeOrigin(rawOrigin) {
const normalized = String(rawOrigin || 'https://dociva.io').trim().replace(/\/$/, '');
return new URL(normalized);
}
function normalizeEndpoint(rawEndpoint) {
const endpointUrl = new URL(String(rawEndpoint || defaultEndpoint).trim());
if (endpointUrl.pathname === '/' || !endpointUrl.pathname) {
endpointUrl.pathname = '/indexnow';
}
return endpointUrl.toString();
}
async function pathExists(filePath) {
try {
await access(filePath);
return true;
} catch {
return false;
}
}
async function writeJsonFile(filePath, value) {
await mkdir(path.dirname(filePath), { recursive: true });
await writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`, 'utf8');
}
async function ensureKeyFile(key) {
const keyFileName = `${key}.txt`;
const targets = [publicDir, distDir];
for (const targetDir of targets) {
if (!(await pathExists(targetDir))) {
continue;
}
await mkdir(targetDir, { recursive: true });
await writeFile(path.join(targetDir, keyFileName), key, 'utf8');
}
}
async function resolveKey() {
const envKey = String(process.env.INDEXNOW_KEY || '').trim();
if (envKey) {
if (!keyPattern.test(envKey)) {
throw new Error('INDEXNOW_KEY is not a valid IndexNow key.');
}
await ensureKeyFile(envKey);
return envKey;
}
for (const baseDir of [distDir, publicDir]) {
if (!(await pathExists(baseDir))) {
continue;
}
const entries = await readdir(baseDir);
for (const entry of entries.sort()) {
if (!entry.endsWith('.txt')) {
continue;
}
const candidateKey = entry.slice(0, -4);
if (!keyPattern.test(candidateKey)) {
continue;
}
const contents = (await readFile(path.join(baseDir, entry), 'utf8')).trim();
if (contents === candidateKey) {
return candidateKey;
}
}
}
return '';
}
function extractLocs(xml) {
return [...xml.matchAll(/<loc>([^<]+)<\/loc>/g)].map((match) => match[1].trim());
}
async function collectSitemapFiles() {
const sitemapFiles = [];
for (const baseDir of [distDir, publicDir]) {
const nestedSitemapDir = path.join(baseDir, 'sitemaps');
if (await pathExists(nestedSitemapDir)) {
const entries = await readdir(nestedSitemapDir);
for (const entry of entries.sort()) {
if (entry.endsWith('.xml')) {
sitemapFiles.push(path.join(nestedSitemapDir, entry));
}
}
}
const rootSitemap = path.join(baseDir, 'sitemap.xml');
if (await pathExists(rootSitemap)) {
sitemapFiles.push(rootSitemap);
}
if (sitemapFiles.length > 0) {
break;
}
}
return sitemapFiles;
}
async function collectUrls(siteOrigin) {
const urls = new Set();
const sitemapFiles = await collectSitemapFiles();
for (const sitemapFile of sitemapFiles) {
const xml = await readFile(sitemapFile, 'utf8');
for (const loc of extractLocs(xml)) {
let parsedUrl;
try {
parsedUrl = new URL(loc);
} catch {
continue;
}
if (parsedUrl.host !== siteOrigin.host) {
continue;
}
if (parsedUrl.pathname.endsWith('.xml')) {
continue;
}
urls.add(parsedUrl.toString());
}
}
return [...urls].sort();
}
function chunkUrls(urlList, chunkSize) {
const chunks = [];
for (let index = 0; index < urlList.length; index += chunkSize) {
chunks.push(urlList.slice(index, index + chunkSize));
}
return chunks;
}
async function loadPreviousUrls() {
if (!(await pathExists(stateFile))) {
return [];
}
try {
const payload = JSON.parse(await readFile(stateFile, 'utf8'));
if (!Array.isArray(payload.urls)) {
return [];
}
return payload.urls.filter((url) => typeof url === 'string');
} catch {
return [];
}
}
function diffUrlLists(currentUrls, previousUrls, submitAll = false) {
if (submitAll || previousUrls.length === 0) {
return [...new Set(currentUrls)].sort();
}
const currentSet = new Set(currentUrls);
const previousSet = new Set(previousUrls);
const changedUrls = new Set();
for (const url of currentSet) {
if (!previousSet.has(url)) {
changedUrls.add(url);
}
}
for (const url of previousSet) {
if (!currentSet.has(url)) {
changedUrls.add(url);
}
}
return [...changedUrls].sort();
}
async function persistSubmittedUrls(currentUrls) {
if (dryRun) {
return false;
}
await writeJsonFile(stateFile, {
updatedAt: new Date().toISOString(),
urls: [...new Set(currentUrls)].sort(),
});
return true;
}
async function submitBatch(endpoint, payload, batchIndex, totalBatches) {
if (dryRun) {
console.log(`Dry run: batch ${batchIndex}/${totalBatches} -> ${payload.urlList.length} URLs`);
console.log(JSON.stringify(payload, null, 2));
return;
}
const response = await fetch(endpoint, {
method: 'POST',
headers: {
'Content-Type': 'application/json; charset=utf-8',
},
body: JSON.stringify(payload),
});
const responseText = (await response.text()).trim();
if (!response.ok) {
throw new Error(
`IndexNow request failed with ${response.status}${responseText ? `: ${responseText}` : ''}`,
);
}
console.log(
`Submitted IndexNow batch ${batchIndex}/${totalBatches} with ${payload.urlList.length} URLs (${response.status}).`,
);
}
async function main() {
const siteOrigin = normalizeOrigin(process.env.VITE_SITE_DOMAIN || process.env.SITE_DOMAIN);
const endpoint = normalizeEndpoint(process.env.INDEXNOW_ENDPOINT);
const key = await resolveKey();
if (!key) {
console.log('Skipping IndexNow submission: no verification key was found.');
return;
}
const currentUrls = await collectUrls(siteOrigin);
if (currentUrls.length === 0) {
console.log('Skipping IndexNow submission: no sitemap URLs were found.');
return;
}
const previousUrls = await loadPreviousUrls();
const urlList = diffUrlLists(currentUrls, previousUrls, forceFullSubmit);
if (urlList.length === 0) {
console.log('Skipping IndexNow submission: no changed URLs were detected since the previous successful submission.');
return;
}
const keyLocation = `${siteOrigin.origin}/${key}.txt`;
const payloads = chunkUrls(urlList, 10000).map((chunk) => ({
host: siteOrigin.host,
key,
keyLocation,
urlList: chunk,
}));
console.log(
`${dryRun ? 'Preparing' : 'Submitting'} ${urlList.length} URLs to ${endpoint} using ${keyLocation}.`,
);
for (const [index, payload] of payloads.entries()) {
await submitBatch(endpoint, payload, index + 1, payloads.length);
}
if (await persistSubmittedUrls(currentUrls)) {
console.log(`Saved IndexNow state snapshot to ${stateFile}.`);
}
}
if (isDirectRun) {
main().catch((error) => {
console.error(`IndexNow submission failed: ${error.message}`);
if (strictMode) {
process.exitCode = 1;
}
});
}
export {
diffUrlLists,
extractLocs,
main,
normalizeEndpoint,
normalizeOrigin,
};

40
frontend/scripts/submit-indexnow.test.mjs Normal file
View File

@@ -0,0 +1,40 @@
import { describe, expect, it } from 'vitest';
import {
diffUrlLists,
extractLocs,
normalizeEndpoint,
normalizeOrigin,
} from './submit-indexnow.mjs';
describe('submit-indexnow helpers', () => {
it('normalizes endpoints to the /indexnow path', () => {
expect(normalizeEndpoint('https://www.bing.com')).toBe('https://www.bing.com/indexnow');
expect(normalizeEndpoint('https://www.bing.com/indexnow')).toBe('https://www.bing.com/indexnow');
});
it('normalizes origins without a trailing slash', () => {
expect(normalizeOrigin('https://dociva.io/').toString()).toBe('https://dociva.io/');
});
it('extracts loc entries from sitemap xml', () => {
const xml = `<?xml version="1.0" encoding="UTF-8"?>\n<urlset>\n <url><loc>https://dociva.io/a</loc></url>\n <url><loc>https://dociva.io/b</loc></url>\n</urlset>`;
expect(extractLocs(xml)).toEqual(['https://dociva.io/a', 'https://dociva.io/b']);
});
it('returns the full set on the first submission', () => {
expect(diffUrlLists(['https://dociva.io/a', 'https://dociva.io/b'], [])).toEqual([
'https://dociva.io/a',
'https://dociva.io/b',
]);
});
it('returns only added and removed urls after the first submission', () => {
expect(
diffUrlLists(
['https://dociva.io/a', 'https://dociva.io/c'],
['https://dociva.io/a', 'https://dociva.io/b'],
),
).toEqual(['https://dociva.io/b', 'https://dociva.io/c']);
});
});

56
scripts/deploy.sh
View File

@@ -28,20 +28,44 @@ if [ ! -f ".env" ]; then
exit 1
fi
echo -e "${YELLOW}1/7 — Pulling latest code...${NC}"
read_env_value() {
local key="$1"
local fallback="${2:-}"
local shell_value="${!key-}"
local file_value
file_value="$(grep -E "^${key}=" .env 2>/dev/null | tail -n 1 | cut -d= -f2- || true)"
if [ -n "$shell_value" ]; then
printf '%s' "$shell_value"
elif [ -n "$file_value" ]; then
printf '%s' "$file_value"
else
printf '%s' "$fallback"
fi
}
normalize_bool() {
printf '%s' "$1" | tr '[:upper:]' '[:lower:]'
}
INDEXNOW_AUTO_SUBMIT_VALUE="$(normalize_bool "$(read_env_value INDEXNOW_AUTO_SUBMIT 1)")"
INDEXNOW_STRICT_VALUE="$(normalize_bool "$(read_env_value INDEXNOW_STRICT false)")"
echo -e "${YELLOW}1/8 — Pulling latest code...${NC}"
git pull origin main 2>/dev/null || echo "Not a git repo or no remote, skipping pull."
echo -e "${YELLOW}2/7 — Building Docker images...${NC}"
echo -e "${YELLOW}2/8 — Building Docker images...${NC}"
docker compose -f docker-compose.prod.yml build --no-cache
echo -e "${YELLOW}3/7 — Stopping old containers...${NC}"
echo -e "${YELLOW}3/8 — Stopping old containers...${NC}"
docker compose -f docker-compose.prod.yml down --remove-orphans
echo -e "${YELLOW}4/7 — Starting services...${NC}"
echo -e "${YELLOW}4/8 — Starting services...${NC}"
docker compose -f docker-compose.prod.yml up -d
if [ "${SKIP_AI_RUNTIME_CHECKS:-0}" != "1" ]; then
echo -e "${YELLOW}5/7 — Verifying AI runtime in backend + worker...${NC}"
echo -e "${YELLOW}5/8 — Verifying AI runtime in backend + worker...${NC}"
for service in backend celery_worker; do
if ! docker compose -f docker-compose.prod.yml exec -T "$service" python - <<'PY'
import importlib.util
@@ -69,10 +93,10 @@ PY
fi
done
else
echo -e "${YELLOW}5/7 — Skipping AI runtime checks (SKIP_AI_RUNTIME_CHECKS=1).${NC}"
echo -e "${YELLOW}5/8 — Skipping AI runtime checks (SKIP_AI_RUNTIME_CHECKS=1).${NC}"
fi
echo -e "${YELLOW}6/7 — Waiting for health check...${NC}"
echo -e "${YELLOW}6/8 — Waiting for health check...${NC}"
sleep 10
# Health check
@@ -84,7 +108,23 @@ else
exit 1
fi
echo -e "${YELLOW}7/7 — Current containers:${NC}"
if [ "${INDEXNOW_AUTO_SUBMIT_VALUE:-1}" = "1" ] || [ "${INDEXNOW_AUTO_SUBMIT_VALUE:-true}" = "true" ]; then
echo -e "${YELLOW}7/8 — Submitting URLs to IndexNow...${NC}"
if docker compose -f docker-compose.prod.yml run --rm frontend_build_step node scripts/submit-indexnow.mjs; then
echo -e "${GREEN}✓ IndexNow notification completed.${NC}"
else
if [ "$INDEXNOW_STRICT_VALUE" = "1" ] || [ "$INDEXNOW_STRICT_VALUE" = "true" ]; then
echo -e "${RED}✗ IndexNow notification failed and INDEXNOW_STRICT is enabled.${NC}"
exit 1
fi
echo -e "${YELLOW}! IndexNow notification failed; deployment will continue.${NC}"
fi
else
echo -e "${YELLOW}7/8 — Skipping IndexNow notification (INDEXNOW_AUTO_SUBMIT=0).${NC}"
fi
echo -e "${YELLOW}8/8 — Current containers:${NC}"
docker compose -f docker-compose.prod.yml ps
echo ""