Form Update

public/robots.txt

@@ -0,0 +1,94 @@
+# =============================================================================
+# robots.txt — gregkilmer.com
+# =============================================================================
+# Strict anti-scraping policy.
+#
+#   - Google/Bing/etc. (legitimate search) are ALLOWED for SEO.
+#   - /api/ and /contact are disallowed for everyone (the form + endpoints
+#     should not be indexed or crawled).
+#   - Known AI training bots are blocked by User-Agent below. Note that
+#     robots.txt is voluntary; determined scrapers will ignore it. This is a
+#     first layer (paired with noindex on /contact and email obfuscation).
+# =============================================================================
+
+# --- Default policy: allow crawling of public pages, block the API + contact
+User-agent: *
+Allow: /
+Disallow: /api/
+Disallow: /contact
+
+# --- Block known AI training / scraping bots explicitly ---------------------
+# OpenAI
+User-agent: GPTBot
+Disallow: /
+
+User-agent: ChatGPT-User
+Disallow: /
+
+User-agent: OAI-SearchBot
+Disallow: /
+
+# Anthropic
+User-agent: ClaudeBot
+Disallow: /
+
+User-agent: Claude-Web
+Disallow: /
+
+User-agent: anthropic-ai
+Disallow: /
+
+# Google's AI/Vertex crawlers (separate from regular Googlebot, which is allowed)
+User-agent: Google-Extended
+Disallow: /
+
+# Common Crawl (feeds many AI training datasets)
+User-agent: CCBot
+Disallow: /
+
+# ByteDance / TikTok
+User-agent: Bytespider
+Disallow: /
+
+# Meta
+User-agent: meta-externalagent
+Disallow: /
+
+User-agent: Meta-ExternalAgent
+Disallow: /
+
+User-agent: FacebookBot
+Disallow: /
+
+# Amazon / Alexa
+User-agent: Amazonbot
+Disallow: /
+
+# Apple's AI/Applebot-Extended (regular Applebot remains allowed for Siri/Suggestions)
+User-agent: Applebot-Extended
+Disallow: /
+
+# Perplexity
+User-agent: PerplexityBot
+Disallow: /
+
+# Cohere
+User-agent: cohere-ai
+Disallow: /
+
+# Diffbot
+User-agent: Diffbot
+Disallow: /
+
+# Imagesift / OC-72 / other image scrapers
+User-agent: ImagesiftBot
+Disallow: /
+
+User-agent: omgili
+Disallow: /
+
+User-agent: Omgilibot
+Disallow: /
+
+# Sitemap (uncomment once a sitemap.xml is generated)
+# Sitemap: https://gregkilmer.com/sitemap.xml