#
# This robots.txt file provides general permissions for common search engine and
# essential utility crawlers.
#

# 1. Global Rule for All User-Agents (*)
# The 'Allow: /' directive means all public search engines (Google, Bing, DuckDuckGo, etc.)
# are permitted to crawl the entire website. This is the best default for SEO.
User-agent: *
Allow: /


# Disallow common administrative, temporary, or private directories that
# search engines don't need to index.
# NOTE: Customize these lines based on your actual site structure (e.g., if you use WordPress, you might add /wp-admin/)
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /private/
Disallow: /backup/
Disallow: /cache/
Disallow: /shop/*color-*
Disallow: /shop/*size-*
Disallow: /*?*

# 2. Specific Rule for Google AdSense Crawler
# This crawler (Mediapartners-Google) is crucial for validating the ad content
# and context on your pages, especially for personalized ads. It must be allowed.
User-agent: Mediapartners-Google
Allow: /


# 3. Reference to your Sitemap (Highly Recommended)
# Replace 'sitemap.xml' with the actual path to your sitemap file.
Sitemap: https://experiencepuertomorelos.com/sitemap_index.xml