Como validar XML Sitemap automaticamente com Python (SEO técnico)

import requests
import xml.etree.ElementTree as ET
import pandas as pd
from concurrent.futures import ThreadPoolExecutor

# Configurações
SITEMAP_URL = "https://adrock.com.br/sitemap.xml"
HEADERS = {"User-Agent": "Mozilla/5.0"}
TIMEOUT = 10
MAX_THREADS = 10


def get_sitemap_urls(sitemap_url):
    response = requests.get(sitemap_url, headers=HEADERS)
    root = ET.fromstring(response.content)

    urls = []

    for sitemap in root.findall("{http://www.sitemaps.org/schemas/sitemap/0.9}sitemap"):
        loc = sitemap.find("{http://www.sitemaps.org/schemas/sitemap/0.9}loc").text
        urls.extend(get_sitemap_urls(loc))

    for url in root.findall("{http://www.sitemaps.org/schemas/sitemap/0.9}url"):
        loc = url.find("{http://www.sitemaps.org/schemas/sitemap/0.9}loc").text
        urls.append(loc)

    return urls


def check_url(url):
    try:
        response = requests.get(url, headers=HEADERS, timeout=TIMEOUT, allow_redirects=True)
        status = response.status_code
        final_url = response.url

        return {
            "url": url,
            "status": status,
            "final_url": final_url,
            "redirect": url != final_url,
            "response_time": response.elapsed.total_seconds()
        }

    except Exception as e:
        return {
            "url": url,
            "status": "ERROR",
            "final_url": None,
            "redirect": False,
            "response_time": None
        }


def validate_sitemap():
    print("🔍 Carregando sitemap...")
    urls = get_sitemap_urls(SITEMAP_URL)

    print(f"📊 Total de URLs: {len(urls)}")

    results = []

    with ThreadPoolExecutor(max_workers=MAX_THREADS) as executor:
        results = list(executor.map(check_url, urls))

    df = pd.DataFrame(results)

    return df


def generate_report(df):
    print("\n📈 RELATÓRIO\n")

    print("URLs OK (200):", len(df[df["status"] == 200]))
    print("Redirecionamentos:", len(df[df["redirect"] == True]))
    print("Erros:", len(df[df["status"] != 200]))

    df.to_csv("sitemap_report.csv", index=False)
    print("\n📁 Relatório salvo em sitemap_report.csv")


if __name__ == "__main__":
    df = validate_sitemap()
    generate_report(df)

from bs4 import BeautifulSoup

def check_canonical(url):
    try:
        response = requests.get(url, headers=HEADERS, timeout=TIMEOUT)
        soup = BeautifulSoup(response.text, "html.parser")

        canonical_tag = soup.find("link", rel="canonical")

        if canonical_tag:
            canonical_url = canonical_tag.get("href")

            return {
                "url": url,
                "canonical": canonical_url,
                "canonical_ok": url == canonical_url
            }
        else:
            return {
                "url": url,
                "canonical": None,
                "canonical_ok": False
            }

    except:
        return {
            "url": url,
            "canonical": None,
            "canonical_ok": False
        }

import hashlib

def get_content_hash(url):
    try:
        response = requests.get(url, headers=HEADERS, timeout=TIMEOUT)
        content = response.text

        content_hash = hashlib.md5(content.encode("utf-8")).hexdigest()

        return {
            "url": url,
            "hash": content_hash
        }

    except:
        return {
            "url": url,
            "hash": None
        }

def find_duplicates(df):
    duplicates = df[df.duplicated("hash", keep=False)]
    return duplicates.sort_values("hash")