[Python] Ultimate Harvester - Proxy Destekli, Durdur/Devam Et Özellikli Veri Avcısı

import asyncio
import aiohttp
from bs4 import BeautifulSoup
import re
import sqlite3
from urllib.parse import urljoin
import pickle
import os
import random

START_URL = "https://forum.ornekdomain.com" # <<<--- HEDEF FORUMUN ADRESİNİ BURAYA YAPIŞTIR
DB_FILE = "forum_data.sqlite"
CHECKPOINT_FILE = "forum_checkpoint.pkl"

PROXY_LIST = [
    # "http://kullanici:sifre@ip:port",
]

MAX_PAGES_TO_VISIT = 500_000
CONCURRENT_REQUESTS = 50

REGEX_PATTERNS = {
    "E-posta": r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,7}\b',
    "Telefon (Akıllı)": r'\b(?:\+?\d{1,3}[-.\s]?)?\(?([2-58]\d{2})\)?[-.\s]?(\d{3})[-.\s]?(\d{2})[-.\s]?(\d{2})\b',
    "API_Key (Akıllı)": r'\b((?:sk|pk|rk)_(?:live|test)_[a-zA-Z0-9]{20,}|(?:AIzaSy|ghp_)[a-zA-Z0-9-]{20,}|(?=[a-zA-Z0-9-]{32,})(?=.*[a-z])(?=.*[A-Z])(?=.*[0-9])[a-zA-Z0-9-]+)\b',
    "Username_Potential": r'(?:user|üye|profil|kullanıcı)[\s:/]+([a-zA-Z0-9_.-]{3,})'
}

def setup_database():
    conn = sqlite3.connect(DB_FILE)
    cursor = conn.cursor()
    cursor.execute('''
        CREATE TABLE IF NOT EXISTS found_data (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            source_url TEXT NOT NULL,
            data_type TEXT NOT NULL,
            data_value TEXT NOT NULL,
            found_timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
            UNIQUE(source_url, data_type, data_value)
        )
    ''')
    conn.commit()
    return conn

def load_checkpoint():
    if os.path.exists(CHECKPOINT_FILE):
        with open(CHECKPOINT_FILE, 'rb') as f:
            return pickle.load(f)
    return {START_URL}, set()

def save_checkpoint(to_visit, visited):
    with open(CHECKPOINT_FILE, 'wb') as f:
        pickle.dump((to_visit, visited), f)

async def process_url(session, url, db_conn):
    proxy = random.choice(PROXY_LIST) if PROXY_LIST else None
    new_links = set()
    try:
        async with session.get(url, proxy=proxy, timeout=15) as response:
            if response.status != 200 or 'text/html' not in response.headers.get('Content-Type',''):
                return 0, new_links
         
            html = await response.text()
            soup = BeautifulSoup(html, 'lxml')
            page_text = soup.get_text(" ", strip=True)
         
            data_to_insert = []
            for dtype, pattern in REGEX_PATTERNS.items():
                for match in re.finditer(pattern, page_text):
                    data_to_insert.append((url, dtype, match.group(0)))

            if data_to_insert:
                cursor = db_conn.cursor()
                cursor.executemany("INSERT OR IGNORE INTO found_data (source_url, data_type, data_value) VALUES (?, ?, ?)", data_to_insert)
                db_conn.commit()

            for a_tag in soup.find_all('a', href=True):
                full_url = urljoin(url, a_tag['href'].split('#')[0].strip())
                if full_url.startswith(START_URL):
                    new_links.add(full_url)
         
            return len(data_to_insert), new_links
         
    except Exception:
        return 0, new_links

async def main():
    urls_to_visit, visited_urls = load_checkpoint()
    db_conn = setup_database()
 
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36"}
 
    async with aiohttp.ClientSession(headers=headers) as session:
        while urls_to_visit and len(visited_urls) < MAX_PAGES_TO_VISIT:
            tasks = []
            batch_urls = []
            while urls_to_visit and len(batch_urls) < CONCURRENT_REQUESTS:
                url = urls_to_visit.pop()
                if url not in visited_urls:
                    batch_urls.append(url)
                    visited_urls.add(url)
         
            for url in batch_urls:
                tasks.append(process_url(session, url, db_conn))
         
            results = await asyncio.gather(*tasks)

            for _, new_links in results:
                urls_to_visit.update(new_links - visited_urls)
         
            if len(visited_urls) % 200 == 0:
                save_checkpoint(urls_to_visit, visited_urls)
                cursor = db_conn.cursor()
                total_data = cursor.execute("SELECT COUNT(id) FROM found_data").fetchone()[0]
                print(f"Tarandı: {len(visited_urls)} | Sırada: {len(urls_to_visit)} | Toplam Veri: {total_data}")

    save_checkpoint(urls_to_visit, visited_urls)
    print("Operasyon tamamlandı.")
    db_conn.close()

if __name__ == "__main__":
    asyncio.run(main())