Python:
from typing import List
from requests.adapters import HTTPAdapter, Retry
from requests import Session, Response
from pathlib import Path
import logging
import re
import os
import argparse
BASE_URL: str = 'http://www.russian-bot.net'
HOME_URL: str = BASE_URL + '/learning.html'
DOCUMENT_URL: str = BASE_URL + '/files/learning/'
OUTPUT_PATH: Path = Path(__file__).parent / 'hacking_content_russian-bot.net'
logging.basicConfig(
format='%(asctime)s - %(levelname)s - %(message)s',
level=logging.INFO
)
logger = logging.getLogger(__name__)
def init_client(**kwargs) -> Session:
"""
Initializes and returns a configured HTTP session with retry logic.
Args:
**kwargs: Arbitrary keyword arguments passed to configure the Retry object.
Returns:
Session: A session object preconfigured with retry logic for both HTTP and HTTPS.
"""
session = Session()
retry = Retry(**kwargs)
session.mount('http://', HTTPAdapter(max_retries=retry))
session.mount('https://', HTTPAdapter(max_retries=retry))
return session
def get_document_names(content: str | Response) -> List[str]:
"""
Extracts document filenames from the given HTML content or Response object.
Args:
content (str | Response): The HTML content as a string or a requests.Response object.
Returns:
List[str]: A list of filenames extracted from the content.
"""
if isinstance(content, Response):
content = content.text
pattern = r'href="files/learning/(?P<filename>.*?)"'
matches = re.findall(pattern, content)
return matches
def download_documents(max_retries: int, continue_on_fail: bool) -> None:
"""
Downloads documents listed in the HOME_URL page and saves them locally.
Args:
max_retries (int): Maximum number of retries for HTTP requests
continue_on_fail (bool): Whether to continue downloading if a single download fails
Raises:
Exception: If the homepage cannot be fetched or a critical error occurs during download.
"""
logger.info('Starting document download process.')
session = init_client(total=max_retries)
logger.info('HTTP session initialized with retry logic.')
logger.info(f'Fetching documents from {HOME_URL}')
resp = session.get(HOME_URL)
resp.raise_for_status()
names = get_document_names(resp.text)
assert names, 'No document URLs found.'
logger.info(f'{len(names)} URLs found')
os.makedirs(OUTPUT_PATH, exist_ok=True)
for name in names:
try:
logger.info(f'Downloading {name}...')
resp = session.get(DOCUMENT_URL + name)
resp.raise_for_status()
output_file = OUTPUT_PATH / name
with open(output_file, 'wb') as f:
f.write(resp.content)
except KeyboardInterrupt:
logger.warning('Process interrupted by user. Exiting.')
return
except Exception as ex:
logger.error(f'Failed to download {name}: {ex}')
if not continue_on_fail:
raise ex
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Download documents from Russian Botnet website')
parser.add_argument(
'--max_retries',
type=int,
default=5,
help='Maximum number of retries for HTTP requests (default: 5)'
)
parser.add_argument(
'--continue_on_fail',
action='store_true',
default=False,
help='Continue downloading even if a single download fails (default: False)'
)
args = parser.parse_args()
try:
download_documents(
max_retries=args.max_retries,
continue_on_fail=args.continue_on_fail
)
except Exception as e:
logger.critical(f'An error occurred: {e}')
Kodu geliştirmeye çalıştım
Kullanıcı komut satırı argümanlarıyla indirme sırasında denenecek girişim sayısını, bu girişim sayısı aşılırsa devam edilip edilmeyeceğini seçebiliyor
Kullanıcı süreci yarıda kesmek için Control + C tuşlarını kullanabiliyor
Kullanıcının süreci canlı takip edebilmesi açısından da logging mekanizması ekledim
400'den fazla doküman var web sitesinde, bu kadar dokümanı daha hızlı indirmek için asenkronize bir yapı implement edilebilir
Ayrıca backward compatibility için Python2 ve Python3'ü aynı anda desteklenecek şekilde de uyarlanabilir, kullandığım kodu eski sürümlerde test etmedim ve büyük ihtimal çalışmayacak
Birileri yaparsa iyi olur
