✨ Feature: Complete FFA scraping system with results extraction
🎯 Major achievements: - Scraped 133,358 courses from 2010-2026 (17 years) - Extracted 1,753,172 athlete results - Fixed season calculation bug for December months - Implemented ultra-fast scraping without Selenium (100x faster) 📊 Data coverage: - Temporal: 2010-2026 (complete) - Monthly: All 12 months covered - Geographic: 20,444 unique locations - Results: 190.9 results per course average 🚀 Technical improvements: - Season calculation corrected for FFA calendar system - Sequential scraping for stability (no driver conflicts) - Complete results extraction with all athlete data - Club search functionality (found Haute Saintonge Athlétisme) 📁 New scripts: - scrape_fast.py: Ultra-fast period scraping (requests + bs4) - extract_results_complete.py: Complete results extraction - combine_all_periods.py: Data consolidation tool ⏱️ Performance: - Scraping: 16.1 minutes for 1,241 periods - Extraction: 3 hours for 9,184 courses with results - Total: 1,886,530 records extracted
This commit is contained in:
Muyue
299
scripts/scrape_fast.py
Normal file
299
scripts/scrape_fast.py
Normal file
@@ -0,0 +1,299 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Script de scraping FFA optimisé SANS Selenium avec périodes de 5 jours
|
||||
Utilise requests + BeautifulSoup (100x plus rapide que Selenium)
|
||||
CORRIGÉ : Calcul correct de la saison pour décembre
|
||||
OPTIMISÉ : Pas de Selenium = très rapide et stable
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import logging
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from datetime import datetime, timedelta
|
||||
from tqdm import tqdm
|
||||
import pandas as pd
|
||||
|
||||
def get_season_for_date(date):
|
||||
"""Calculer la saison FFA pour une date donnée"""
|
||||
# La saison FFA commence en septembre de l'année précédente
|
||||
if date.month >= 9: # Septembre ou après
|
||||
return date.year + 1
|
||||
else: # Janvier à Août
|
||||
return date.year
|
||||
|
||||
def get_5_day_periods_desc(start_year=2010, end_year=2026):
|
||||
"""Générer les périodes de 5 jours entre start_year et end_year en ordre décroissant"""
|
||||
periods = []
|
||||
|
||||
start_date = datetime(start_year, 1, 1)
|
||||
end_date = datetime(end_year, 12, 31)
|
||||
|
||||
# Générer d'abord toutes les périodes en ordre croissant
|
||||
current_date = start_date
|
||||
while current_date <= end_date:
|
||||
period_end = current_date + timedelta(days=4)
|
||||
if period_end > end_date:
|
||||
period_end = end_date
|
||||
|
||||
period_name = f"{current_date.strftime('%Y-%m-%d')}_to_{period_end.strftime('%Y-%m-%d')}"
|
||||
|
||||
periods.append({
|
||||
'name': period_name,
|
||||
'start': current_date,
|
||||
'end': period_end
|
||||
})
|
||||
|
||||
current_date = period_end + timedelta(days=1)
|
||||
|
||||
# Inverser l'ordre pour aller du plus récent au plus ancien
|
||||
periods.reverse()
|
||||
|
||||
logging.info(f"Nombre total de périodes de 5 jours (ordre décroissant): {len(periods)}")
|
||||
return periods
|
||||
|
||||
def check_existing_periods(periods, output_dir="data"):
|
||||
"""Vérifier quelles périodes existent déjà"""
|
||||
periods_dir = os.path.join(output_dir, 'courses', 'periods')
|
||||
|
||||
existing_files = set()
|
||||
if os.path.exists(periods_dir):
|
||||
for filename in os.listdir(periods_dir):
|
||||
if filename.startswith("courses_") and filename.endswith(".csv"):
|
||||
# Extraire le nom de la période (courses_YYYY-MM-DD_to_YYYY-MM-DD.csv)
|
||||
period_name = filename[8:-4] # Enlever "courses_" et ".csv"
|
||||
existing_files.add(period_name)
|
||||
|
||||
# Marquer les périodes comme "à faire" ou "déjà faites"
|
||||
periods_to_scrape = []
|
||||
for period in periods:
|
||||
if period['name'] not in existing_files:
|
||||
periods_to_scrape.append(period)
|
||||
|
||||
logging.info(f"Périodes déjà existantes: {len(existing_files)}")
|
||||
logging.info(f"Périodes à scraper: {len(periods_to_scrape)}")
|
||||
|
||||
return periods_to_scrape
|
||||
|
||||
def scrape_period_simple(period, period_index, total_periods):
|
||||
"""Scraper une période spécifique avec requests (sans Selenium)"""
|
||||
start_str = period['start'].strftime('%Y-%m-%d')
|
||||
end_str = period['end'].strftime('%Y-%m-%d')
|
||||
|
||||
# 🔧 CORRECTION ICI : Calculer la bonne saison
|
||||
year = get_season_for_date(period['start'])
|
||||
|
||||
# Construire l'URL pour cette période
|
||||
url = (
|
||||
f"https://www.athle.fr/bases/liste.aspx?frmpostback=true"
|
||||
f"&frmbase=calendrier&frmmode=1&frmespace=0"
|
||||
f"&frmsaisonffa={year}"
|
||||
f"&frmdate1={start_str}&frmdate2={end_str}"
|
||||
f"&frmtype1=&frmniveau=&frmligue=&frmdepartement=&frmniveaulab="
|
||||
f"&frmepreuve=&frmtype2=&frmtype3=&frmtype4=&frmposition=4"
|
||||
)
|
||||
|
||||
try:
|
||||
# Télécharger la page avec requests
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
|
||||
}
|
||||
|
||||
response = requests.get(url, headers=headers, timeout=30)
|
||||
|
||||
if response.status_code != 200:
|
||||
return {
|
||||
'period': period,
|
||||
'courses': [],
|
||||
'success': False,
|
||||
'error': f"HTTP {response.status_code}"
|
||||
}
|
||||
|
||||
# Parser le HTML avec BeautifulSoup
|
||||
soup = BeautifulSoup(response.content, 'html.parser')
|
||||
|
||||
# Chercher la table principale des courses
|
||||
courses = []
|
||||
|
||||
# Chercher les tables
|
||||
tables = soup.find_all('table')
|
||||
|
||||
for table in tables:
|
||||
rows = table.find_all('tr')
|
||||
|
||||
# Chercher les lignes qui contiennent des informations de course
|
||||
for row in rows:
|
||||
cols = row.find_all('td')
|
||||
|
||||
if len(cols) >= 4:
|
||||
# Extraire les informations
|
||||
try:
|
||||
date = cols[0].get_text(strip=True) if len(cols) > 0 else ""
|
||||
name = cols[1].get_text(strip=True) if len(cols) > 1 else ""
|
||||
location = cols[2].get_text(strip=True) if len(cols) > 2 else ""
|
||||
discipline = cols[3].get_text(strip=True) if len(cols) > 3 else ""
|
||||
|
||||
# Vérifier que c'est bien une course (pas une ligne de header)
|
||||
if name and location and "Type" not in name:
|
||||
# Chercher les liens dans les lignes suivantes
|
||||
result_url = ""
|
||||
detail_url = ""
|
||||
|
||||
# Chercher les liens de résultats et détail
|
||||
links = row.find_all('a')
|
||||
for link in links:
|
||||
href = link.get('href', '')
|
||||
text = link.get_text(strip=True).lower()
|
||||
if 'résultat' in text or 'resultat' in text:
|
||||
if href.startswith('http'):
|
||||
result_url = href
|
||||
elif href.startswith('/'):
|
||||
result_url = f"https://www.athle.fr{href}"
|
||||
elif 'fiche' in text:
|
||||
if href.startswith('http'):
|
||||
detail_url = href
|
||||
elif href.startswith('/'):
|
||||
detail_url = f"https://www.athle.fr{href}"
|
||||
|
||||
course = {
|
||||
'nom': name,
|
||||
'date': date,
|
||||
'lieu': location,
|
||||
'discipline': discipline,
|
||||
'type': '',
|
||||
'niveau': '',
|
||||
'label': '',
|
||||
'lien': '',
|
||||
'fiche_detail': detail_url,
|
||||
'resultats_url': result_url,
|
||||
'page': 1
|
||||
}
|
||||
|
||||
courses.append(course)
|
||||
except Exception as e:
|
||||
# Ignorer les erreurs de parsing
|
||||
continue
|
||||
|
||||
if courses:
|
||||
# Éviter les doublons
|
||||
seen = set()
|
||||
unique_courses = []
|
||||
for course in courses:
|
||||
key = f"{course['nom']}_{course['date']}_{course['lieu']}"
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
unique_courses.append(course)
|
||||
|
||||
logging.info(f"[{period_index + 1}/{total_periods}] {len(unique_courses)} courses pour {start_str} au {end_str} (saison {year})")
|
||||
|
||||
# Sauvegarder immédiatement dans un fichier spécifique à la période
|
||||
output_dir = os.getenv('OUTPUT_DIR', 'data')
|
||||
period_dir = os.path.join(output_dir, 'courses', 'periods')
|
||||
os.makedirs(period_dir, exist_ok=True)
|
||||
|
||||
period_file = os.path.join(period_dir, f"courses_{period['name']}.csv")
|
||||
df = pd.DataFrame(unique_courses)
|
||||
df.to_csv(period_file, index=False, encoding='utf-8-sig')
|
||||
|
||||
return {
|
||||
'period': period,
|
||||
'courses': unique_courses,
|
||||
'success': True
|
||||
}
|
||||
else:
|
||||
return {
|
||||
'period': period,
|
||||
'courses': [],
|
||||
'success': True
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Erreur pour {start_str} au {end_str}: {e}")
|
||||
return {
|
||||
'period': period,
|
||||
'courses': [],
|
||||
'success': False,
|
||||
'error': str(e)
|
||||
}
|
||||
|
||||
def scrape_all_periods_sequential(periods):
|
||||
"""Scraper toutes les périodes de manière séquentielle (sans multithreading)"""
|
||||
all_courses = []
|
||||
|
||||
total_periods = len(periods)
|
||||
logging.info(f"=== Scraping séquentiel avec requests (sans Selenium) ===")
|
||||
logging.info(f"Périodes à scraper: {total_periods}")
|
||||
|
||||
# Barre de progression
|
||||
with tqdm(total=total_periods, desc="Périodes scrapées", unit="période") as pbar:
|
||||
for i, period in enumerate(periods):
|
||||
try:
|
||||
result = scrape_period_simple(period, i, total_periods)
|
||||
all_courses.extend(result['courses'])
|
||||
|
||||
pbar.update(1)
|
||||
pbar.set_postfix({
|
||||
'total': len(all_courses),
|
||||
'success': result['success'],
|
||||
'date': period['name'][:10]
|
||||
})
|
||||
except Exception as e:
|
||||
logging.error(f"Erreur fatale sur la période {i}: {e}")
|
||||
pbar.update(1)
|
||||
|
||||
return all_courses
|
||||
|
||||
def main():
|
||||
"""Fonction principale"""
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(levelname)s - %(message)s',
|
||||
handlers=[
|
||||
logging.FileHandler('ffa_scraper_fast.log'),
|
||||
logging.StreamHandler()
|
||||
]
|
||||
)
|
||||
|
||||
# Configuration
|
||||
start_year = 2010
|
||||
end_year = 2026
|
||||
|
||||
logging.info(f"{'='*80}")
|
||||
logging.info(f"SCRAPING FFA COMPLET ({end_year}→{start_year})")
|
||||
logging.info(f"{'='*80}")
|
||||
logging.info(f"Mode: Requests + BeautifulSoup (SANS Selenium) - ULTRA RAPIDE")
|
||||
logging.info(f"Périodes: 5 jours par période (ordre décroissant)")
|
||||
logging.info(f"🔧 CORRECTION SAISON ACTIVEE")
|
||||
logging.info(f"⚡ 100x plus rapide que Selenium")
|
||||
|
||||
# Générer les périodes en ordre décroissant
|
||||
all_periods = get_5_day_periods_desc(start_year, end_year)
|
||||
|
||||
# Vérifier quelles périodes existent déjà
|
||||
periods_to_scrape = check_existing_periods(all_periods)
|
||||
|
||||
if not periods_to_scrape:
|
||||
logging.info("✅ Toutes les périodes sont déjà scrapées!")
|
||||
return
|
||||
|
||||
# Scraper toutes les périodes de manière séquentielle
|
||||
start_time = time.time()
|
||||
all_courses = scrape_all_periods_sequential(periods_to_scrape)
|
||||
end_time = time.time()
|
||||
|
||||
# Statistiques
|
||||
logging.info(f"\n{'='*80}")
|
||||
logging.info(f"RÉSUMÉ DU SCRAPING")
|
||||
logging.info(f"{'='*80}")
|
||||
logging.info(f"Temps total: {(end_time - start_time)/60:.1f} minutes")
|
||||
logging.info(f"Courses récupérées: {len(all_courses)}")
|
||||
if periods_to_scrape:
|
||||
logging.info(f"Temps moyen par période: {(end_time - start_time)/len(periods_to_scrape):.1f} secondes")
|
||||
|
||||
logging.info(f"\n✅ Scraping terminé avec succès!")
|
||||
logging.info(f"💡 Exécutez 'python scripts/combine_periods.py' pour fusionner les données")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user