Async BeautifulSoup HTML Table Extractor – Import Logging

aiohttp==3.9.5
aiohttp-robots==0.2.1
beautifulsoup4==4.12.3
lxml==5.2.2
pandas==2.2.2
python-dateutil==2.9.0

"""
Async BeautifulSoup HTML Table Extractor
========================================

A production‑ready asynchronous script that fetches a list of URLs,
extracts every HTML `` on each page, converts them to pandas
DataFrames, and saves the results as CSV files together with a JSON
manifest.

Usage example:
    python async_table_extractor.py --input urls.txt --output tables --concurrency 10

The script respects robots.txt for a user‑agent named "AsyncTableBot",
applies exponential back‑off retries for transient HTTP errors, and
logs all network or parsing issues to `error.log`.
"""

import argparse
import asyncio
import csv
import json
import logging
import os
import re
import sys
from pathlib import Path
from typing import List, Tuple, Dict, Any

import aiohttp
import aiohttp_robots
import pandas as pd
from bs4 import BeautifulSoup
from dateutil import parser as date_parser

# --------------------------------------------------------------------------- #
# Logging configuration
# --------------------------------------------------------------------------- #
logging.basicConfig(
    filename="error.log",
    filemode="a",
    level=logging.ERROR,
    format="%(asctime)s %(levelname)s %(message)s",
)
logger = logging.getLogger(__name__)

# --------------------------------------------------------------------------- #
# Helper functions
# --------------------------------------------------------------------------- #
def sanitize_filename(name: str) -> str:
    """Return a filesystem‑safe version of *name*."""
    return re.sub(r"[^\w\-_.]", "_", name)

def extract_tables(html: str) -> List[pd.DataFrame]:
    """
    Parse *html* with BeautifulSoup and return a list of DataFrames,
    one per  element.
    """
    soup = BeautifulSoup(html, "lxml")
    tables = soup.find_all("table")
    dataframes: List[pd.DataFrame] = []

    for idx, table in enumerate(tables, start=1):
        try:
            # Build a matrix of cells handling rowspan/colspan
            rows: List[List[Any]] = []
            for row in table.find_all("tr"):
                cells: List[Any] = []
                for cell in row.find_all(["td", "th"]):
                    # Determine span
                    rowspan = int(cell.get("rowspan", 1))
                    colspan = int(cell.get("colspan", 1))
                    text = cell.get_text(strip=True)
                    # Expand cell for colspan
                    for _ in range(colspan):
                        cells.append(text)
                    # Store span info for