File "probe_sections.py"

Full path: /home/algopkco/public_html/scraper/probe_sections.py
File size: 1.04 B
MIME-type: text/x-script.python
Charset: utf-8

Download   Open   Edit   Advanced Editor &nnbsp; Back

import re

import requests

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36"
}

sections = [
    "data-interpretation",
    "logical-reasoning",
    "verbal-reasoning",
    "non-verbal-reasoning",
    "general-knowledge",
    "current-affairs",
    "computer-science",
    "programming",
]

for sec in sections:
    url = f"https://www.indiabix.com/{sec}/"
    try:
        r = requests.get(url, headers=headers, timeout=30)
        print("=" * 70)
        print(url, "->", r.status_code)
        if r.status_code == 200:
            links = re.findall(r'href="(https://www\.indiabix\.com/[^"]+)"', r.text)
            cur = [l for l in links if f"/{sec}/" in l]
            seen = []
            for l in cur:
                if l not in seen:
                    seen.append(l)
            for l in seen[:50]:
                if "discussion" not in l and "formulas" not in l:
                    print("  ", l)
    except Exception as e:
        print(url, "ERROR", e)