File "auto_import_loop.py"

Full path: /home/algopkco/public_html/scraper/auto_import_loop.py
File size: 1.83 B
MIME-type: text/x-script.python
Charset: utf-8

Download   Open   Edit   Advanced Editor &nnbsp; Back

import os
import shutil
import subprocess
import sys
import time

BASE = os.path.dirname(os.path.abspath(__file__))
DATA = os.path.join(BASE, "data")
LOG = os.path.join(BASE, "auto_import.log")

FILES = ["examsbook_mcqs.jsonl", "netex_mcqs.jsonl", "examsbook_questions_mcqs.jsonl",
         "pakmcqs_mcqs.jsonl", "pakmcqs_questions_mcqs.jsonl", "tpointtech_mcqs.jsonl"]
IMPORTED = {}

def log(msg):
    with open(LOG, "a", encoding="utf-8") as f:
        f.write(f"{time.strftime('%H:%M:%S')} {msg}\n")

def rows(path):
    try:
        with open(path, encoding="utf-8") as f:
            return sum(1 for _ in f)
    except Exception:
        return 0

def main():
    while True:
        for fn in FILES:
            p = os.path.join(DATA, fn)
            n = rows(p)
            if IMPORTED.get(fn, 0) >= n:
                continue
            snap = p + ".snap"
            try:
                shutil.copyfile(p, snap)
            except Exception as e:
                log(f"copy fail {fn} {e}")
                continue
            env = dict(os.environ, PYTHONIOENCODING="utf-8")
            t0 = time.time()
            r = subprocess.run([sys.executable, os.path.join(BASE, "import_global.py"),
                                "--sources", snap], capture_output=True, text=True, env=env)
            out = (r.stdout or "") + (r.stderr or "")
            tail = out.strip().splitlines()
            log(f"[{fn}] {n} rows rc={r.returncode} time={round(time.time()-t0)}s -> "
                + (tail[-1][:200] if tail else "no output"))
            if r.returncode != 0:
                log("  stderr tail: " + (r.stderr or "").strip().splitlines()[-1][:300] if r.stderr else "")
                continue
            try:
                os.remove(snap)
            except Exception:
                pass
        time.sleep(300)

if __name__ == "__main__":
    main()