"""
Google Site-Search Term Counter
===============================
Collects Google "site:<domain>" result counts for specified search terms
across a list of institution domains. Used to compare relative usage of
terminology (e.g. "student" vs "learner") across university websites.
Authors: Professor Harriet Dunbar-Morris & Dr Peet Morris
Licence: CC BY 4.0
URL: https://harrietdm.com
Requirements:
pip install playwright playwright-stealth
python -m playwright install chromium
Usage:
python search_terms.py [--input INSTS_FILE] [--output RESULTS_FILE]
e.g., python C:\temp\search_terms.py --input c:\temp\insts.txt
The script is resumable: it saves progress after every domain and skips
domains that already have complete results. Re-run to fill gaps left by
rate limiting.
IMPORTANT: Always search bare domains (e.g. "ox.ac.uk", not "www.ox.ac.uk").
Bare-domain searches capture all subdomains and give consistent, complete
counts. Using "www." misses content on subdomains and produces unreliable data.
Google will aggressively rate limit you and will occasionally show you a CAPTCHA to
solve - often, there will be many, so this may take some time.
Our insts.txt input list for the UUK universities:
https://courtauld.ac.uk/
https://greatermanchester.ac.uk/
https://norwichuni.ac.uk/
https://www.abdn.ac.uk/
https://www.aber.ac.uk/
https://www.abertay.ac.uk/
https://www.anglia.ac.uk/
https://www.arts.ac.uk/
https://www.aston.ac.uk/
https://www.bangor.ac.uk/
https://www.bath.ac.uk/
https://www.bathspa.ac.uk/
https://www.bbk.ac.uk/
https://www.bcu.ac.uk/
https://www.beds.ac.uk/
https://www.birmingham.ac.uk/
https://www.bournemouth.ac.uk/
https://www.bradford.ac.uk/
https://www.brighton.ac.uk/
https://www.bristol.ac.uk/
https://www.brookes.ac.uk/
https://www.brunel.ac.uk/
https://www.buckingham.ac.uk/
https://www.bucks.ac.uk/
https://www.cam.ac.uk/
https://www.canterbury.ac.uk/
https://www.cardiff.ac.uk/
https://www.cardiffmet.ac.uk/
https://www.chester.ac.uk/
https://www.chi.ac.uk/
https://www.city.ac.uk/
https://www.coventry.ac.uk/
https://www.cranfield.ac.uk/
https://www.cssd.ac.uk/
https://www.cumbria.ac.uk/
https://www.derby.ac.uk/
https://www.dmu.ac.uk/
https://www.dundee.ac.uk/
https://www.dur.ac.uk/
https://www.ed.ac.uk/
https://www.edgehill.ac.uk/
https://www.essex.ac.uk/
https://www.exeter.ac.uk/
https://www.falmouth.ac.uk/
https://www.gcu.ac.uk/
https://www.gla.ac.uk/
https://www.glos.ac.uk/
https://www.glyndwr.ac.uk/
https://www.gold.ac.uk/
https://www.gre.ac.uk/
https://www.gsa.ac.uk/
https://www.gsmd.ac.uk/
https://www.harper-adams.ac.uk/
https://www.herts.ac.uk/
https://www.hope.ac.uk/
https://www.hud.ac.uk/
https://www.hull.ac.uk/
https://www.hw.ac.uk/
https://www.imperial.ac.uk/
https://www.kcl.ac.uk/
https://www.keele.ac.uk/
https://www.kent.ac.uk/
https://www.lancaster.ac.uk/
https://www.lboro.ac.uk/
https://www.le.ac.uk/
https://www.leeds.ac.uk/
https://www.leedsbeckett.ac.uk/
https://www.leedstrinity.ac.uk/
https://www.lincoln.ac.uk/
https://www.lincolnbishop.ac.uk/
https://www.liverpool.ac.uk/
https://www.ljmu.ac.uk/
https://www.london.ac.uk/
https://www.londonmet.ac.uk/
https://www.lsbu.ac.uk/
https://www.lse.ac.uk/
https://www.lshtm.ac.uk/
https://www.manchester.ac.uk/
https://www.marjon.ac.uk/
https://www.mdx.ac.uk/
https://www.mmu.ac.uk/
https://www.napier.ac.uk/
https://www.ncl.ac.uk/
https://www.newman.ac.uk/
https://www.northampton.ac.uk/
https://www.northumbria.ac.uk/
https://www.nottingham.ac.uk/
https://www.ntu.ac.uk/
https://www.open.ac.uk/
https://www.ox.ac.uk/
https://www.plymouth.ac.uk/
https://www.port.ac.uk/
https://www.qmu.ac.uk/
https://www.qmul.ac.uk/
https://www.qub.ac.uk/
https://www.rca.ac.uk/
https://www.rcm.ac.uk/
https://www.reading.ac.uk/
https://www.regents.ac.uk/
https://www.rgu.ac.uk/
https://www.roehampton.ac.uk/
https://www.royalholloway.ac.uk/
https://www.rvc.ac.uk/
https://www.salford.ac.uk/
https://www.sheffield.ac.uk/
https://www.shu.ac.uk/
https://www.soas.ac.uk/
https://www.solent.ac.uk/
https://www.southampton.ac.uk/
https://www.southwales.ac.uk/
https://www.st-andrews.ac.uk/
https://www.staffs.ac.uk/
https://www.stir.ac.uk/
https://www.stmarys.ac.uk/
https://www.strath.ac.uk/
https://www.sunderland.ac.uk/
https://www.surrey.ac.uk/
https://www.sussex.ac.uk/
https://www.swansea.ac.uk/
https://www.tees.ac.uk/
https://www.trinitylaban.ac.uk/
https://www.uca.ac.uk/
https://www.ucl.ac.uk/
https://www.uclan.ac.uk/
https://www.uea.ac.uk/
https://www.uel.ac.uk/
https://www.uhi.ac.uk/
https://www.ulster.ac.uk/
https://www.uos.ac.uk/
https://www.uwe.ac.uk/
https://www.uwl.ac.uk/
https://www.uws.ac.uk/
https://www.uwtsd.ac.uk/
https://www.warwick.ac.uk/
https://www.westminster.ac.uk/
https://www.winchester.ac.uk/
https://www.wlv.ac.uk/
https://www.worcester.ac.uk/
https://www.york.ac.uk/
https://www.yorksj.ac.uk/
"""
import argparse
import json
import os
import re
import random
import sys
import time
from playwright.sync_api import sync_playwright
try:
from playwright_stealth import Stealth
HAS_STEALTH = True
except ImportError:
HAS_STEALTH = False
print("Warning: playwright-stealth not installed. Running without stealth.")
print(" Install with: pip install playwright-stealth")
# --- Configuration -----------------------------------------------------------
TERMS = ["learner", "learners", "student", "students"]
# Delays between searches (seconds). Longer = less likely to trigger CAPTCHAs.
DELAY_BETWEEN_TERMS = (8, 15)
DELAY_BETWEEN_DOMAINS = (3, 6)
DELAY_AFTER_RATE_LIMIT = (45, 90)
DELAY_COOLDOWN_EVERY_N = 20
DELAY_COOLDOWN = (30, 60)
MAX_RETRIES = 3
# --- Argument parsing --------------------------------------------------------
def parse_args():
parser = argparse.ArgumentParser(
description="Collect Google site: search counts for terminology comparison."
)
parser.add_argument(
"--input", "-i",
default=os.path.join(os.path.dirname(__file__), "insts.txt"),
help="Path to file listing institution URLs, one per line (default: insts.txt)"
)
parser.add_argument(
"--output", "-o",
default=os.path.join(os.path.dirname(__file__), "search_results_v2.json"),
help="Path to JSON results file (default: search_results_v2.json)"
)
return parser.parse_args()
# --- Domain normalisation ----------------------------------------------------
def normalise_domain(url):
"""Strip scheme and www. prefix to get a bare domain for site: searches."""
d = url.strip()
d = re.sub(r'^https?://', '', d)
d = d.rstrip('/')
if d.startswith('www.'):
d = d[4:]
return d
# --- Google result-count extraction ------------------------------------------
def extract_count(page):
"""
Extract the result count from a Google search results page.
Returns:
int >= 0: the count
-999: rate limited / CAPTCHA detected
-1: could not extract (page format issue)
"""
try:
el = page.query_selector("#result-stats")
if el:
txt = el.inner_text()
m = re.search(r'([\d,]+)\s+result', txt)
if m:
return int(m[1].replace(",", ""))
except Exception:
pass
try:
body = page.content()
if "did not match any documents" in body:
return 0
if "unusual traffic" in body or "/sorry/" in page.url:
return -999
except Exception:
pass
return -1
def search_term(page, domain, term):
"""
Perform a single Google site: search and return the result count.
Retries on rate limiting (with increasing delays).
"""
query = f'site:{domain} "{term}"'
url = f"https://www.google.com/search?q={query}"
for attempt in range(1, MAX_RETRIES + 1):
try:
page.goto(url, wait_until="domcontentloaded", timeout=15000)
time.sleep(random.uniform(2, 4))
act_human(page)
count = extract_count(page)
if count == -999:
wait = random.uniform(*DELAY_AFTER_RATE_LIMIT)
print(f" RATE LIMITED (attempt {attempt}/{MAX_RETRIES})"
f" - waiting {wait:.0f}s", flush=True)
print(" Solve the CAPTCHA in the browser window if visible.", flush=True)
time.sleep(wait)
continue
if count == -1:
time.sleep(2)
count = extract_count(page)
if count == -1:
count = 0
return count
except Exception as e:
print(f" ERROR on attempt {attempt}: {e}", flush=True)
if attempt < MAX_RETRIES:
time.sleep(5)
return -999
# --- Institution name extraction ----------------------------------------------
# Known junk strings that Google puts in <h3> elements on non-result pages
JUNK_NAMES = {
"try google search console",
"search console",
"about this page",
"people also ask",
"related searches",
}
def get_inst_name(page, domain):
"""
Try to get the institution's proper name from a Google site: search.
Falls back to the domain if the name can't be reliably extracted.
"""
try:
page.goto(
f"https://www.google.com/search?q=site:{domain}",
wait_until="domcontentloaded", timeout=15000
)
time.sleep(random.uniform(2, 4))
act_human(page)
# Check we're not on a CAPTCHA/sorry page
if "/sorry/" in page.url or "unusual traffic" in (page.content() or ""):
return domain
# Try each <h3> until we find one that isn't Google junk
headings = page.query_selector_all("h3")
for el in headings:
txt = el.inner_text().strip()
if not txt:
continue
if txt.lower() in JUNK_NAMES:
continue
if "google" in txt.lower():
continue
# Clean common suffixes/prefixes
for suffix in [" - Home", " | Home", ": Home", " Homepage",
" - Official Site", " - Official Website",
" | Official Website", " | University",
" - University", " - Welcome",
" | Welcome", ": Welcome"]:
if txt.endswith(suffix):
txt = txt[:-len(suffix)]
for prefix in ["Official Website of the ",
"Official Website of ",
"Welcome to the ",
"Welcome to "]:
if txt.startswith(prefix):
txt = txt[len(prefix):]
return txt.strip()
except Exception:
pass
return domain
# --- Human-like behaviour ----------------------------------------------------
def act_human(page):
"""Random mouse movement and scrolling to reduce bot-detection signals."""
try:
x = random.randint(200, 900)
y = random.randint(200, 500)
page.mouse.move(x, y)
time.sleep(random.uniform(0.3, 0.8))
if random.random() < 0.4:
page.mouse.wheel(0, random.randint(100, 400))
time.sleep(random.uniform(0.5, 1.0))
except Exception:
pass
def accept_cookies(page):
"""Try to accept Google's cookie consent banner."""
try:
btn = page.query_selector('button:has-text("Accept all")')
if btn:
btn.click()
time.sleep(1)
print(" [Accepted cookies]")
except Exception:
pass
# --- Results management ------------------------------------------------------
def load_results(path):
"""Load existing results, or return an empty dict if none exist."""
try:
with open(path, encoding="utf-8") as f:
data = json.load(f)
non_meta = {k: v for k, v in data.items() if not k.startswith('_')}
print(f"Resuming: {len(non_meta)} domains already in {path}")
return data
except (FileNotFoundError, json.JSONDecodeError):
return {}
def save_results(results, path):
"""Write results to JSON, pretty-printed."""
with open(path, "w", encoding="utf-8") as f:
json.dump(results, f, indent=2, ensure_ascii=False)
def is_complete(entry):
"""Check whether a domain entry has all four term counts (no negatives)."""
return (
isinstance(entry, dict)
and entry.get("name")
and all(entry.get(t, -1) >= 0 for t in TERMS)
)
# --- Main --------------------------------------------------------------------
def main():
args = parse_args()
# Load domains
if not os.path.isfile(args.input):
print(f"Error: input file not found: {args.input}")
sys.exit(1)
with open(args.input, encoding="utf-8") as f:
raw_lines = f.read().strip().splitlines()
domains = [normalise_domain(line) for line in raw_lines if line.strip()]
print(f"Loaded {len(domains)} domains from {args.input}")
# Load existing results
results = load_results(args.output)
# Launch browser
pw_context = Stealth().use_sync(sync_playwright()) if HAS_STEALTH else sync_playwright()
with pw_context as p:
browser = p.chromium.launch(
headless=False,
args=['--disable-blink-features=AutomationControlled', '--no-sandbox'],
)
context = browser.new_context(
user_agent=(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/131.0.0.0 Safari/537.36"
),
locale="en-GB",
timezone_id="Europe/London",
viewport={"width": 1366, "height": 768},
)
page = context.new_page()
# Accept cookies on first visit
page.goto("https://www.google.com", wait_until="domcontentloaded", timeout=15000)
time.sleep(2)
accept_cookies(page)
time.sleep(1)
searches_this_session = 0
total = len(domains)
for idx, domain in enumerate(domains):
# Skip already-complete domains
if domain in results and is_complete(results[domain]):
print(f"[{idx+1}/{total}] {domain} - SKIPPED (complete)")
continue
# Periodic cooldown to avoid rate limiting
if searches_this_session > 0 and searches_this_session % DELAY_COOLDOWN_EVERY_N == 0:
pause = random.uniform(*DELAY_COOLDOWN)
print(f" [Cooling off for {pause:.0f}s after"
f" {searches_this_session} searches...]", flush=True)
time.sleep(pause)
print(f"[{idx+1}/{total}] {domain}...", flush=True)
entry = results.get(domain, {})
# Search each term
rate_limited = False
for term in TERMS:
if entry.get(term, -1) >= 0:
print(f" {term}: {entry[term]} (cached)", flush=True)
continue
count = search_term(page, domain, term)
entry[term] = count
searches_this_session += 1
print(f" {term}: {count}", flush=True)
if count == -999:
rate_limited = True
time.sleep(random.uniform(*DELAY_BETWEEN_TERMS))
# Get institution name if not already set (or if it's a junk value)
current_name = entry.get("name", "")
if not current_name or current_name == domain or current_name.lower() in JUNK_NAMES:
name = get_inst_name(page, domain)
entry["name"] = name
print(f" Name: {name}", flush=True)
time.sleep(random.uniform(*DELAY_BETWEEN_DOMAINS))
results[domain] = entry
save_results(results, args.output)
if rate_limited:
cooldown = random.uniform(60, 120)
print(f" Extra cooldown {cooldown:.0f}s...", flush=True)
time.sleep(cooldown)
else:
time.sleep(random.uniform(*DELAY_BETWEEN_DOMAINS))
browser.close()
# Summary
all_entries = {k: v for k, v in results.items() if not k.startswith('_')}
done = sum(1 for v in all_entries.values() if is_complete(v))
incomplete = len(all_entries) - done
print(f"\nFinished. {done}/{len(all_entries)} domains fully complete.")
if incomplete:
print(f"{incomplete} domains still have gaps - re-run to retry them.")
print(f"Results saved to {args.output}")
if __name__ == "__main__":
main()