this is also my code.. and we have the same problem

\# ai\_firefox\_scraper.py (Fixed)

import asyncio

import json

import os

import csv

import time

import random

from pathlib import Path

from playwright.async\_api import async\_playwright

SAVE\_DIR = Path("scraped\_data")

SAVE\_DIR.mkdir(exist\_ok=True)

class FirefoxSmartScraper:

def \_\_init\_\_(self, max\_pages=5, throttle=(4, 8)):

self.max\_pages = max\_pages

self.throttle = throttle

async def search\_and\_scrape(self, topic: str):

async with async\_playwright() as p:

browser = await p.firefox.launch(headless=False)

context = await browser.new\_context()

page = await context.new\_page()

print(f"🔍 Searching DuckDuckGo for: {topic}")

await page.goto("https://duckduckgo.com", timeout=30000)

await page.wait\_for\_selector("input\[name='q'\]")

\# Type like a human

for c in topic:

await page.type("input\[name='q'\]", c, delay=random.randint(100, 200))

await page.keyboard.press("Enter")

await page.wait\_for\_selector("a.result\_\_a", timeout=20000)

await asyncio.sleep(random.uniform(\*self.throttle))

\# Extract real links only

items = await page.query\_selector\_all("a.result\_\_a")

urls = \[\]

for item in items\[:self.max\_pages\]:

try:

title = await item.inner\_text()

href = await item.get\_attribute("href")

\# Ensure it's a valid URL

if href and href.startswith("http"):

urls.append({"title": title.strip(), "url": href})

except Exception as e:

print(f"\[!\] Failed to parse link: {e}")

continue

if not urls:

print("❌ No links found.")

await browser.close()

return

print(f"🔗 Visiting {len(urls)} pages...")

scraped = \[\]

for idx, link in enumerate(urls):

print(f"\\n📄 \[{idx+1}\] {link\['title'\]}")

try:

await page.goto(link\["url"\], timeout=30000)

await asyncio.sleep(random.uniform(\*self.throttle))

content = await page.text\_content("body")

scraped.append({

"title": link\["title"\],

"url": link\["url"\],

"content": content\[:1500\] # Limit content

})

except Exception as e:

print(f"\[!\] Failed to scrape: {link\['url'\]}\\nReason: {e}")

continue

await browser.close()

self.save\_data(topic, scraped)

def save\_data(self, topic: str, data: list):

filename\_json = SAVE\_DIR / f"{topic.replace(' ', '\_')}\_data.json"

filename\_csv = SAVE\_DIR / f"{topic.replace(' ', '\_')}\_data.csv"

\# Save as JSON

with open(filename\_json, "w", encoding="utf-8") as f:

json.dump(data, f, ensure\_ascii=False, indent=2)

\# Save as CSV

with open(filename\_csv, "w", newline="", encoding="utf-8") as f:

writer = csv.DictWriter(f, fieldnames=\["title", "url", "content"\])

writer.writeheader()

for entry in data:

writer.writerow(entry)

print(f"\\n✅ Saved {len(data)} entries to:\\n- {filename\_json}\\n- {filename\_csv}")

def main():

topic = input("🔎 Enter topic to crawl web for data: ").strip()

if not topic:

print("❌ No topic entered.")

return

scraper = FirefoxSmartScraper()

asyncio.run(scraper.search\_and\_scrape(topic))

if \_\_name\_\_ == "\_\_main\_\_":

main()

this is my code in making an overall web scrapping.. i don't know whats wrong, it doesn't fetch data in the internet. or maybe websites are really protected