🔍 LLM Scraper Playground

\n Enter a web page URL and a prompt describing what you want to extract from it.
\n The LLM will scrape, analyze, and extract results for you!\n

\n \n\n

\n \n \n\n\n\n// Main interactive logic for LLM Scraper Playground, now with Websim AI support and web search!\n\n// @tweakable The maximum number of characters to include from the fetched page when context is trimmed\nconst MAX_CONTEXT_LENGTH = 16000;\n\n/* @tweakable Should \"web search\" results be prefetched and appended to the LLM context? */\nconst ENABLE_WEB_SEARCH = true;\n\n/* @tweakable The number of search engine results to fetch for web search mode */\nconst NUM_SEARCH_RESULTS = 5;\n\n/* @tweakable The search engine to use: 'bing', 'google', 'duckduckgo' */\nconst SEARCH_ENGINE = \"bing\";\n\n/* @tweakable Should LLM always return results in CSV unless the prompt requests otherwise? */\nconst FORCE_CSV_OUTPUT = true;\n\n/* @tweakable The default CSV delimiter (comma, tab, semicolon, etc) */\nconst CSV_DELIMITER = \",\";\n\n/* @tweakable Should CSV results be downloadable as a file by default? */\nconst CSV_DOWNLOAD_ENABLED = true;\n\n// UI Elements\nconst form = document.getElementById('scrapeForm');\nconst statusDiv = document.getElementById('status');\nconst resultDiv = document.getElementById('result');\nconst urlInput = document.getElementById('urlInput');\nconst promptInput = document.getElementById('promptInput');\n\n// Add Web Search support to the UI (new feature)\nlet searchToggle, searchQueryInput;\nfunction injectSearchControls() {\n if (document.getElementById('searchToggle')) return; // Already present\n\n const searchBox = document.createElement('div');\n searchBox.id = \"websearch-controls\";\n searchBox.style = \"margin-bottom: 1em;display:flex;align-items:baseline;gap:1em;flex-wrap:wrap\";\n \n searchToggle = document.createElement('input');\n searchToggle.type = 'checkbox';\n searchToggle.id = 'searchToggle';\n searchToggle.checked = false;\n searchToggle.style.marginRight = \"0.5em\";\n // @tweakable Should web search be the default mode on? Set `searchToggle.checked` default.\n \n const searchLabel = document.createElement('label');\n searchLabel.textContent = \"Web Search (Enter query below, LLM will use SERP context)\";\n searchLabel.style = \"margin-right:1em\";\n searchLabel.htmlFor = 'searchToggle';\n\n searchQueryInput = document.createElement('input');\n searchQueryInput.type = 'text';\n searchQueryInput.id = 'searchQueryInput';\n searchQueryInput.placeholder = \"Optional: e.g. 'latest OpenAI news'\";\n searchQueryInput.style = \"width:300px;max-width:90%\";\n searchQueryInput.disabled = true;\n\n // Enable/disable query box if using search\n searchToggle.onchange = () => {\n searchQueryInput.disabled = !searchToggle.checked;\n urlInput.disabled = searchToggle.checked;\n if (searchToggle.checked) {\n urlInput.placeholder = \"Disabled in Web Search mode\";\n } else {\n urlInput.placeholder = \"https://example.com\";\n }\n };\n \n searchBox.appendChild(searchToggle);\n searchBox.appendChild(searchLabel);\n searchBox.appendChild(searchQueryInput);\n\n const formElem = document.getElementById('scrapeForm');\n formElem.insertBefore(searchBox, formElem.firstChild);\n}\ninjectSearchControls();\n\n// Dark mode toggle\nconst toggleDark = document.getElementById('toggleDark');\ntoggleDark.onclick = () => {\n document.body.classList.toggle('dark');\n};\n\n// Helper functions\nfunction setStatus(msg) {\n statusDiv.textContent = msg;\n}\nfunction clearStatus() {\n statusDiv.textContent = '';\n}\nfunction showResult(html) {\n resultDiv.innerHTML = html;\n}\n\nfunction renderCSV(csv, displayHtml = true, filename = \"results.csv\") {\n // Download link/button if desired\n let btn = \"\";\n if (CSV_DOWNLOAD_ENABLED) {\n const blob = new Blob([csv], { type: \"text/csv\" });\n const url = URL.createObjectURL(blob);\n btn = `Download CSV
`;\n }\n // Show as

 as well\n  const prettied = `${escapeHtml(csv)}
`;\n  return btn + (displayHtml ? prettied : \"\");\n}\n\nfunction escapeHtml(s) {\n  return String(s)\n    .replace(/&/g, \"&\")\n    .replace(//g, \">\");\n}\n\n// WebSim AI Web Search Helper\nasync function fetchWebSearch(query, {engine = SEARCH_ENGINE, numResults = NUM_SEARCH_RESULTS} = {}) {\n  setStatus(\"Running web search...\");\n  // Powered by websim LLM: Let the LLM search for you\n  // Use websim.chat.completions.create to emulate a smart API search/summary wrapper\n  // (In reality, a proper integration would call a real-time search API,\n  // but here we let the LLM synthesize result-like context.)\n\n  // @tweakable The LLM \"persona\" for web search (controls how results are summarized/surfaced)\n  const WEB_SEARCH_PERSONA = \"You are a helpful web search assistant. Given a search query, simulate search results like Bing/Google with links, titles, and 1-2 sentence snippets. Use current knowledge up to 2024. Include each result as:\\n\\n1. [Title](URL)\\nSnippet.\\n\";\n  const queryPrompt = `Simulate top ${numResults} ` + \n    engine + ` search results for the query: \"${query}\". \nReturn a markdown-formatted list, each result as:\n\\n1. [Page Title](URL)\nSnippet (1-2 sentences summarizing relevance).\n\nInclude only relevant and credible links. If nothing found, say \"No relevant results found.\"`;\n\n  const completion = await websim.chat.completions.create({\n    messages: [\n      { role: \"system\", content: [{type: \"text\", text: WEB_SEARCH_PERSONA}] },\n      { role: \"user\", content: [{type:\"text\", text: queryPrompt }] }\n    ]\n  });\n\n  return completion.content;\n}\n\n// Fetch webpage text helper (unchanged)\nasync function fetchPageText(url) {\n  setStatus(\"Fetching page content...\");\n  try {\n    const res = await fetch('https://api.allorigins.win/get?url=' + encodeURIComponent(url));\n    if (!res.ok) throw new Error('Failed to fetch page');\n    const data = await res.json();\n    // Get text from HTML body, stripping scripts/styles\n    const parser = new DOMParser();\n    const doc = parser.parseFromString(data.contents, \"text/html\");\n    [...doc.querySelectorAll('script, style, noscript')].forEach(e => e.remove());\n    let text = doc.body ? doc.body.innerText : doc.textContent;\n    if (!text) text = data.contents;\n    if (text.length > MAX_CONTEXT_LENGTH) text = text.slice(0, MAX_CONTEXT_LENGTH);\n    return text;\n  } catch (e) {\n    setStatus(\"Error fetching or parsing page: \" + e);\n    return null;\n  }\n}\n\n// Helper: linkify URLs/emails in result\nfunction linkify(text) {\n  const urlRegex = /(https?:\\/\\/[^\\s]+)/g;\n  const emailRegex = /([a-zA-Z0-9.-_]+@[a-zA-Z0-9.-_]+\\.[a-zA-Z]{2,})/g;\n  return text\n    .replace(urlRegex, '$1')\n    .replace(emailRegex, '$1');\n}\n\n// Main LLM call: normal extraction OR web search\nasync function queryLLMScraper({pageUrl, contextText, userPrompt, searchContext}) {\n  setStatus(\"Asking the LLM...\");\n  // Compose prompt depending on mode\n  \n  // This value is set when we want to force CSV formatting regardless of prompt\n  let csvDirectives = \"\";\n  if (FORCE_CSV_OUTPUT) {\n    // Only ask for CSV if the prompt doesn't already mention CSV or JSON\n    const lowered = userPrompt.toLowerCase();\n    if (!/csv|json|spreadsheet/i.test(lowered)) {\n      csvDirectives = `Please output the result as CSV (comma-separated values), using \"${CSV_DELIMITER}\" as the delimiter. \nUse the first row for column headers, and then one row per item. Do not include explanations or any other text.`;\n    }\n  }\n\n  let messages;\n  if (searchContext) {\n    // Web search mode!\n    /* @tweakable The system prompt for extractions with web search context */\n    const searchSysPrompt =\n`You are a smart web search information extractor. Given simulated search results and a user request, only answer the direct question or extract what is asked for.\n${csvDirectives}\nDo NOT repeat prompt, result list, or narrate. Answer in user's requested format (CSV/plain/JSON).`;\n\n    messages = [\n      {\n        role: \"system\",\n        content: [{ type: \"text\", text: searchSysPrompt }]\n      },\n      {\n        role: \"user\",\n        content: [\n          { type: \"text\", text:\n`Search results for context (extracted, for AI comprehension):\n\n${searchContext}\n\n----\nUser's instructions: ${userPrompt}\n${csvDirectives}\nJust answer the question, no extra text.`\n          }\n        ]\n      }\n    ];\n  } else {\n    /* @tweakable The system prompt for direct page extraction (csv directives auto-appended when FORCE_CSV_OUTPUT is ON) */\n    const pageSysPrompt =\n`You are a web page content extraction assistant. Given a user's request and the page content, output only the answer, without explanations or preamble.\n${csvDirectives}\nThe extraction instructions from the user may ask for summary, lists, or specific data (like emails).\nRespond with CSV (comma-separated values) if the user's prompt requests CSV, JSON if they request JSON, otherwise use CSV by default.`;\n\n    messages = [\n      {\n        role: \"system\",\n        content: [{\n          type: \"text\",\n          text: pageSysPrompt\n        }]\n      },\n      {\n        role: \"user\",\n        content: [{\n          type: \"text\",\n          text:\n`Page URL: ${pageUrl}\n\nPage content (truncated to fit budget):\n\n${contextText}\n\n[END OF TEXT]\n\nInstructions: ${userPrompt}\n${csvDirectives}\nRespond with ONLY the answer, no explanation, no repetition.`\n        }]\n      }\n    ];\n  }\n  setStatus(\"Waiting for LLM...\");\n  const completion = await websim.chat.completions.create({ messages });\n  return completion.content;\n}\n\n// Enhanced main submit handler\nform.onsubmit = async (e) => {\n  e.preventDefault();\n  showResult('');\n  setStatus(\"Starting...\");\n\n  const prompt = promptInput.value.trim();\n  const useWebSearch = searchToggle && searchToggle.checked;\n  let answer, resHtml;\n\n  if (useWebSearch) {\n    // Web Search Mode\n    const searchQuery = searchQueryInput.value.trim() || prompt;\n    if (!searchQuery) {\n      setStatus(\"Enter a search query or extraction prompt.\");\n      return;\n    }\n    setStatus(\"Fetching web search data...\");\n    let searchResults = await fetchWebSearch(searchQuery, {engine: SEARCH_ENGINE, numResults: NUM_SEARCH_RESULTS});\n    setStatus(\"Calling LLM with search context...\");\n    answer = await queryLLMScraper({\n      pageUrl: null,\n      contextText: null,\n      userPrompt: prompt,\n      searchContext: searchResults\n    });\n\n    clearStatus();\n    resHtml = '';\n\n    // Heuristically decide if answer is CSV\n    if (isCSV(answer)) {\n      resHtml = renderCSV(answer, true, \"websearch_results.csv\");\n    } else {\n      // Try JSON pretty-print first\n      try {\n        const asJSON = JSON.parse(answer);\n        resHtml = `${JSON.stringify(asJSON, null, 2)}
`;\n      } catch {\n        resHtml = `${linkify(answer)}
`;\n      }\n    }\n    // Optionally show the fetched context with a collapsible (hidden by default)\n    resHtml += `Show web search results context
${linkify(searchResults)}
`;\n    showResult(resHtml);\n    return;\n\n  } else {\n    // Original URL extraction mode\n    const url = urlInput.value.trim();\n    if (!url) {\n      setStatus(\"Please enter a valid URL.\");\n      return;\n    }\n    if (!prompt) {\n      setStatus(\"Please enter an extraction prompt.\");\n      return;\n    }\n    // 1. Fetch page contents\n    const ctxText = await fetchPageText(url);\n    if (!ctxText) return;\n    // 2. Call LLM for extraction\n    answer = await queryLLMScraper({\n      pageUrl: url,\n      contextText: ctxText,\n      userPrompt: prompt,\n      searchContext: null\n    });\n\n    clearStatus();\n\n    // Nice formatting depending on answer type\n    resHtml = '';\n    if (isCSV(answer)) {\n      resHtml = renderCSV(answer, true, \"extracted_results.csv\");\n    } else {\n      // Try JSON pretty-print\n      try {\n        const asJSON = JSON.parse(answer);\n        resHtml = `${JSON.stringify(asJSON, null, 2)}
`;\n      } catch {\n        resHtml = `${linkify(answer)}`;\n      }\n    }\n    showResult(resHtml);\n  }\n};\n\n// Heuristic to check if output is CSV\nfunction isCSV(ans) {\n  // Simple: If it has at least 1 newline, and the first line has delimiter and >1 row, treat as CSV\n  if (!ans || typeof ans !== \"string\") return false;\n  const lines = ans.trim().split(/\\r?\\n/);\n  if (lines.length >= 2 && lines[0].split(CSV_DELIMITER).length > 1) return true;\n  // Tab/semicolon CSV?\n  if (lines.length >= 2 && (lines[0].includes(\"\\t\") || lines[0].includes(\";\"))) return true;\n  return false;\n}","url","link_url","/p/c7w7nj3kghllzafhvamp","versioned_link_url","/p/c7w7nj3kghllzafhvamp/1","yapping","\n```index.html\n```\n\n```main.js\n```\n\n```logo.png\n```\n","page_description","cursor","eyJpZCI6ImM3dzduajNrZ2hsbHphZmh2YW1wIn0","token","included",[],"about",{"_183":-5},"parent","routes/browser/layout","actionData","errors"]

The LLM Intelligence Chronicle