Merge pull request #34 from ScrapeGraphAI/removed-render_heavy_ks

feat: remove render_heavy_js

Author: VinciGit00

api-reference/endpoint/smartcrawler/start.mdx

@@ -36,7 +36,6 @@ Content-Type: `application/json`
     "same_domain": "boolean"
   },
   "sitemap": "boolean",
-  "render_heavy_js": "boolean",
   "stealth": "boolean"
   "webhook_url": str
 }
@@ -58,7 +57,6 @@ Content-Type: `application/json`
 | schema | object | No | - | JSON Schema object for structured output |
 | rules | object | No | - | Crawl rules for filtering URLs. Object with optional fields: `exclude` (array of regex URL patterns), `include_paths` (array of path patterns to include, supports wildcards `*` and `**`), `exclude_paths` (array of path patterns to exclude, takes precedence over `include_paths`), `same_domain` (boolean, default: true). See Rules section below for details. |
 | sitemap | boolean | No | false | Use sitemap.xml for discovery |
-| render_heavy_js | boolean | No | false | Enable heavy JavaScript rendering |
 | stealth | boolean | No | false | Enable stealth mode to bypass bot protection using advanced anti-detection techniques. Adds +4 credits to the request cost |
 | webhook_url | str | No | None | Webhook URL to send the job result to. When provided, a signed webhook notification will be sent upon job completion. See [Webhook Signature Verification](#webhook-signature-verification) below.
 

api-reference/endpoint/smartscraper/start.mdx

@@ -54,12 +54,6 @@ SmartScraper allows you to extract specific information from any webpage using A
   Range: 0-50
 </ParamField>
 
-<ParamField body="render_heavy_js" type="boolean">
-  Optional parameter to enable enhanced JavaScript rendering for heavy JS websites (React, Vue, Angular, SPAs). Use when standard rendering doesn't capture all content.
-
-  Default: false
-</ParamField>
-
 <ParamField body="mock" type="boolean">
   Optional parameter to enable mock mode. When set to true, the request will return mock data instead of performing an actual extraction. Useful for testing and development.
 
@@ -117,7 +111,6 @@ curl -X POST 'https://api.scrapegraphai.com/v1/smartscraper' \
   "user_prompt": "Extract all the headlines from this section into a table with the date and URL of the news",
   "total_pages": 2,
   "stealth": true,
-  "render_heavy_js": true,
   "headers": {
     "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
     "Cookie": "cookie1=value1; cookie2=value2"

integrations/x402.mdx

@@ -68,7 +68,6 @@ curl -X POST 'https://x402.orth.sh/scrapegraph/v1/scrape' \
   -H 'X-Payment: {{paymentHeader}}' \
   -d '{
     "website_url": "example",
-    "render_heavy_js": true,
     "branding": true,
     "stealth": "example"
   }'
@@ -102,7 +101,6 @@ curl -X POST 'https://x402.orth.sh/scrapegraph/v1/crawl' \
     "schema": "",
     "rules": "",
     "sitemap": "example",
-    "render_heavy_js": "example",
     "stealth": "example"
   }'
 ```
@@ -133,7 +131,6 @@ curl -X POST 'https://x402.orth.sh/scrapegraph/v1/smartscraper' \
     " website_markdown": "example",
     "total_pages": 123,
     " number_of_scrolls": 123,
-    " render_heavy_js": true,
     " mock": true,
     " cookies": "",
     " steps": ""

sdks/javascript.mdx

@@ -94,7 +94,6 @@ const response = await smartScraper(
 | websiteUrl      | string  | Yes      | The URL of the webpage that needs to be scraped.                                    |
 | prompt          | string  | Yes      | A textual description of what you want to achieve.                                  |
 | schema          | object  | No       | The Pydantic or Zod object that describes the structure and format of the response. |
-| renderHeavyJs   | boolean | No       | Enable enhanced JavaScript rendering for heavy JS websites (React, Vue, Angular, etc.). Default: false |
 
 <Accordion title="Basic Schema Example" icon="code">
 Define a simple schema using Zod:
@@ -201,8 +200,7 @@ try {
     apiKey,
     'https://example-react-store.com/products/123',
     'Extract product details including name, price, description, and availability',
-    ProductSchema,
-    true  // Enable render_heavy_js for JavaScript-heavy sites
+    ProductSchema
   );
 
   console.log('Product:', response.result.name);
@@ -214,13 +212,6 @@ try {
 }
 ```
 
-**When to use `renderHeavyJs`:**
-- React, Vue, or Angular applications
-- Single Page Applications (SPAs)
-- Sites with heavy client-side rendering
-- Dynamic content loaded via JavaScript
-- Interactive elements that depend on JavaScript execution
-
 </Accordion>
 
 ### SearchScraper

sdks/mocking.mdx

@@ -422,7 +422,7 @@ async function basicMockUsage() {
 
   try {
     // Test scrape endpoint
-    const scrapeResult = await scrape(API_KEY, 'https://example.com', { renderHeavyJs: true });
+    const scrapeResult = await scrape(API_KEY, 'https://example.com');
     console.log('Scrape result:', scrapeResult);
 
     // Test smartScraper endpoint

sdks/python.mdx

@@ -69,7 +69,6 @@ response = client.smartscraper(
 | website_url      | string  | Yes      | The URL of the webpage that needs to be scraped.                                    |
 | user_prompt      | string  | Yes      | A textual description of what you want to achieve.                                  |
 | output_schema    | object  | No       | The Pydantic object that describes the structure and format of the response.        |
-| render_heavy_js  | boolean | No       | Enable enhanced JavaScript rendering for heavy JS websites (React, Vue, Angular, etc.). Default: False |
 
 <Accordion title="Basic Schema Example" icon="code">
 Define a simple schema for basic data extraction:
@@ -142,43 +141,6 @@ for office in response.offices:
 ```
 </Accordion>
 
-<Accordion title="Enhanced JavaScript Rendering Example" icon="code">
-For modern web applications built with React, Vue, Angular, or other JavaScript frameworks:
-
-```python
-from scrapegraph_py import Client
-from pydantic import BaseModel, Field
-
-class ProductInfo(BaseModel):
-    name: str = Field(description="Product name")
-    price: str = Field(description="Product price")
-    description: str = Field(description="Product description")
-    availability: str = Field(description="Product availability status")
-
-client = Client(api_key="your-api-key")
-
-# Enable enhanced JavaScript rendering for a React-based e-commerce site
-response = client.smartscraper(
-    website_url="https://example-react-store.com/products/123",
-    user_prompt="Extract product details including name, price, description, and availability",
-    output_schema=ProductInfo,
-    render_heavy_js=True  # Enable for React/Vue/Angular sites
-)
-
-print(f"Product: {response['result']['name']}")
-print(f"Price: {response['result']['price']}")
-print(f"Available: {response['result']['availability']}")
-```
-
-**When to use `render_heavy_js`:**
-- React, Vue, or Angular applications
-- Single Page Applications (SPAs)
-- Sites with heavy client-side rendering
-- Dynamic content loaded via JavaScript
-- Interactive elements that depend on JavaScript execution
-
-</Accordion>
-
 ### SearchScraper
 
 Search and extract information from multiple web sources using AI:

services/additional-parameters/proxy.mdx

@@ -122,11 +122,6 @@ The following parameters in API requests can affect proxy behavior:
 - **Default**: No specific country (uses optimal routing)
 - **Format**: ISO 3166-1 alpha-2 (e.g., `us`, `gb`, `de`)
 
-### `render_heavy_js` (optional)
-- **Type**: Boolean
-- **Description**: Whether to render JavaScript-heavy pages. This may affect which proxy provider is used.
-- **Default**: `false`
-
 ## Usage Examples
 
 ### Basic Request (Automatic Proxy Selection)
@@ -204,46 +199,6 @@ const response = await smartScraper(
 
 </CodeGroup>
 
-### Request with JavaScript Rendering and Country Code
-
-<CodeGroup>
-
-```python Python
-from scrapegraph_py import Client
-
-client = Client(api_key="your-api-key")
-
-# Combine JavaScript rendering with geotargeting
-response = client.smartscraper(
-    website_url="https://example.com",
-    user_prompt="Extract product information",
-    render_heavy_js=True,
-    country_code="uk"
-)
-```
-
-```javascript JavaScript
-import { smartScraper } from 'scrapegraph-js';
-
-const apiKey = 'your-api-key';
-
-// Combine JavaScript rendering with geotargeting
-const response = await smartScraper(
-  apiKey,
-  'https://example.com',
-  'Extract product information',
-  null, // schema
-  null, // numberOfScrolls
-  null, // totalPages
-  null, // cookies
-  { country_code: 'uk' }, // options
-  false, // plain_text
-  true   // renderHeavyJs
-);
-```
-
-</CodeGroup>
-
 ### Real-World Use Cases
 
 #### Accessing Geo-Restricted Content
@@ -355,8 +310,7 @@ If your scraping request fails:
 1. **Verify the URL**: Make sure the URL is correct and accessible
 2. **Check the website**: Some websites may block automated access regardless of proxy
 3. **Retry the request**: The system uses automatic retries, but you can manually retry after a delay
-4. **Try different parameters**: Sometimes using `render_heavy_js: true` can help with JavaScript-heavy sites
-5. **Try a different country**: If geo-restriction is the issue, try a different `country_code`
+4. **Try a different country**: If geo-restriction is the issue, try a different `country_code`
 </Accordion>
 
 ### Rate Limiting

services/additional-parameters/wait-ms.mdx

@@ -150,25 +150,13 @@ async def scrape_with_wait():
 
 2. **Test incrementally** - If the default doesn't capture all content, try increasing in 1000ms increments (4000, 5000, etc.) rather than setting a very high value.
 
-3. **Combine with other parameters** - Use `wait_ms` together with `render_heavy_js` for JavaScript-heavy pages:
-
-```python
-response = client.smartscraper(
-    website_url="https://heavy-js-site.com",
-    user_prompt="Extract all products",
-    wait_ms=8000,
-    render_heavy_js=True
-)
-```
-
-4. **Balance speed and completeness** - Higher wait times ensure more content is captured but increase response time and resource usage.
+3. **Balance speed and completeness** - Higher wait times ensure more content is captured but increase response time and resource usage.
 
 ## Troubleshooting
 
 <Accordion title="Content still missing after increasing wait_ms" icon="exclamation-triangle">
 If increasing `wait_ms` doesn't capture all content:
 
-- Try enabling `render_heavy_js=True` for JavaScript-heavy pages
 - Check if the content requires user interaction (clicks, scrolls) - use `number_of_scrolls` for infinite scroll pages
 - Verify the content isn't behind authentication - use custom headers/cookies
 </Accordion>

services/mcp-server.mdx

@@ -182,8 +182,7 @@ AI‑powered extraction with optional infinite scrolls.
 smartscraper(
   user_prompt: str,
   website_url: str,
-  number_of_scrolls: int | None = None,
-  render_heavy_js: bool | None = None
+  number_of_scrolls: int | None = None
 )
 ```
 
@@ -199,10 +198,10 @@ searchscraper(
 ```
 
 ### 4. scrape
-Fetch raw HTML with optional heavy JS rendering.
+Fetch raw HTML from a URL.
 
 ```python
-scrape(website_url: str, render_heavy_js: bool | None = None)
+scrape(website_url: str)
 ```
 
 ### 5. sitemap

services/scrape.mdx

@@ -34,8 +34,7 @@ sgai_client = Client(api_key="your-api-key")
 # Scrape request
 response = sgai_client.htmlify(
     website_url="https://example.com",
-    render_heavy_js=False,  # Set to True for heavy JavaScript rendering
-    branding=True           # Set to True to extract brand design and metadata
+    branding=True  # Set to True to extract brand design and metadata
 )
 
 print("HTML Content:", response.html)
@@ -53,8 +52,7 @@ const apiKey = 'your-api-key';
 const url = 'https://example.com';
 
 try {
-  // htmlify(apiKey, url, renderHeavyJs = false, options = { branding: true })
-  const response = await htmlify(apiKey, url, false, { branding: true }); // enable branding extraction
+  const response = await htmlify(apiKey, url, { branding: true }); // enable branding extraction
   console.log('HTML Content:', response.html);
   console.log('Request ID:', response.scrape_request_id);
   console.log('Status:', response.status);
@@ -72,7 +70,6 @@ curl -X POST https://api.scrapegraphai.com/v1/scrape \
   -H "SGAI-APIKEY: your-api-key" \
   -d '{
     "website_url": "https://example.com",
-    "render_heavy_js": false,
     "branding": true
   }'
 ```
@@ -85,7 +82,6 @@ curl -X POST https://api.scrapegraphai.com/v1/scrape \
 |-----------|------|----------|-------------|
 | apiKey | string | Yes | The ScrapeGraph API Key. |
 | website_url | string | Yes | The URL of the webpage to scrape. |
-| render_heavy_js | boolean | No | Set to true for heavy JavaScript rendering. Default: false |
 | branding | boolean | No | Return extracted brand design and metadata. Default: false |
 | stealth | boolean | No | Enable stealth mode for anti-bot protection. Adds additional credits. Default: false |
 
@@ -171,9 +167,6 @@ When `branding=true` is passed, the response includes a `branding` object with b
   <Card title="Raw HTML Access" icon="code">
     Get complete HTML structure including all elements
   </Card>
-  <Card title="JavaScript Rendering" icon="bolt">
-    Optional support for heavy JavaScript rendering
-  </Card>
   <Card title="Branding Extraction" icon="palette">
     Optionally extract brand colors, fonts, typography, UI components, images, and metadata
   </Card>
@@ -209,24 +202,6 @@ When `branding=true` is passed, the response includes a `branding` object with b
 Want to learn more about our AI-powered scraping technology? Visit our [main website](https://scrapegraphai.com) to discover how we're revolutionizing web data extraction.
 </Note>
 
-## JavaScript Rendering
-
-The `render_heavy_js` parameter controls whether JavaScript should be executed on the target page:
-
-### When to Use JavaScript Rendering
-
-- **Single Page Applications (SPAs)**: React, Vue, Angular apps
-- **Dynamic Content**: Content loaded via AJAX/fetch
-- **Interactive Elements**: Dropdowns, modals, infinite scroll
-- **Client-side Routing**: Hash-based or history API routing
-
-### When to Skip JavaScript Rendering
-
-- **Static HTML Pages**: Traditional server-rendered content
-- **Performance**: Faster processing for simple pages
-- **Cost Optimization**: Lower API usage for basic scraping
-- **Reliability**: More predictable results for static content
-
 ## Advanced Usage
 
 ### Async Support
@@ -240,8 +215,7 @@ import asyncio
 async def main():
     async with AsyncClient(api_key="your-api-key") as client:
         response = await client.htmlify(
-            website_url="https://example.com",
-            render_heavy_js=True
+            website_url="https://example.com"
         )
         print(response)
 
@@ -271,7 +245,7 @@ async def main():
         "https://github.com/ScrapeGraphAI/Scrapegraph-ai",
     ]
 
-    tasks = [sgai_client.htmlify(website_url=url, render_heavy_js=False) for url in urls]
+    tasks = [sgai_client.htmlify(website_url=url) for url in urls]
 
     # Execute requests concurrently
     responses = await asyncio.gather(*tasks, return_exceptions=True)
@@ -304,8 +278,7 @@ if __name__ == "__main__":
 ## Best Practices
 
 ### Performance Optimization
-1. Use `render_heavy_js=false` for static content
-2. Process multiple URLs concurrently
+1. Process multiple URLs concurrently
 3. Cache results when possible
 4. Monitor API usage and costs