Shifterと一緒に使用する Python

import requests

PROXY_USER = "customer-USERNAME-country-us-sid-123ABC"
PROXY_PASS = "PASSWORD"
PROXY_HOST = "p.shifter.io"
PROXY_PORT = "443"

proxies = {
    "http": f"http://{PROXY_USER}:{PROXY_PASS}@{PROXY_HOST}:{PROXY_PORT}",
    "https": f"http://{PROXY_USER}:{PROXY_PASS}@{PROXY_HOST}:{PROXY_PORT}",
}

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
}

response = requests.get(
    "https://example.com",
    proxies=proxies,
    headers=headers,
    timeout=30,
)

print(f"Status: {response.status_code}")
print(f"Content length: {len(response.text)}")

import aiohttp
import asyncio
import uuid

# sid pins the same IP for every request that shares this session id;
# ttl-300 keeps that IP for up to 300 seconds, then rotates.
session_id = uuid.uuid4().hex[:8]
PROXY_URL = (
    f"customer-USERNAME-country-us-sid-{session_id}-ttl-300:"
    f"PASSWORD@p.shifter.io:443"
)

async def fetch(session, url):
    async with session.get(
        url, proxy=PROXY_URL, timeout=aiohttp.ClientTimeout(total=30)
    ) as response:
        return await response.text()

async def main():
    urls = [
        "https://example.com/login",
        "https://example.com/dashboard",
        "https://example.com/orders",
    ]

    async with aiohttp.ClientSession() as session:
        tasks = [fetch(session, url) for url in urls]
        results = await asyncio.gather(*tasks, return_exceptions=True)

        for url, result in zip(urls, results):
            if isinstance(result, Exception):
                print(f"Error fetching {url}: {result}")
            else:
                print(f"Fetched {url}: {len(result)} bytes")

asyncio.run(main())

# middlewares.py
class ShifterProxyMiddleware:
    PROXY_USER = "customer-USERNAME-country-uk-sid-456DEF"
    PROXY_PASS = "PASSWORD"
    PROXY_HOST = "p.shifter.io"
    PROXY_PORT = "443"

    def process_request(self, request, spider):
        request.meta["proxy"] = (
            f"http://{self.PROXY_USER}:{self.PROXY_PASS}"
            f"@{self.PROXY_HOST}:{self.PROXY_PORT}"
        )

# settings.py
DOWNLOADER_MIDDLEWARES = {
    "myproject.middlewares.ShifterProxyMiddleware": 350,
}

# spider.py
import scrapy

class ProductSpider(scrapy.Spider):
    name = "products"
    start_urls = ["https://example.co.uk/products"]

    def parse(self, response):
        for product in response.css(".product-card"):
            yield {
                "title": product.css("h2::text").get(),
                "price": product.css(".price::text").get(),
                "url": product.css("a::attr(href)").get(),
            }

        next_page = response.css("a.next-page::attr(href)").get()
        if next_page:
            yield response.follow(next_page, self.parse)

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

PROXY_HOST = "p.shifter.io"
PROXY_PORT = "443"
PROXY_USER = "customer-USERNAME-country-us-city-newyork-sid-789GHI"
PROXY_PASS = "PASSWORD"

chrome_options = Options()
chrome_options.add_argument(
    f"--proxy-server=http://{PROXY_HOST}:{PROXY_PORT}"
)

driver = webdriver.Chrome(options=chrome_options)

# Handle proxy authentication via browser extension or seleniumwire if needed
driver.get("https://example.com")

# Wait for dynamic content to load
wait = WebDriverWait(driver, 10)
element = wait.until(
    EC.presence_of_element_located((By.CSS_SELECTOR, ".content"))
)

print(f"Page title: {driver.title}")
print(f"Content: {element.text[:200]}")

driver.quit()