urllib.request.Request

The Request class represents an HTTP request with URL, data, headers, and other configuration. It's the primary way to create customized HTTP requests using urllib.

Class Overview

class Request:
    """
    Represents an HTTP request.
    
    This class encapsulates all the information needed to make an HTTP request,
    including the URL, optional data for POST requests, headers, and metadata.
    """

Constructor

Request(url, data=None, headers={}, origin_req_host=None, unverifiable=False, method=None)

Parameters

Parameter	Type	Default	Description
url	str	Required	The URL to request
data	bytes or None	None	Data to send in request body (makes it a POST)
headers	dict	{}	Dictionary of HTTP headers
origin_req_host	str or None	None	Original request host for cookie policy
unverifiable	bool	False	Whether request is unverifiable (for cookies)
method	str or None	None	HTTP method (GET, POST, PUT, DELETE, etc.)

Basic Usage

Simple GET Request

import urllib.request

# Create a basic GET request
req = urllib.request.Request('https://httpbin.org/get')

# Open the request
with urllib.request.urlopen(req) as response:
    data = response.read().decode('utf-8')
    print(data)

POST Request with Data

import urllib.request
import urllib.parse

# Prepare POST data
post_data = urllib.parse.urlencode({
    'name': 'John Doe',
    'email': 'john@example.com'
}).encode('utf-8')

# Create POST request
req = urllib.request.Request('https://httpbin.org/post', data=post_data)

with urllib.request.urlopen(req) as response:
    result = response.read().decode('utf-8')
    print(result)

Request with Custom Headers

import urllib.request

# Create request with custom headers
req = urllib.request.Request('https://api.github.com/user')
req.add_header('User-Agent', 'MyApp/1.0')
req.add_header('Accept', 'application/json')
req.add_header('Authorization', 'token your-token-here')

with urllib.request.urlopen(req) as response:
    user_data = response.read().decode('utf-8')
    print(user_data)

Request Methods and Properties

Header Management Methods

Method	Description	Example
add_header(key, val)	Add a header to the request	req.add_header('Content-Type', 'application/json')
add_unredirected_header(key, val)	Add header that won't be sent on redirects	req.add_unredirected_header('Authorization', 'Bearer token')
get_header(header_name, default=None)	Get header value	content_type = req.get_header('Content-Type')
has_header(header_name)	Check if header exists	if req.has_header('Authorization'):
remove_header(header_name)	Remove a header	req.remove_header('User-Agent')
header_items()	Get all headers as list of tuples	headers = req.header_items()

Request Properties

Property	Type	Description	Example
full_url	str	Complete URL of the request	print(req.full_url)
data	bytes or None	Request body data	body = req.data
get_method()	str	HTTP method (GET, POST, etc.)	method = req.get_method()
get_full_url()	str	Same as full_url property	url = req.get_full_url()

Proxy Configuration

Method	Description	Example
set_proxy(host, type)	Set proxy for the request	req.set_proxy('proxy.example.com:8080', 'http')
has_proxy()	Check if request has proxy	if req.has_proxy():

Primary Use Cases

1. REST API Client

import urllib.request
import json

class APIClient:
    def __init__(self, base_url, api_key):
        self.base_url = base_url
        self.api_key = api_key
    
    def get(self, endpoint):
        url = f"{self.base_url}/{endpoint}"
        req = urllib.request.Request(url)
        req.add_header('Authorization', f'Bearer {self.api_key}')
        req.add_header('Accept', 'application/json')
        
        with urllib.request.urlopen(req) as response:
            return json.loads(response.read().decode('utf-8'))
    
    def post(self, endpoint, data):
        url = f"{self.base_url}/{endpoint}"
        json_data = json.dumps(data).encode('utf-8')
        
        req = urllib.request.Request(url, data=json_data)
        req.add_header('Authorization', f'Bearer {self.api_key}')
        req.add_header('Content-Type', 'application/json')
        req.add_header('Accept', 'application/json')
        
        with urllib.request.urlopen(req) as response:
            return json.loads(response.read().decode('utf-8'))

# Usage
client = APIClient('https://api.example.com', 'your-api-key')
users = client.get('users')
new_user = client.post('users', {'name': 'Alice', 'email': 'alice@example.com'})

2. Web Scraping with Custom Headers

import urllib.request
import time
import random

class WebScraper:
    def __init__(self):
        self.user_agents = [
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36'
        ]
    
    def scrape_page(self, url, delay=1):
        # Random user agent to avoid detection
        user_agent = random.choice(self.user_agents)
        
        req = urllib.request.Request(url)
        req.add_header('User-Agent', user_agent)
        req.add_header('Accept', 'text/html,application/xhtml+xml')
        req.add_header('Accept-Language', 'en-US,en;q=0.9')
        req.add_header('Accept-Encoding', 'gzip, deflate')
        req.add_header('Connection', 'keep-alive')
        
        try:
            with urllib.request.urlopen(req) as response:
                content = response.read().decode('utf-8')
                time.sleep(delay)  # Be respectful to servers
                return content
        except urllib.error.HTTPError as e:
            print(f"HTTP Error {e.code}: {e.reason}")
            return None

# Usage
scraper = WebScraper()
html = scraper.scrape_page('https://example.com')

3. File Upload with Multipart Data

import urllib.request
import mimetypes
import os

def upload_file(url, file_path, field_name='file'):
    """Upload a file using multipart/form-data encoding."""
    
    # Read file content
    with open(file_path, 'rb') as f:
        file_content = f.read()
    
    # Get file info
    filename = os.path.basename(file_path)
    content_type, _ = mimetypes.guess_type(file_path)
    if content_type is None:
        content_type = 'application/octet-stream'
    
    # Create boundary
    boundary = '----WebKitFormBoundary7MA4YWxkTrZu0gW'
    
    # Build multipart data
    body = []
    body.append(f'--{boundary}'.encode())
    body.append(f'Content-Disposition: form-data; name="{field_name}"; filename="{filename}"'.encode())
    body.append(f'Content-Type: {content_type}'.encode())
    body.append(b'')
    body.append(file_content)
    body.append(f'--{boundary}--'.encode())
    
    body_bytes = b'\r\n'.join(body)
    
    # Create request
    req = urllib.request.Request(url, data=body_bytes)
    req.add_header('Content-Type', f'multipart/form-data; boundary={boundary}')
    req.add_header('Content-Length', str(len(body_bytes)))
    
    with urllib.request.urlopen(req) as response:
        return response.read().decode('utf-8')

# Usage
# result = upload_file('https://httpbin.org/post', '/path/to/file.txt')

4. HTTP Method Override

import urllib.request
import json

def make_request(url, method='GET', data=None, headers=None):
    """Make HTTP request with any method."""
    
    if headers is None:
        headers = {}
    
    # Prepare data
    if data and isinstance(data, dict):
        data = json.dumps(data).encode('utf-8')
        headers['Content-Type'] = 'application/json'
    
    # Create request
    req = urllib.request.Request(url, data=data, method=method)
    
    # Add headers
    for key, value in headers.items():
        req.add_header(key, value)
    
    try:
        with urllib.request.urlopen(req) as response:
            return {
                'status': response.getcode(),
                'headers': dict(response.headers),
                'data': response.read().decode('utf-8')
            }
    except urllib.error.HTTPError as e:
        return {
            'status': e.code,
            'error': e.reason,
            'data': e.read().decode('utf-8') if hasattr(e, 'read') else None
        }

# Usage examples
get_result = make_request('https://httpbin.org/get')
post_result = make_request('https://httpbin.org/post', 'POST', {'key': 'value'})
put_result = make_request('https://httpbin.org/put', 'PUT', {'updated': True})
delete_result = make_request('https://httpbin.org/delete', 'DELETE')

Common Errors and Troubleshooting

HTTP Method Determination

import urllib.request

# Method is automatically determined
req1 = urllib.request.Request('https://example.com')
print(req1.get_method())  # 'GET'

req2 = urllib.request.Request('https://example.com', data=b'test')
print(req2.get_method())  # 'POST'

# Explicit method override
req3 = urllib.request.Request('https://example.com', method='PUT')
print(req3.get_method())  # 'PUT'

Header Case Sensitivity

import urllib.request

req = urllib.request.Request('https://example.com')

# Headers are case-insensitive
req.add_header('content-type', 'application/json')
req.add_header('Content-Type', 'text/html')  # Overwrites previous

print(req.get_header('Content-Type'))  # 'text/html'
print(req.get_header('content-type'))  # 'text/html'

Data Encoding Issues

import urllib.request

# WRONG - string data
try:
    req = urllib.request.Request('https://httpbin.org/post', data='test string')
    # This will cause TypeError
except TypeError as e:
    print(f"Error: {e}")

# CORRECT - bytes data
data = 'test string'.encode('utf-8')
req = urllib.request.Request('https://httpbin.org/post', data=data)

Common Error Patterns

import urllib.request
import urllib.error

def safe_request(url, **kwargs):
    """Make a request with comprehensive error handling."""
    try:
        req = urllib.request.Request(url, **kwargs)
        with urllib.request.urlopen(req) as response:
            return response.read().decode('utf-8')
            
    except urllib.error.HTTPError as e:
        print(f"HTTP {e.code}: {e.reason}")
        # Server returned error response
        if hasattr(e, 'read'):
            error_body = e.read().decode('utf-8')
            print(f"Error body: {error_body}")
        return None
        
    except urllib.error.URLError as e:
        print(f"URL Error: {e.reason}")
        # Network problem (DNS, connection refused, etc.)
        return None
        
    except UnicodeDecodeError as e:
        print(f"Encoding error: {e}")
        # Response encoding issue
        return None
        
    except Exception as e:
        print(f"Unexpected error: {e}")
        return None

Performance Considerations

Memory Usage with Large Requests

import urllib.request

# For large data, consider streaming
def upload_large_file(url, file_path):
    with open(file_path, 'rb') as f:
        # Read in chunks instead of loading entire file
        req = urllib.request.Request(url, data=f)
        req.add_header('Content-Type', 'application/octet-stream')
        
        with urllib.request.urlopen(req) as response:
            return response.read()

Header Efficiency

import urllib.request

# Efficient header management
def create_api_request(url, headers_dict):
    req = urllib.request.Request(url)
    
    # Add multiple headers efficiently
    for key, value in headers_dict.items():
        req.add_header(key, value)
    
    return req

# Pre-defined header templates
API_HEADERS = {
    'User-Agent': 'MyApp/1.0',
    'Accept': 'application/json',
    'Accept-Encoding': 'gzip, deflate'
}

req = create_api_request('https://api.example.com', API_HEADERS)

When to Use Request Class

Ideal Use Cases

• Custom HTTP headers required
• Authentication headers needed
• Specific HTTP methods (PUT, DELETE, PATCH)
• Proxy configuration required
• Fine-grained control over request creation
• Building reusable request templates
• Integration with urllib ecosystem

When NOT to Use Request Class

• Simple GET requests → Use urllib.request.urlopen(url) directly
• Complex session management → Use requests.Session
• Async requests → Use aiohttp or httpx
• File uploads with complex multipart → Use requests library
• OAuth or complex authentication → Use specialized libraries

Alternative Approaches

# Simple GET - no Request needed
import urllib.request
response = urllib.request.urlopen('https://example.com')

# Using requests library for complex scenarios
import requests
response = requests.get('https://example.com', headers={'User-Agent': 'MyApp'})

Related Classes

• urllib.request.OpenerDirector - Manages request handlers
• urllib.error.HTTPError - HTTP error responses
• urllib.error.URLError - URL-related errors

Additional Learning Resources

Official Python Resources

• urllib.request Documentation
• Python urllib HOW TO
• HTTP/1.1 Method Definitions

Examples and Tutorials

• Real Python urllib Guide
• HTTP Headers Reference
• REST API Best Practices