← Back to Cheat Sheet

Python API & Requests

Working with APIs and making HTTP requests in Python

The Python requests library is a powerful tool for making HTTP requests to web services and APIs. It simplifies working with HTTP methods, handling responses, and managing authentication.

This guide covers the basics of making HTTP requests, working with APIs, handling responses, authentication methods, and best practices for API consumption.

Installation and Setup

The requests library is not part of the Python standard library, so you'll need to install it: 

pip install requests

# Import the library
import requests

Basic HTTP Requests

The requests library provides simple methods for different HTTP methods: 

import requests

# GET request
response = requests.get('https://api.example.com/data')

# POST request
response = requests.post('https://api.example.com/submit', data={'key': 'value'})

# PUT request
response = requests.put('https://api.example.com/update', data={'key': 'new_value'})

# DELETE request
response = requests.delete('https://api.example.com/delete')

# HEAD request (only get headers)
response = requests.head('https://api.example.com/data')

# OPTIONS request
response = requests.options('https://api.example.com')

URL Parameters

You can pass URL parameters (query string) in several ways: 

# Method 1: Add parameters directly to URL
response = requests.get('https://api.example.com/search?q=python&page=1')

# Method 2: Use the params parameter (recommended)
payload = {
    'q': 'python',
    'page': 1,
    'sort': 'relevance'
}
response = requests.get('https://api.example.com/search', params=payload)

# The URL becomes: https://api.example.com/search?q=python&page=1&sort=relevance
print(response.url)

Working with Responses

After making a request, you can access various properties of the response: 

import requests

response = requests.get('https://api.github.com/users/python')

# Status code
print(f"Status code: {response.status_code}")  # e.g., 200, 404, 500

# Check if request was successful
if response.status_code == 200:
    print("Success!")
elif response.status_code == 404:
    print("Not found!")

# Alternative way to check for success
if response.ok:  # True if status_code is less than 400
    print("Request was successful")

# Response content
print(response.text)  # Raw text response

# JSON response
data = response.json()  # Parse JSON response into a Python dictionary
print(data['name'])

# Response headers
print(response.headers)
print(response.headers['Content-Type'])

# Response encoding
print(response.encoding)

# Response cookies
print(response.cookies)

# Response URL (useful for redirects)
print(response.url)

# Response history (list of redirects)
print(response.history)

Error Handling

import requests

try:
    response = requests.get('https://api.example.com/data', timeout=5)
    
    # Raise an exception if response was unsuccessful
    response.raise_for_status()
    
    # Process the successful response
    data = response.json()
    print(data)
    
except requests.exceptions.HTTPError as errh:
    print(f"HTTP Error: {errh}")
except requests.exceptions.ConnectionError as errc:
    print(f"Connection Error: {errc}")
except requests.exceptions.Timeout as errt:
    print(f"Timeout Error: {errt}")
except requests.exceptions.RequestException as err:
    print(f"Request Error: {err}")
except ValueError as ve:
    print(f"JSON parsing failed: {ve}")

Sending Data

Form Data

# Send form data (application/x-www-form-urlencoded)
form_data = {
    'username': 'user123',
    'password': 'securepassword',
    'remember': True
}
response = requests.post('https://api.example.com/login', data=form_data)

# Send multipart/form-data (for file uploads)
files = {
    'profile_image': open('image.jpg', 'rb')
}
form_data = {
    'username': 'user123'
}
response = requests.post('https://api.example.com/upload', 
                         files=files, 
                         data=form_data)

JSON Data

# Send JSON data
json_data = {
    'name': 'John Doe',
    'age': 30,
    'hobbies': ['coding', 'reading'],
    'is_active': True
}
response = requests.post('https://api.example.com/users', json=json_data)

# Alternative: manually converting to JSON
import json
json_string = json.dumps(json_data)
headers = {'Content-Type': 'application/json'}
response = requests.post('https://api.example.com/users', 
                         data=json_string, 
                         headers=headers)

HTTP Headers

Customize HTTP headers for your requests: 

# Custom headers
headers = {
    'User-Agent': 'Python Requests Client/1.0',
    'Authorization': 'Bearer your_access_token',
    'Accept': 'application/json',
    'Content-Type': 'application/json',
    'X-Custom-Header': 'custom-value'
}

response = requests.get('https://api.example.com/data', headers=headers)

Authentication Methods

Basic Authentication

# Method 1: Using auth parameter
response = requests.get('https://api.example.com/private', 
                       auth=('username', 'password'))

# Method 2: Using the HTTPBasicAuth
from requests.auth import HTTPBasicAuth
response = requests.get('https://api.example.com/private',
                       auth=HTTPBasicAuth('username', 'password'))

Token Authentication

# Bearer token authentication
token = "your_access_token"
headers = {
    'Authorization': f'Bearer {token}'
}
response = requests.get('https://api.example.com/data', headers=headers)

# API key in query parameters
params = {
    'api_key': 'your_api_key',
    'q': 'search_term'
}
response = requests.get('https://api.example.com/search', params=params)

OAuth 2.0

For OAuth 2.0, you can use the requests-oauthlib library: 

# pip install requests-oauthlib
from requests_oauthlib import OAuth2Session

client_id = "your_client_id"
client_secret = "your_client_secret"
authorization_base_url = 'https://example.com/oauth/authorize'
token_url = 'https://example.com/oauth/token'
redirect_uri = 'https://your-app.com/callback'

# Create OAuth session
oauth = OAuth2Session(client_id, redirect_uri=redirect_uri)

# Authorization URL
authorization_url, state = oauth.authorization_url(authorization_base_url)
print(f'Please go to {authorization_url} and authorize access')

# Get authorization code from callback
authorization_response = input('Enter the full callback URL: ')

# Fetch token
token = oauth.fetch_token(token_url, 
                         authorization_response=authorization_response,
                         client_secret=client_secret)

# Use the token to access protected resources
response = oauth.get('https://example.com/api/resource')

Using Sessions

Sessions allow you to persist certain parameters across requests: 

import requests

# Create a session
session = requests.Session()

# Set persistent headers and cookies
session.headers.update({
    'User-Agent': 'Python Requests Client/1.0',
    'Accept': 'application/json'
})

# Session automatically manages cookies across requests
# Login and get cookies
session.post('https://api.example.com/login', 
            data={'username': 'user', 'password': 'pass'})

# All subsequent requests will include cookies and headers
response = session.get('https://api.example.com/dashboard')

# Add auth to the session
session.auth = ('username', 'password')

# Make authenticated requests
response = session.get('https://api.example.com/protected')

# Close the session when done
session.close()

Advanced Features

Timeouts

# Set timeout in seconds (connection timeout, read timeout)
response = requests.get('https://api.example.com/data', timeout=5)  # 5 seconds

# Different timeouts for connection and read
response = requests.get('https://api.example.com/data', timeout=(3, 10))  
# 3 seconds for connection, 10 seconds for reading

Streaming Responses

# For large files, use streaming to avoid loading the entire file into memory
with requests.get('https://example.com/large-file.zip', stream=True) as response:
    # Check if the request was successful
    response.raise_for_status()
    
    # Write response to file in chunks
    with open('large-file.zip', 'wb') as f:
        for chunk in response.iter_content(chunk_size=8192): 
            f.write(chunk)

Proxies

# Using proxies
proxies = {
    'http': 'http://10.10.10.10:8000',
    'https': 'http://10.10.10.10:8000',
}
response = requests.get('https://api.example.com/data', proxies=proxies)

# With authentication
proxies = {
    'http': 'http://user:pass@10.10.10.10:8000',
    'https': 'http://user:pass@10.10.10.10:8000',
}

SSL Verification

# Disable SSL verification (not recommended for production)
response = requests.get('https://api.example.com/data', verify=False)

# Specify a local cert
response = requests.get('https://api.example.com/data', 
                       verify='/path/to/certfile')

# Client-side certificates
response = requests.get('https://api.example.com/data', 
                       cert=('/path/to/client.cert', '/path/to/client.key'))

Best Practices

  • Always use timeouts to prevent your application from hanging indefinitely.
  • Handle errors properly using try/except blocks and response.raise_for_status().
  • Use sessions for multiple requests to the same site.
  • Secure sensitive data like API keys and tokens using environment variables or secure storage.
  • Respect rate limits by implementing proper backoff strategies.
  • Stream large responses to avoid memory issues.
  • Keep SSL verification enabled in production environments.
  • Add user agents to identify your application to the API provider.

Rate Limiting Example

import requests
import time
from requests.exceptions import RequestException

def make_api_request(url, max_retries=3, backoff_factor=1):
    """Make API request with exponential backoff retry logic"""
    retries = 0
    
    while retries <= max_retries:
        try:
            response = requests.get(url, timeout=10)
            
            # If we get a 429 Too Many Requests, back off and retry
            if response.status_code == 429:
                wait_time = backoff_factor * (2 ** retries)
                print(f"Rate limited. Retrying in {wait_time} seconds...")
                time.sleep(wait_time)
                retries += 1
                continue
                
            # For other status codes, raise for status as usual
            response.raise_for_status()
            return response
            
        except RequestException as e:
            print(f"Request failed: {e}")
            
            if retries >= max_retries:
                print("Maximum retries exceeded.")
                raise
                
            wait_time = backoff_factor * (2 ** retries)
            print(f"Retrying in {wait_time} seconds...")
            time.sleep(wait_time)
            retries += 1
    
    return None  # Should not reach here due to raise above

Real-World Examples

GitHub API

import requests

# Get user information from GitHub API
def get_github_user(username):
    url = f"https://api.github.com/users/{username}"
    headers = {
        'Accept': 'application/vnd.github.v3+json',
        'User-Agent': 'PythonRequestsExample/1.0'
    }
    
    response = requests.get(url, headers=headers)
    
    if response.status_code == 200:
        return response.json()
    elif response.status_code == 404:
        return {"error": "User not found"}
    else:
        response.raise_for_status()

# Get list of repositories for a user
def get_user_repos(username):
    url = f"https://api.github.com/users/{username}/repos"
    headers = {
        'Accept': 'application/vnd.github.v3+json',
        'User-Agent': 'PythonRequestsExample/1.0'
    }
    
    response = requests.get(url, headers=headers)
    response.raise_for_status()
    
    return response.json()

# Example usage
try:
    user = get_github_user('python')
    print(f"User: {user['name']}")
    print(f"Bio: {user['bio']}")
    print(f"Followers: {user['followers']}")
    
    repos = get_user_repos('python')
    print(f"\nTotal repos: {len(repos)}")
    
    print("\nTop 5 repositories:")
    for repo in sorted(repos, key=lambda r: r['stargazers_count'], reverse=True)[:5]:
        print(f"- {repo['name']}: {repo['stargazers_count']} stars")
        
except requests.exceptions.RequestException as e:
    print(f"Error: {e}")

Weather API

import requests
import os

# Get weather information from OpenWeatherMap API
def get_weather(city, api_key):
    base_url = "https://api.openweathermap.org/data/2.5/weather"
    
    params = {
        'q': city,
        'appid': api_key,
        'units': 'metric'  # Use metric units (Celsius)
    }
    
    response = requests.get(base_url, params=params)
    response.raise_for_status()
    
    return response.json()

# Example usage
try:
    # Get API key from environment variable (best practice)
    api_key = os.getenv('OPENWEATHERMAP_API_KEY')
    
    if not api_key:
        print("Please set the OPENWEATHERMAP_API_KEY environment variable")
        exit(1)
    
    city = "London"
    weather_data = get_weather(city, api_key)
    
    temp = weather_data['main']['temp']
    humidity = weather_data['main']['humidity']
    description = weather_data['weather'][0]['description']
    
    print(f"Weather in {city}:")
    print(f"Temperature: {temp}°C")
    print(f"Humidity: {humidity}%")
    print(f"Conditions: {description}")
    
except requests.exceptions.RequestException as e:
    print(f"Error fetching weather data: {e}")

Practice Exercises

Try these exercises to strengthen your API and requests skills:

  1. Create a function to search for books using the Google Books API
  2. Build a simple command-line weather app using a weather API
  3. Write a script to download and save images from an image API
  4. Create a program that fetches and displays cryptocurrency prices
  5. Build a function to post updates to a social media platform API