Debian系统下Python并发处理配置指南
一 基础准备与环境检查
python3 --version、pip3 --version;若未安装,执行 sudo apt update && sudo apt install -y python3 python3-pip。python3 -m venv .venv && source .venv/bin/activate。二 常用并发模型与最小可用示例
from concurrent.futures import ThreadPoolExecutor, as_completed
import requests, time
urls = ["https://httpbin.org/delay/1"] * 8
def fetch(u):
r = requests.get(u, timeout=5)
return r.status_code
start = time.time()
with ThreadPoolExecutor(max_workers=8) as pool:
futures = [pool.submit(fetch, u) for u in urls]
for f in as_completed(futures):
print(f.result())
print("Threads:", time.time() - start)
from concurrent.futures import ProcessPoolExecutor
import math
def work(n):
return sum(math.factorial(i) for i in range(n))
if __name__ == "__main__":
nums = [3000, 3000, 3000, 3000]
with ProcessPoolExecutor(max_workers=4) as pool:
print(pool.map(work, nums))
import asyncio, aiohttp, time
urls = ["https://httpbin.org/delay/1"] * 20
async def fetch(session, u):
async with session.get(u) as r:
return r.status
async def main():
async with aiohttp.ClientSession() as s:
tasks = [fetch(s, u) for u in urls]
return await asyncio.gather(*tasks)
start = time.time()
results = asyncio.run(main())
print("AsyncIO:", time.time() - start)
print(results[:10])
pip install gevent)import gevent
from gevent import monkey; monkey.patch_all()
import requests, time
urls = ["https://httpbin.org/delay/1"] * 20
def fetch(u):
return requests.get(u, timeout=5).status_code
start = time.time()
jobs = [gevent.spawn(fetch, u) for u in urls]
gevent.joinall(jobs)
print("gevent:", time.time() - start)
三 关键系统与服务配置
sudo apt install -y libopenmp-dev。export OMP_NUM_THREADS=$(nproc)export MKL_NUM_THREADS=$(nproc)、export OPENBLAS_NUM_THREADS=$(nproc)os.environ 设置,或在程序启动前导出。apt-get install -y supervisorsupervisord -n -c /etc/supervisord.confnodaemon=true、[include] files=/etc/supervisor/conf.d/*.conf四 调参与最佳实践
requests.Session、aiohttp.TCPConnector(limit=...))、设置 超时 与 重试。htop、perf、strace 等定位瓶颈。