pubmed-database

PubMed 数据库

概述

PubMed 是美国国家医学图书馆的数据库，免费提供来自 MEDLINE 和生命科学期刊的 3,600 多万条生物医学引文。本技能涵盖通过 E-utilities REST API 进行程序化访问，以及使用布尔运算符、MeSH 词和字段标签构建高级搜索查询。

适用场景

• 使用结构化的布尔/MeSH 查询搜索生物医学文献
• 构建自动化文献监测或提取流水线
• 开展系统性文献综述或荟萃分析
• 通过 PMID/DOI 检索文章元数据、摘要或引文信息
• 以编程方式查找相关文章或探索引文网络
• 批量处理大量 PubMed 记录
• 对于原生 Python 的 PubMed 访问，优先使用 BioPython（Bio.Entrez）——本技能涵盖直接使用 REST API
• 对于更广泛的学术搜索（非生物医学），请使用 OpenAlex 或 Semantic Scholar API

先决条件

pip install requests  # HTTP client for E-utilities API
# Optional: pip install biopython  — Bio.Entrez wrapper (higher-level API)

API 速率限制：

• 不使用 API 密钥：每秒 3 次请求
• 使用 API 密钥：每秒 10 次请求（在 https://www.ncbi.nlm.nih.gov/account/ 注册）
• 始终在请求中包含带有联系邮箱的 User-Agent 标头

快速入门

import requests
import time

BASE_URL = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/"
API_KEY = "YOUR_API_KEY"  # Optional but recommended

def pubmed_request(endpoint, params):
    """Reusable helper for E-utilities API calls with rate limiting."""
    params.setdefault("api_key", API_KEY)
    response = requests.get(f"{BASE_URL}{endpoint}", params=params)
    response.raise_for_status()
    time.sleep(0.1 if API_KEY != "YOUR_API_KEY" else 0.34)  # Rate limit
    return response

# Search → Fetch workflow
search_resp = pubmed_request("esearch.fcgi", {
    "db": "pubmed", "term": "CRISPR[tiab] AND 2024[dp]",
    "retmax": 5, "retmode": "json"
})
pmids = search_resp.json()["esearchresult"]["idlist"]
print(f"Found {len(pmids)} articles: {pmids}")

fetch_resp = pubmed_request("efetch.fcgi", {
    "db": "pubmed", "id": ",".join(pmids),
    "rettype": "abstract", "retmode": "text"
})
print(fetch_resp.text[:500])

核心 API

1. 搜索查询构建

使用布尔运算符、字段标签和 MeSH 词构建 PubMed 查询。

# Boolean operators: AND, OR, NOT (must be uppercase)
queries = {
    "basic": "diabetes AND treatment AND 2024[dp]",
    "synonyms": "(metformin OR insulin) AND type 2 diabetes",
    "exclude": "cancer NOT review[pt]",
    "phrase": '"gene expression" AND RNA-seq',
    "field_tags": "smith ja[au] AND cancer[tiab] AND 2023[dp]",
}

# Common field tags:
# [tiab] = title/abstract    [au] = author       [mh] = MeSH term
# [pt]   = publication type   [dp] = date          [ta] = journal
# [1au]  = first author       [lastau] = last author
# [affil] = affiliation       [doi] = DOI          [pmid] = PubMed ID

# Date filtering
date_queries = {
    "single_year": "cancer AND 2024[dp]",
    "range": "cancer AND 2020:2024[dp]",
    "specific": "cancer AND 2024/03/15[dp]",
}

# MeSH terms — controlled vocabulary for precise searching
mesh_queries = {
    # [mh] includes narrower terms automatically
    "broad": "diabetes mellitus[mh]",
    # [majr] limits to major topic focus
    "focused": "diabetes mellitus[majr]",
    # MeSH + subheading
    "therapy": "diabetes mellitus, type 2[mh]/drug therapy",
    # Substance name
    "drug": "metformin[nm] AND diabetes mellitus[mh]",
}

# Common MeSH subheadings:
# /diagnosis  /drug therapy  /epidemiology  /etiology
# /prevention & control  /therapy  /genetics

2. ESearch — 搜索并检索 PMID

# Basic search
resp = pubmed_request("esearch.fcgi", {
    "db": "pubmed",
    "term": "CRISPR[tiab] AND genome editing[tiab] AND 2024[dp]",
    "retmax": 100,
    "retmode": "json",
    "sort": "relevance",  # or "pub_date", "first_author"
})
result = resp.json()["esearchresult"]
pmids = result["idlist"]
total = result["count"]
print(f"Total hits: {total}, Retrieved: {len(pmids)}")

# With history server (for large result sets > 500)
resp = pubmed_request("esearch.fcgi", {
    "db": "pubmed",
    "term": "cancer AND 2024[dp]",
    "usehistory": "y",
    "retmode": "json",
})
result = resp.json()["esearchresult"]
webenv = result["webenv"]
query_key = result["querykey"]
total = int(result["count"])
print(f"Stored {total} results on history server")

3. EFetch — 下载完整记录

# Fetch abstracts as text
resp = pubmed_request("efetch.fcgi", {
    "db": "pubmed",
    "id": ",".join(pmids[:10]),
    "rettype": "abstract",
    "retmode": "text",
})
print(resp.text[:500])

# Fetch XML for structured parsing
resp = pubmed_request("efetch.fcgi", {
    "db": "pubmed",
    "id": ",".join(pmids[:10]),
    "rettype": "xml",
    "retmode": "xml",
})

# Fetch from history server (batch processing)
batch_size = 500
for start in range(0, total, batch_size):
    resp = pubmed_request("efetch.fcgi", {
        "db": "pubmed",
        "query_key": query_key,
        "WebEnv": webenv,
        "retstart": start,
        "retmax": batch_size,
        "rettype": "xml",
        "retmode": "xml",
    })
    print(f"Fetched records {start}–{start + batch_size}")
    time.sleep(0.5)  # Extra delay for large batches

4. ESummary 和 ELink — 摘要与相关文章

# ESummary — lightweight document summaries
resp = pubmed_request("esummary.fcgi", {
    "db": "pubmed",
    "id": ",".join(pmids[:5]),
    "retmode": "json",
})
for uid, data in resp.json()["result"].items():
    if uid == "uids":
        continue
    print(f"PMID {uid}: {data.get('title', '')[:80]}")
    print(f"  Journal: {data.get('fulljournalname', '')}, "
          f"Date: {data.get('pubdate', '')}")

# ELink — find related articles
resp = pubmed_request("elink.fcgi", {
    "dbfrom": "pubmed",
    "db": "pubmed",
    "id": pmids[0],
    "cmd": "neighbor",
    "retmode": "json",
})
# Links to other NCBI databases
resp = pubmed_request("elink.fcgi", {
    "dbfrom": "pubmed",
    "db": "pmc",  # PubMed Central
    "id": pmids[0],
    "retmode": "json",
})

5. 引文匹配和标识符查询

# Search by identifiers
id_queries = {
    "pmid": "12345678[pmid]",
    "doi": "10.1056/NEJMoa123456[doi]",
    "pmc": "PMC123456[pmc]",
}

# ECitMatch — match partial citations to PMIDs
# Format: journal|year|volume|first_page|author_name|key|
citation = "Science|2008|320|5880|1185|key1|"
resp = pubmed_request("ecitmatch.cgi", {
    "db": "pubmed",
    "rettype": "xml",
    "bdata": citation,
})
print(f"Matched PMID: {resp.text.strip()}")

# Batch citation matching
citations = [
    "Nature|2020|580|7801|71|ref1|",
    "Science|2019|366|6463|347|ref2|",
]
resp = pubmed_request("ecitmatch.cgi", {
    "db": "pubmed",
    "rettype": "xml",
    "bdata": "\r".join(citations),
})

6. 出版物筛选

# Filter by publication type
type_filters = {
    "rcts": "randomized controlled trial[pt]",
    "reviews": "systematic review[pt]",
    "meta": "meta-analysis[pt]",
    "guidelines": "guideline[pt]",
    "case_reports": "case reports[pt]",
}

# Filter by text availability
availability = {
    "free_text": "free full text[sb]",
    "has_abstract": "hasabstract[text]",
}

# Combine filters
query = (
    "diabetes mellitus[mh] AND "
    "randomized controlled trial[pt] AND "
    "2023:2024[dp] AND "
    "free full text[sb] AND "
    "english[la]"
)
resp = pubmed_request("esearch.fcgi", {
    "db": "pubmed", "term": query, "retmax": 100, "retmode": "json"
})
print(f"Free RCTs on diabetes (2023-2024): {resp.json()['esearchresult']['count']}")

关键概念

E-utilities 端点摘要

端点	用途	关键参数
esearch.fcgi	搜索并返回 PMID	term、retmax、sort、usehistory
efetch.fcgi	下载完整记录	id/query_key+WebEnv、rettype、retmode
esummary.fcgi	轻量级摘要	id、retmode
epost.fcgi	将 UID 上传到服务端	id（逗号分隔）
elink.fcgi	相关文章、跨数据库检索	id、dbfrom、db、cmd
einfo.fcgi	列出数据库/字段	db（可选）
egquery.fcgi	统计各数据库中的命中数	term
espell.fcgi	拼写建议	term
ecitmatch.cgi	将引文匹配到 PMID	bdata

History Server 模式

对于超过 500 篇文章的结果集，使用 History Server 避免 URL 长度限制：

1. 使用 usehistory=y 执行 ESearch → 返回 WebEnv + QueryKey
2. 使用 WebEnv + QueryKey + retstart/retmax 分批执行 EFetch
3. 使用 EPost 将其他 PMID 上传到同一 WebEnv

自动术语映射（ATM）

未指定字段标签时，PubMed 会依次通过 MeSH 翻译表 → 期刊翻译表 → 作者索引 → 全文映射术语。使用显式字段标签或带引号的短语可绕过 ATM。

常见 MeSH 副主题词

副主题词	缩写	用途
/diagnosis	/DI	诊断方法
/drug therapy	/DT	药物治疗
/epidemiology	/EP	疾病模式
/etiology	/ET	疾病病因
/genetics	/GE	遗传学方面
/prevention & control	/PC	预防措施
/therapy	/TH	治疗方法

常见工作流

工作流 1：系统性综述搜索

import requests, time, json

# 1. Define PICO-structured query
query = (
    # Population
    "(diabetes mellitus, type 2[mh] OR type 2 diabetes[tiab]) AND "
    # Intervention + Comparison
    "(metformin[nm] OR lifestyle modification[tiab]) AND "
    # Outcome
    "(glycemic control[tiab] OR HbA1c[tiab]) AND "
    # Study design filter
    "(randomized controlled trial[pt] OR systematic review[pt]) AND "
    # Date range
    "2020:2024[dp]"
)

# 2. Search with history server
resp = pubmed_request("esearch.fcgi", {
    "db": "pubmed", "term": query,
    "usehistory": "y", "retmode": "json"
})
result = resp.json()["esearchresult"]
total = int(result["count"])
print(f"Systematic review hits: {total}")

# 3. Batch fetch all results as XML
import xml.etree.ElementTree as ET
articles = []
for start in range(0, total, 200):
    resp = pubmed_request("efetch.fcgi", {
        "db": "pubmed", "query_key": result["querykey"],
        "WebEnv": result["webenv"],
        "retstart": start, "retmax": 200,
        "rettype": "xml", "retmode": "xml"
    })
    root = ET.fromstring(resp.text)
    for article in root.findall('.//PubmedArticle'):
        pmid = article.findtext('.//PMID')
        title = article.findtext('.//ArticleTitle')
        articles.append({"pmid": pmid, "title": title})
    time.sleep(0.5)

print(f"Retrieved {len(articles)} articles for screening")

工作流 2：文献监测流水线

import json, datetime

# 1. Construct monitoring query
topic_query = (
    "(CRISPR[tiab] OR gene editing[tiab]) AND "
    "(therapeutics[tiab] OR clinical trial[pt])"
)

# 2. Search recent publications (last 30 days)
today = datetime.date.today()
start_date = today - datetime.timedelta(days=30)
query = f"{topic_query} AND {start_date.strftime('%Y/%m/%d')}:{today.strftime('%Y/%m/%d')}[dp]"

resp = pubmed_request("esearch.fcgi", {
    "db": "pubmed", "term": query,
    "retmax": 100, "retmode": "json", "sort": "pub_date"
})
pmids = resp.json()["esearchresult"]["idlist"]

# 3. Get summaries for new articles
if pmids:
    resp = pubmed_request("esummary.fcgi", {
        "db": "pubmed", "id": ",".join(pmids), "retmode": "json"
    })
    for uid in pmids:
        info = resp.json()["result"].get(uid, {})
        print(f"[{uid}] {info.get('title', 'N/A')[:80]}")
        print(f"  {info.get('fulljournalname', '')} — {info.get('pubdate', '')}")

关键参数

参数	端点	默认值	作用
term	ESearch	必填	使用布尔运算符／字段标签的搜索查询
retmax	ESearch/EFetch	20	返回的最大记录数（最多 10,000 条）
retstart	ESearch/EFetch	0	分页偏移量
rettype	EFetch	full	输出类型：abstract、medline、xml、uilist
retmode	All	xml	输出格式：xml、json、text
sort	ESearch	relevance	排序方式：relevance、pub_date、first_author
usehistory	ESearch	n	启用历史记录服务：大型结果集设为 y
api_key	All	无	NCBI API 密钥，可支持每秒 10 次请求（不使用时为 3 次）
cmd	ELink	neighbor	链接类型：neighbor、neighbor_score、prlinks
datetype	ESearch	pdat	日期字段：pdat（出版日期）、edat（Entrez 日期）

最佳实践

1. 始终使用 API 密钥 — 在 NCBI 注册，获得每秒 10 次请求的额度，而不是 3 次
2. 结果超过 500 条时使用历史记录服务 — 避免 URL 长度限制，并支持批量获取
3. 加入速率限制 — 使用 API 密钥时设置 time.sleep(0.1)，不使用时设置 time.sleep(0.34)
4. 在本地缓存结果 — PubMed 数据变化缓慢；缓存响应可最大限度减少 API 调用
5. 结合使用 MeSH 词和自由文本 — 组合 [mh] 和 [tiab] 以全面检索：(diabetes mellitus[mh] OR diabetes[tiab])
6. 记录搜索策略 — 对于系统综述，记录确切的查询、日期和结果数
7. 解析 XML 以获取结构化数据 — 文本输出便于人工阅读，但 XML 会保留字段结构，便于自动提取

故障排除

问题	原因	解决方案
HTTP 429（请求过多）	超出速率限制	添加 time.sleep()；使用 API 密钥提高限额
HTTP 414（URI 过长）	URL 中的 PMID 过多	使用历史记录服务（usehistory=y）或 EPost
结果集为空	查询条件过于严格	逐一移除筛选条件；使用 EInfo 检查 ATM
MeSH 映射结果异常	自动术语映射	使用显式字段标签：用 term[tiab] 代替裸词 term
缺少摘要	1975 年以前的文章或特定类型的文章	添加筛选条件：hasabstract[text]
XML 解析错误	响应格式错误	检查 retmode=xml 和 rettype=xml；处理编码
历史记录服务过期	会话过期（不活跃 8 小时）	使用 usehistory=y 重新运行 ESearch 以获取新的 WebEnv
结果被截断	默认值为 retmax=20	将 retmax 设为 100 或更高（最多 10,000）

常用示例

示例：下载一组基因的摘要

import requests
import time

def fetch_abstracts(gene_list, max_per_gene=5):
    """Retrieve PubMed abstracts for each gene in a list."""
    base = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils"
    records = []
    for gene in gene_list:
        r = requests.get(f"{base}/esearch.fcgi",
                         params={"db": "pubmed", "term": f"{gene}[gene] AND Homo sapiens[orgn]",
                                 "retmax": max_per_gene, "retmode": "json"})
        ids = r.json()["esearchresult"]["idlist"]
        if ids:
            fetch = requests.get(f"{base}/efetch.fcgi",
                                 params={"db": "pubmed", "id": ",".join(ids), "rettype": "abstract"})
            records.append({"gene": gene, "pmids": ids, "text": fetch.text[:500]})
        time.sleep(0.34)
    return records

results = fetch_abstracts(["BRCA1", "TP53", "EGFR"])
for r in results:
    print(f"{r['gene']}: {r['pmids']}")

示例：通过日期筛选追踪新发表的文献

import requests
from datetime import date, timedelta

# Find papers published in the last 7 days on a topic
week_ago = (date.today() - timedelta(days=7)).strftime("%Y/%m/%d")
today = date.today().strftime("%Y/%m/%d")

resp = requests.get("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi",
                    params={"db": "pubmed", "term": "CRISPR AND cancer",
                            "datetype": "pdat", "mindate": week_ago, "maxdate": today,
                            "retmax": 20, "retmode": "json"})
data = resp.json()["esearchresult"]
print(f"New CRISPR+cancer papers this week: {data['count']}")
print("PMIDs:", data["idlist"])

捆绑资源

• references/search_syntax.md — 完整的字段标签参考、布尔／通配符／邻近检索语法、自动术语映射规则、所有筛选类型（年龄组、物种、语言）以及临床查询筛选条件
• references/common_queries.md — 按领域整理的即用型查询模板（疾病、特定人群、方法、药物研究、流行病学），含约 40 个示例模式

未从原版迁移：references/api_reference.md（298 行）— 端点参数详情已整合到核心 API 第 2–5 节以及「关键概念」中的 E-utilities 端点摘要表。

参考资料

• PubMed 帮助：https://pubmed.ncbi.nlm.nih.gov/help/
• E-utilities 文档：https://www.ncbi.nlm.nih.gov/books/NBK25501/
• NCBI API 密钥注册：https://www.ncbi.nlm.nih.gov/account/

相关技能

• biopython — E-utilities 的更高层 Python 封装（Bio.Entrez）
• openalex-database — 覆盖生物医学以外领域的更广泛学术文献
• literature-review — 系统综述方法和 PRISMA 框架