mdpi_search.py
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
MDPI 论文搜索脚本
"""
import argparse
import requests
from bs4 import BeautifulSoup
import json
def search_mdpi_papers(keyword, journal=None, max_results=10):
"""
搜索 MDPI 论文
Args:
keyword: 搜索关键词
journal: 期刊名称（可选）
max_results: 最大结果数
Returns:
论文列表
"""
try:
# 尝试访问搜索页面，使用更简单的请求头
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2",
"Connection": "keep-alive",
"Upgrade-Insecure-Requests": "1",
"TE": "Trailers",
}
# 使用不同的搜索 URL 格式
if journal:
url = f"https://www.mdpi.com/search?term={keyword}&journal={journal}&pageCount={max_results}"
else:
url = f"https://www.mdpi.com/search?term={keyword}&pageCount={max_results}"
print(f"正在尝试访问: {url}")
# 使用会话来保持连接
session = requests.Session()
response = session.get(url, headers=headers, timeout=10)
response.raise_for_status()
print(f"响应状态: {response.status_code}")
print(f"响应内容长度: {len(response.text)}")
# 首先解析整个 HTML 以便备用方法使用
soup = BeautifulSoup(response.text, 'html.parser')
# 使用字符串搜索直接查找 __NUXT_DATA__
print("使用字符串搜索查找 __NUXT_DATA__")
start = response.text.find('id="__NUXT_DATA__">')
if start != -1:
start += len('id="__NUXT_DATA__">')
end = response.text.find('</script>', start)
if end != -1:
print("成功找到 __NUXT_DATA__ 内容")
try:
# 提取 JSON 数据
json_str = response.text[start:end]
data = json.loads(json_str)
print("JSON 数据解析成功")
# 查找 articles 信息
papers = []
def find_articles(obj):
if isinstance(obj, dict):
if 'articles' in obj:
return obj['articles']
for key in obj:
result = find_articles(obj[key])
if result:
return result
elif isinstance(obj, list):
for item in obj:
result = find_articles(item)
if result:
return result
return None
articles_list = find_articles(data)
if articles_list and isinstance(articles_list, list):
print(f"找到 {len(articles_list)} 篇文章")
for i, article in enumerate(articles_list):
if len(papers) >= max_results:
break
# 提取论文信息
paper_info = {}
# 提取标题
paper_info['title'] = article.get('title', 'No title')
# 提取作者信息
if 'authors' in article:
authors = []
for author in article['authors']:
if isinstance(author, dict) and 'name' in author:
authors.append(author['name'])
paper_info['authors'] = ', '.join(authors) if authors else 'No authors'
else:
paper_info['authors'] = 'No authors'
# 提取期刊信息
if 'journal' in article:
paper_info['pub_info'] = article['journal']
else:
paper_info['pub_info'] = 'No info'
# 提取链接
if 'identifier' in article:
paper_info['link'] = f"https://www.mdpi.com/10.3390/{article['identifier']}"
elif 'doi' in article:
paper_info['link'] = f"https://www.mdpi.com/{article['doi']}"
else:
paper_info['link'] = "No link"
papers.append(paper_info)
return papers
except Exception as e:
print(f"解析 JSON 数据失败: {e}")
print("尝试使用备用方法...")
else:
print("未找到 __NUXT_DATA__ 结束标签")
else:
print("未找到 __NUXT_DATA__ 开始标签")
# 如果无法解析 JSON 数据，使用备用方法
# 尝试查找所有包含 href 属性且包含 /article/ 或 /10.3390/ 的链接
papers = []
links = soup.find_all('a', href=True)
for link in links:
href = link['href']
# 检查是否是论文链接
if (('/article/' in href or '/10.3390/' in href) and
len(papers) < max_results):
# 尝试提取标题
title = link.text.strip()
# 检查是否有相邻的标题元素
parent = link.find_parent('div')
if parent:
h2 = parent.find('h2')
if h2:
title = h2.text.strip()
# 提取作者信息（如果可用）
authors = "No authors"
if parent:
author_elem = parent.find(class_='authors')
if not author_elem:
author_elem = parent.find('div', string=lambda text: 'by' in str(text))
if author_elem:
authors = author_elem.text.strip().replace('by', '').strip()
# 构建完整链接
if not href.startswith('http'):
href = f"https://www.mdpi.com{href}"
# 添加到结果列表
papers.append({
"title": title if title else "No title",
"link": href,
"authors": authors,
"pub_info": "No info"
})
# 去重结果
unique_papers = []
seen_links = set()
for paper in papers:
if paper['link'] not in seen_links:
seen_links.add(paper['link'])
unique_papers.append(paper)
# 限制结果数量
return unique_papers[:max_results]
except requests.exceptions.RequestException as e:
print(f"搜索失败: {e}")
return []
def main():
parser = argparse.ArgumentParser(description="搜索 MDPI 期刊论文")
parser.add_argument("keyword", help="搜索关键词")
parser.add_argument("--journal", help="特定期刊名称（可选）")
parser.add_argument("--max-results", type=int, default=10, help="最大结果数")
args = parser.parse_args()
print(f"搜索 MDPI 论文: {args.keyword}")
if args.journal:
print(f"期刊: {args.journal}")
papers = search_mdpi_papers(args.keyword, args.journal, args.max_results)
if papers:
print(f"\n找到 {len(papers)} 篇论文:\n")
for i, paper in enumerate(papers, 1):
print(f"{i}. {paper['title']}")
print(f" 作者: {paper['authors']}")
print(f" 信息: {paper['pub_info']}")
print(f" 链接: {paper['link']}")
print()
else:
print("未找到匹配的论文")
if __name__ == "__main__":
main()