python自动从arxiv下载paper的示例代码

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time  : 2020/02/11 21:44
# @Author : dangxusheng
# @Email  : dangxusheng163@163.com
# @File  : download_by_href.py
'''
自动从arxiv.org 下载文献
'''

import os
import os.path as osp
import requests
from lxml import etree
from pprint import pprint
import re
import time
import glob

headers = {
 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36",
 "Host": 'arxiv.org'
}

HREF_CN = 'http://cn.arxiv.org/pdf/'
HREF_SRC = 'http://cn.arxiv.org/pdf/'
SAVE_PATH = '/media/dangxs/E/Paper/download_at_20200730'
os.makedirs(SAVE_PATH, exist_ok=True)

FAIL_URLS = []
FAIL_URLS_TXT = f'{SAVE_PATH}/fail_urls.txt'

def download(url, title):
 pattern = r'[\\/:*?"\'<>|\r\n]+'
 new_title = re.sub(pattern, " ", title)
 print(f'new title: {new_title}')
 save_filepath = '％s/％s.pdf' ％ (SAVE_PATH, new_title)
 if osp.exists(save_filepath) and osp.getsize(save_filepath) > 50 * 1024:
   print(f'this pdf is be existed.')
   return True
 try:
   with open(save_filepath, 'wb') as file:
     # 分字节下载
     r = requests.get(url, stream=True, timeout=None)
     for i in r.iter_content(2048):
       file.write(i)
   if osp.getsize(save_filepath) >= 10 * 1024:
     print('％s 下载成功.' ％ title)
     return True
 except Exception as e:
   print(e)
 return False

# 从arxiv.org 去下载
def search(start_size=0, title_keywords='Facial Expression'):
 # 访问地址: https://arxiv.org/find/grp_eess,grp_stat,grp_cs,grp_econ,grp_math/1/ti:+Face/0/1/0/past,2018,2019/0/1?skip=200&query_id=1c582e6c8afc6146&client_host=cn.arxiv.org
 req_url = 'https://arxiv.org/search/advanced'
 req_data = {
   'advanced': 1,
   'terms-0-operator': 'AND',
   'terms-0-term': title_keywords,
   'terms-0-field': 'title',
   'classification-computer_science': 'y',
   'classification-physics_archives': 'all',
   'classification-include_cross_list': 'include',
   'date-filter_by': 'date_range', # date_range | specific_year
   # 'date-year': DOWN_YEAR,
   'date-year': '',
   'date-from_date': '2015',
   'date-to_date': '2020',
   'date-date_type': 'announced_date_first', # submitted_date | submitted_date_first | announced_date_first
   'abstracts': 'show',
   'size': 50,
   'order': '-announced_date_first',
   'start': start_size,
 }
 res = requests.get(req_url, params=req_data, headers=headers)
 html = res.content.decode()
 html = etree.HTML(html)

total_text = html.xpath('//h1[@class="title is-clearfix"]/text()')
 total_text = ''.join(total_text).replace('\n', '').lstrip(' ').strip(' ')
 # i.e. : Showing 1–50 of 355 results
 num = re.findall('\d+', total_text)
 # Sorry, your query returned no results
 if len(num) == 0: return [], 0

total = int(num[-1]) # 查询总条数
 paper_list = html.xpath('//ol[@class="breathe-horizontal"]/li')
 info_list = []
 for p in paper_list:
   title = p.xpath('./p[@class="title is-5 mathjax"]//text()')
   title = ''.join(title).replace('\n', '').lstrip(' ').strip(' ')
   href = p.xpath('./div/p/a/@href')[0]
   info_list.append({'title': title, 'href': href})

return info_list, total

# 去指定页面下载
def search_special():
 res = requests.get('https://gitee.com/weberyoung/the-gan-zoo?_from=gitee_search')
 html = res.content.decode()
 html = etree.HTML(html)

paper_list = html.xpath('//div[@class="file_content markdown-body"]//li')
 info_list = []
 for p in paper_list:
   title = p.xpath('.//text()')
   title = ''.join(title).replace('\n', '').lstrip(' ').strip(' ')
   href = p.xpath('./a/@href')[0]
   info_list.append({'title': title, 'href': href})

pprint(info_list)
 return info_list

if __name__ == '__main__':
 page_idx = 0
 total = 1000
 keywords = 'Facial Action Unit'
 while page_idx <= total // 50:
   paper_list, total = search(page_idx * 50, keywords)
   print(f'total: {total}')
   if total == 0:
     print('no found .')
     exit(0)

for p in paper_list:
     title = p['title']
     href = HREF_CN + p['href'].split('/')[-1] + '.pdf'
     print(href)
     if not download(href, title):
       print('从国内镜像下载失败，从源地址开始下载 >>>>')
       # 使用国际URL再下载一次
       href = HREF_SRC + p['href'].split('/')[-1] + '.pdf'
       if not download(href, title):
         FAIL_URLS.append(p)
   page_idx += 1

# 下载最后的部分
 last_1 = total - page_idx * 50
 paper_list, total = search(last_1, keywords)
 for p in paper_list:
   title = p['title']
   href = HREF_CN + p['href'].split('/')[-1] + '.pdf'
   if not download(href, title):
     FAIL_URLS.append(p)
   time.sleep(1)

pprint(FAIL_URLS)
 with open(FAIL_URLS_TXT, 'a+') as f:
   for item in FAIL_URLS:
     href = item['href']
     title = item['title']
     f.write(href + '\n')

print('done.')

来源：https://www.cnblogs.com/dxscode/p/13406238.html