CentOS 下多線程下載工具：axel

環境相關：
OS：CentOS release 7.8

一.Linux多線程下載命令axel編譯安裝

wget http://www.ha97.com/code/axel-2.4.tar.gz
或者
wget https://files.cnblogs.com/files/xiaochina/axel-2.4.tar.gz

# 找到一個還可以下載的網址
# ubuntu可以直接使用apt-get下載使用

tar -xf axel-2.4.tar.gz
cd axel-2.4
yum -y install gcc-c++
./configure
make && make install

# 編譯安裝，需要gcc環境

axel -h
# 查看幫助

axel -n 5 -s 1024 'https://mirrors.aliyun.com/centos/7.7.1908/isos/x86_64/CentOS-7-x86_64-DVD-1908.iso'

# 測試下載
# 使用5個線程同時下載文件
# 限定下載速度上限爲1024B/S
# 可以ctrl+c停掉，再次執行
# 發現可以斷點續傳

ln -s /usr/local/bin/axel /bin/axel

二.Yum加速改造

sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf

cd /etc/yum/pluginconf.d/
cat >axelget.conf<<EOF
[main]
enabled=1
onlyhttp=1
enablesize=54000
cleanOnException=1
EOF

cd /usr/lib/yum-plugins/
cat >axelget.py<<EOF
from yum.plugins import PluginYumExit, TYPE_CORE, TYPE_INTERACTIVE
from urlparse import urljoin
import os,time

requires_api_version = '2.3'
plugin_type = (TYPE_CORE, TYPE_INTERACTIVE)

enablesize=300000
trymirrornum=-1
maxconn=10
httpdownloadonly=False
cleanOnException=0

def init_hook(conduit):
  global enablesize,trymirrornum,maxconn,cleanOnException,httpdownloadonly
  enablesize = conduit.confInt('main','enablesize',default=30000)
  trymirrornum = conduit.confInt('main','trymirrornum',default=-1)
  maxconn = conduit.confInt('main','maxconn',default=10)
  httpdownloadonly=conduit.confBool('main','onlyhttp',default=False)
  cleanOnException=conduit.confInt('main','cleanOnException',default=0)
  return

def predownload_hook(conduit):
  global enablesize,cleanOnException,httpdownloadonly
  preffermirror=""
  PkgIdx=0
  TotalPkg=len(conduit.getDownloadPackages())
  for po in (conduit.getDownloadPackages()):
    PkgIdx+=1
    if hasattr(po, 'pkgtype') and po.pkgtype == 'local':
      continue
    totsize = long(po.size)
    ret = False
    if totsize <= enablesize:
      conduit.info(2, "Package %s download size %d less than %d,Skip plugin!"  % (po.repo.id,totsize,enablesize))
      continue
    else:
      conduit.info(2, "[%d/%d]Ok,we will try to use axel to download this big file:%d" % (PkgIdx,TotalPkg,totsize))
    local = po.localPkg()
    if os.path.exists(local):
      if not os.path.exists(local+".st"):
        fstate=os.stat(local)
        if totsize == fstate.st_size:
          conduit.info(2,"Target already exists,skip to next file!")
          continue
    localall = "%s %s" % (local,local+".st")
    rmcmd = "rm -f %s" % (localall)
    curmirroridx = 0
    conduit.info(2,"Before we start,clean all the key files")
    os.system(rmcmd)
    connnum = totsize / enablesize
    if connnum*enablesize<totsize:
      connnum+=1
    if connnum > maxconn:
      connnum = maxconn
    mirrors=[]
    mirrors[:0]=po.repo.urls
    if preffermirror != "":
      mirrors[:0] = [preffermirror]
    for url in mirrors:
      if url.startswith("ftp://") and httpdownloadonly:
        print "Skip Ftp Site:",url
        continue
      if url.startswith("file://"):
        print "Skip Local Site:",url
        continue
      curmirroridx += 1
      if (curmirroridx > trymirrornum) and (trymirrornum != -1):
        conduit.info(2, "Package %s has tried %d mirrors,Skip plugin!" % (po.repo.id,trymirrornum))
        break
      remoteurl =  "%s/%s" % (url,po.remote_path)
      syscmd = "axel -a -n %s %s -o %s" % (connnum,remoteurl,local)
      conduit.info(2, "Execute axel cmd:\n%s"  % syscmd)
      os.system(syscmd)
      time.sleep(2)
      if os.path.exists(local+".st"):
        conduit.info(2,"axel exit by exception,let's try another mirror")
        if cleanOnException:
          conduit.info(2,"because cleanOnException is set to 1,we do remove key file first")
          os.system(rmcmd)
        continue
      elif not os.path.exists(local):#this mirror may not update yet
        continue
      else:
        ret = True
        preffermirror=url
        break
    if not ret:
      conduit.info (2,"try to run rm cmd:%s"  % rmcmd)
      os.system(rmcmd)
EOF

yum -y install zip unzip
# 測試，觀察日誌是否使用axel進行下載

yum -y install zip unzip

# 測試，觀察日誌是否使用axel進行下載