replace urlgrabber with pycurl

cli/koji

@@ -59,14 +59,13 @@ import logging
 import os
 import re
 import pprint
+import pycurl
 import random
 import socket
 import stat
 import string
 import time
 import traceback
-import urlgrabber.grabber as grabber
-import urlgrabber.progress as progress
 import six.moves.xmlrpc_client
 try:
     import libcomps
@@ -6820,15 +6819,29 @@ def anon_handle_download_build(options, session, args):
             url = pathinfo.build(info) + '/' + fname
             urls.append((url, os.path.basename(fname)))
 
-    if suboptions.quiet:
-        pg = None
-    else:
-        pg = progress.TextMeter()
+    def _progress(download_t, download_d, upload_t, upload_d):
+        if download_t == 0:
+            percent_done = 0.0
+        else:
+            percent_done = float(download_d)/float(download_t)
+        percent_done_str = "%02d%%" % (percent_done * 100)
+        data_done = _format_size(download_d)
+
+        sys.stdout.write("[% -36s] % 4s % 10s\r" % ('='*(int(percent_done * 36)), percent_done_str, data_done))
+        sys.stdout.flush()
 
     for url, relpath in urls:
         if '/' in relpath:
             koji.ensuredir(os.path.dirname(relpath))
-        grabber.urlgrab(url, filename=relpath, progress_obj=pg, text=relpath)
+        print(relpath)
+        c = pycurl.Curl()
+        c.setopt(c.URL, url)
+        c.setopt(c.WRITEDATA, open(relpath, 'wb'))
+        if not suboptions.quiet:
+            c.setopt(c.NOPROGRESS, False)
+            c.setopt(c.XFERINFOFUNCTION, _progress)
+        c.perform()
+        print('')
 
 
 def anon_handle_download_logs(options, session, args):

docs/source/writing_koji_code.rst

@@ -667,8 +667,8 @@ You will need to install the following packages to actually run the tests.
  * ``python-krbV``
  * ``python-mock``
  * ``python-simplejson``
- * ``python-urlgrabber``
  * ``python-psycopg2``
+ * ``python-pycurl``
  * ``python-requests``
  * ``python-qpid-proton``
 

koji.spec

@@ -30,7 +30,7 @@ Requires: rpm-python
 Requires: pyOpenSSL
 Requires: python-requests
 Requires: python-requests-kerberos
-Requires: python-urlgrabber
+Requires: python-pycurl
 Requires: python-dateutil
 BuildRequires: python
 %if %{use_systemd}

vm/kojivmd

@@ -40,7 +40,7 @@ import SimpleXMLRPCServer
 import threading
 import base64
 import pwd
-import urlgrabber
+import pycurl
 import fnmatch
 from ConfigParser import ConfigParser
 from optparse import OptionParser
@@ -665,7 +665,10 @@ class VMExecTask(BaseTaskHandler):
             else:
                 raise koji.BuildError('unsupported file type: %s' % type)
             koji.ensuredir(os.path.dirname(localpath))
-            urlgrabber.urlgrab(remote_url, filename=localpath)
+            c = pycurl.Curl()
+            c.setopt(c.URL, remote_url)
+            c.setopt(c.WRITEDATA, open(localpath, 'wb'))
+            c.perform()
 
         return file(localpath, 'r')