particle-os/debian-koji
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

urllib changes

Browse source
This commit is contained in:
Tomas Kopecek 2017-05-03 13:19:18 +02:00
parent 0c0cf92b4c
commit 7d277a8ad5
13 changed files with 50 additions and 55 deletions
Download patch file Download diff file Expand all files Collapse all files

17
koji/__init__.py
View file

@ -72,15 +72,13 @@ import struct
import tempfile
import time
import traceback
import urllib
import urllib2
import urlparse
from . import util
import warnings
import six.moves.xmlrpc_client
import xml.sax
import xml.sax.handler
from six.moves.xmlrpc_client import loads, dumps, Fault
import six.moves.urllib
PROFILE_MODULES = {} # {module_name: module_instance}
@ -1566,7 +1564,7 @@ def openRemoteFile(relpath, topurl=None, topdir=None, tempdir=None):
on options"""
if topurl:
url = "%s/%s" % (topurl, relpath)
src = urllib2.urlopen(url)
src = six.moves.urllib.request.urlopen(url)
fo = tempfile.TemporaryFile(dir=tempdir)
shutil.copyfileobj(src, fo)
src.close()
@ -2156,8 +2154,7 @@ class ClientSession(object):
"""Get the Kerberos principal of the server we're connecting
to, based on baseurl."""
uri = urlparse.urlsplit(self.baseurl)
host, port = urllib.splitport(uri[1])
host = six.moves.urllib.parse.urlparse(self.baseurl).hostname
if self.opts.get('krb_rdns', True):
servername = socket.getfqdn(host)
else:
@ -2175,7 +2172,7 @@ class ClientSession(object):
# force https
old_baseurl = self.baseurl
uri = urlparse.urlsplit(self.baseurl)
uri = six.moves.urllib.parse.urlsplit(self.baseurl)
if uri[0] != 'https':
self.baseurl = 'https://%s%s' % (uri[1], uri[2])
@ -2219,7 +2216,7 @@ class ClientSession(object):
# when API is changed
# force https
uri = urlparse.urlsplit(self.baseurl)
uri = six.moves.urllib.parse.urlsplit(self.baseurl)
if uri[0] != 'https':
self.baseurl = 'https://%s%s' % (uri[1], uri[2])
@ -2294,7 +2291,7 @@ class ClientSession(object):
sinfo = self.sinfo.copy()
sinfo['callnum'] = self.callnum
self.callnum += 1
handler = "%s?%s" % (self.baseurl, urllib.urlencode(sinfo))
handler = "%s?%s" % (self.baseurl, six.moves.urllib.parse.urlencode(sinfo))
elif name == 'sslLogin':
handler = self.baseurl + '/ssllogin'
else:
@ -2565,7 +2562,7 @@ class ClientSession(object):
args['volume'] = volume
size = len(chunk)
self.callnum += 1
handler = "%s?%s" % (self.baseurl, urllib.urlencode(args))
handler = "%s?%s" % (self.baseurl, six.moves.urllib.parse.urlencode(args))
headers = [
('User-Agent', 'koji/1'),
("Content-Type", "application/octet-stream"),

7
koji/compatrequests.py
View file

@ -9,8 +9,7 @@ the bits that koji needs.
from __future__ import absolute_import
import six.moves.http_client
import urlparse
import urllib
import six.moves.urllib
import sys
from .ssl import SSLCommon
import six
@ -27,7 +26,7 @@ class Session(object):
def post(self, url, data=None, headers=None, stream=None, verify=None,
cert=None, timeout=None):
uri = urlparse.urlsplit(url)
uri = six.moves.urllib.parse.urlsplit(url)
if uri[3]:
handler = "%s?%s" % (uri[2], uri[3])
else:
@ -46,7 +45,7 @@ class Session(object):
def get_connection(self, uri, cert, verify, timeout):
scheme = uri[0]
host, port = urllib.splitport(uri[1])
host, port = six.moves.urllib.splitport(uri[1])
key = (scheme, host, cert, verify, timeout)
#if self.connection and self.opts.get('keepalive'):
if self.connection: # XXX honor keepalive

4
koji/tasks.py
View file

@ -27,12 +27,12 @@ import os
import logging
import six.moves.xmlrpc_client
import signal
import urllib2
import shutil
import random
import time
import pprint
from six.moves import range
import six.moves.urllib
def scan_mounts(topdir):
"""Search path for mountpoints"""
@ -311,7 +311,7 @@ class BaseTaskHandler(object):
return fn
self.logger.debug("Downloading %s", relpath)
url = "%s/%s" % (self.options.topurl, relpath)
fsrc = urllib2.urlopen(url)
fsrc = six.moves.urllib.request.urlopen(url)
if not os.path.exists(os.path.dirname(fn)):
os.makedirs(os.path.dirname(fn))
fdst = open(fn, 'w')

2
tests/test_client_session.py
View file

@ -10,7 +10,7 @@ class TestClientSession(unittest.TestCase):
@mock.patch('socket.getfqdn')
def test_server_principal_rdns(self, getfqdn):
opts = {'krb_rdns': True}
session = koji.ClientSession('http://koji.example.com/kojihub', opts)
session = koji.ClientSession('http://koji.example.com:30/kojihub', opts)
cprinc = mock.MagicMock()
cprinc.realm = "REALM"
getfqdn.return_value = 'koji02.example.com'

38
tests/test_compatrequests.py
View file

@ -2,7 +2,7 @@ from __future__ import absolute_import
import six.moves.http_client
import mock
import unittest
import urlparse
import six.moves.urllib
import koji.compatrequests
@ -106,7 +106,7 @@ class TestSessionConnection(unittest.TestCase):
# no cert, no verify, no timeout
session = koji.compatrequests.Session()
url = 'http://www.fakedomain234234.org/KOJIHUB?a=1&b=2'
uri = urlparse.urlsplit(url)
uri = six.moves.urllib.parse.urlsplit(url)
cnx = session.get_connection(uri, None, None, None)
HTTPConnection.assert_called_once_with('www.fakedomain234234.org', 80)
@ -124,7 +124,7 @@ class TestSessionConnection(unittest.TestCase):
def test_cached(self):
session = koji.compatrequests.Session()
url = 'http://www.fakedomain234234.org/KOJIHUB?a=1&b=2'
uri = urlparse.urlsplit(url)
uri = six.moves.urllib.parse.urlsplit(url)
key = ('http', 'www.fakedomain234234.org', None, None, None)
cnx = mock.MagicMock()
session.connection = (key, cnx)
@ -135,10 +135,10 @@ class TestSessionConnection(unittest.TestCase):
def test_badproto(self):
session = koji.compatrequests.Session()
url = 'nosuchproto://www.fakedomain234234.org/KOJIHUB?a=1&b=2'
uri = urlparse.urlsplit(url)
uri = six.moves.urllib.parse.urlsplit(url)
with self.assertRaises(IOError):
ret = session.get_connection(uri, None, None, None)
session.get_connection(uri, None, None, None)
@mock.patch('httplib.HTTPConnection')
@mock.patch('sys.version_info', new=(2, 7, 12, 'final', 0))
@ -146,12 +146,12 @@ class TestSessionConnection(unittest.TestCase):
# no cert, no verify
session = koji.compatrequests.Session()
url = 'http://www.fakedomain234234.org/KOJIHUB?a=1&b=2'
uri = urlparse.urlsplit(url)
uri = six.moves.urllib.parse.urlsplit(url)
timeout = 1701
cnx = session.get_connection(uri, None, None, 1701)
HTTPConnection.assert_called_once_with('www.fakedomain234234.org', 80, timeout=1701)
key = ('http', 'www.fakedomain234234.org', None, None, 1701)
cnx = session.get_connection(uri, None, None, timeout)
HTTPConnection.assert_called_once_with('www.fakedomain234234.org', 80, timeout=timeout)
key = ('http', 'www.fakedomain234234.org', None, None, timeout)
self.assertEqual(session.connection, (key, cnx))
@mock.patch('httplib.HTTPConnection')
@ -160,22 +160,22 @@ class TestSessionConnection(unittest.TestCase):
# no cert, no verify
session = koji.compatrequests.Session()
url = 'http://www.fakedomain234234.org/KOJIHUB?a=1&b=2'
uri = urlparse.urlsplit(url)
uri = six.moves.urllib.parse.urlsplit(url)
timeout = 1701
cnx = session.get_connection(uri, None, None, 1701)
cnx = session.get_connection(uri, None, None, timeout)
HTTPConnection.assert_called_once_with('www.fakedomain234234.org', 80)
key = ('http', 'www.fakedomain234234.org', None, None, 1701)
key = ('http', 'www.fakedomain234234.org', None, None, timeout)
self.assertEqual(session.connection, (key, cnx))
cnx.connect.assert_called_once()
cnx.sock.settimeout.assert_called_with(1701)
cnx.sock.settimeout.assert_called_with(timeout)
@mock.patch('httplib.HTTPSConnection')
def test_https(self, HTTPSConnection):
# no cert, no verify, no timeout
session = koji.compatrequests.Session()
url = 'https://www.fakedomain234234.org/KOJIHUB?a=1&b=2'
uri = urlparse.urlsplit(url)
uri = six.moves.urllib.parse.urlsplit(url)
cnx = session.get_connection(uri, None, None, None)
HTTPSConnection.assert_called_once_with('www.fakedomain234234.org', 443)
@ -188,7 +188,7 @@ class TestSessionConnection(unittest.TestCase):
# no verify, no timeout
session = koji.compatrequests.Session()
url = 'https://www.fakedomain234234.org/KOJIHUB?a=1&b=2'
uri = urlparse.urlsplit(url)
uri = six.moves.urllib.parse.urlsplit(url)
cert = '/path/to/cert/file'
context = mock.MagicMock()
CreateSSLContext.return_value = context
@ -205,7 +205,7 @@ class TestSessionConnection(unittest.TestCase):
# no cert, verify=False, no timeout
session = koji.compatrequests.Session()
url = 'https://www.fakedomain234234.org/KOJIHUB?a=1&b=2'
uri = urlparse.urlsplit(url)
uri = six.moves.urllib.parse.urlsplit(url)
context = mock.MagicMock()
create_unverified_context.return_value = context
@ -221,7 +221,7 @@ class TestSessionConnection(unittest.TestCase):
# no cert, verify=False, no timeout
session = koji.compatrequests.Session()
url = 'https://www.fakedomain234234.org/KOJIHUB?a=1&b=2'
uri = urlparse.urlsplit(url)
uri = six.moves.urllib.parse.urlsplit(url)
cnx = session.get_connection(uri, None, False, None)
HTTPSConnection.assert_called_once_with('www.fakedomain234234.org', 443)
@ -236,7 +236,7 @@ class TestSessionConnection(unittest.TestCase):
# no cert, no timeout
session = koji.compatrequests.Session()
url = 'https://www.fakedomain234234.org/KOJIHUB?a=1&b=2'
uri = urlparse.urlsplit(url)
uri = six.moves.urllib.parse.urlsplit(url)
context = mock.MagicMock()
SSLContext.return_value = context
verify = '/path/to/verify/cert'
@ -257,7 +257,7 @@ class TestSessionConnection(unittest.TestCase):
# no cert, no timeout
session = koji.compatrequests.Session()
url = 'https://www.fakedomain234234.org/KOJIHUB?a=1&b=2'
uri = urlparse.urlsplit(url)
uri = six.moves.urllib.parse.urlsplit(url)
verify = '/path/to/verify/cert'
cnx = session.get_connection(uri, None, verify, None)

2
tests/test_tasks.py
View file

@ -462,7 +462,7 @@ class TasksTestCase(TestCase):
obj = TestTask(123, 'some_method', ['random_arg'], None, options, temp_path)
self.assertEquals(obj.localPath('test.txt'), dummy_file)
@patch('urllib2.urlopen', return_value=StringIO(six.text_type('Important things\nSome more important things\n')))
@patch('six.moves.urllib.request.urlopen', return_value=StringIO(six.text_type('Important things\nSome more important things\n')))
def test_BaseTaskHandler_localPath_no_file(self, mock_urlopen):
"""
"""

2
tests/test_utils.py
View file

@ -82,7 +82,7 @@ class MiscFunctionTestCase(unittest.TestCase):
islink.assert_called_once_with(dst)
move.assert_not_called()
@mock.patch('urllib2.urlopen')
@mock.patch('six.moves.urllib.request.urlopen')
@mock.patch('tempfile.TemporaryFile')
@mock.patch('shutil.copyfileobj')
@mock.patch('__builtin__.open')

9
util/koji-shadow
View file

@ -42,9 +42,8 @@ import socket # for socket.error and socket.setdefaulttimeout
import string
import sys
import time
import urllib2
import urlgrabber.grabber as grabber
import six.moves.xmlrpc_client # for ProtocolError and Fault
import six.moves.urllib
import rpm
# koji.fp.o keeps stalling, probably network errors...
@ -449,7 +448,7 @@ class TrackedBuild(object):
url = "%s/%s" % (pathinfo.build(self.info), pathinfo.rpm(self.srpm))
log("Downloading %s" % url)
#XXX - this is not really the right place for this
fsrc = urllib2.urlopen(url)
fsrc = six.moves.urllib.request.urlopen(url)
fn = "%s/%s.src.rpm" % (options.workpath, self.nvr)
koji.ensuredir(os.path.dirname(fn))
fdst = open(fn, 'w')
@ -891,7 +890,7 @@ class BuildTracker(object):
koji.ensuredir(os.path.dirname(dst))
os.chown(os.path.dirname(dst), 48, 48) #XXX - hack
log ("Downloading %s to %s" % (url, dst))
fsrc = urllib2.urlopen(url)
fsrc = six.moves.urllib.request.urlopen(url)
fdst = open(fn, 'w')
shutil.copyfileobj(fsrc, fdst)
fsrc.close()
@ -904,7 +903,7 @@ class BuildTracker(object):
koji.ensuredir(options.workpath)
dst = "%s/%s" % (options.workpath, fn)
log ("Downloading %s to %s..." % (url, dst))
fsrc = urllib2.urlopen(url)
fsrc = six.moves.urllib.request.urlopen(url)
fdst = open(dst, 'w')
shutil.copyfileobj(fsrc, fdst)
fsrc.close()

4
www/kojiweb/archiveinfo.chtml
View file

@ -1,7 +1,7 @@
#import koji
#from kojiweb import util
#from pprint import pformat
#import urllib
#import six.moves.urllib
#attr _PASSTHROUGH = ['archiveID', 'fileOrder', 'fileStart', 'buildrootOrder', 'buildrootStart']
@ -97,7 +97,7 @@
</tr>
#for $file in $files
<tr class="$util.rowToggle($self)">
<td><a href="fileinfo?archiveID=$archive.id&filename=$urllib.quote($file.name)">$file.name</a></td><td>$file.size</td>
<td><a href="fileinfo?archiveID=$archive.id&filename=$six.moves.urllib.parse.quote($file.name)">$file.name</a></td><td>$file.size</td>
</tr>
#end for
</table>

6
www/kojiweb/fileinfo.chtml
View file

@ -1,12 +1,12 @@
#from kojiweb import util
#import urllib
#import six.moves.urllib
#import datetime
#include "includes/header.chtml"
#if $rpm
<h4>Information for file <a href="fileinfo?rpmID=$rpm.id&amp;filename=$urllib.quote($file.name)">$file.name</a></h4>
<h4>Information for file <a href="fileinfo?rpmID=$rpm.id&amp;filename=$six.moves.urllib.parse.quote($file.name)">$file.name</a></h4>
#elif $archive
<h4>Information for file <a href="fileinfo?archiveID=$archive.id&amp;filename=$urllib.quote($file.name)">$file.name</a></h4>
<h4>Information for file <a href="fileinfo?archiveID=$archive.id&amp;filename=$six.moves.urllib.parse.quote($file.name)">$file.name</a></h4>
#end if
<table>

4
www/kojiweb/rpminfo.chtml
View file

@ -2,7 +2,7 @@
#from kojiweb import util
#from pprint import pformat
#import time
#import urllib
#import six.moves.urllib
#attr _PASSTHROUGH = ['rpmID', 'fileOrder', 'fileStart', 'buildrootOrder', 'buildrootStart']
@ -237,7 +237,7 @@
</tr>
#for $file in $files
<tr class="$util.rowToggle($self)">
<td><a href="fileinfo?rpmID=$rpm.id&amp;filename=$urllib.quote($file.name)">$util.escapeHTML($file.name)</a></td><td>$file.size</td>
<td><a href="fileinfo?rpmID=$rpm.id&amp;filename=$six.moves.urllib.parse.quote($file.name)">$util.escapeHTML($file.name)</a></td><td>$file.size</td>
</tr>
#end for
</table>

4
www/kojiweb/searchresults.chtml
View file

@ -1,5 +1,5 @@
#from kojiweb import util
#import urllib
#import six.moves.urllib
#include "includes/header.chtml"
@ -38,7 +38,7 @@
<tr class="$util.rowToggle($self)">
<td>$result.id</td>
#set $quoted = $result.copy()
#silent $quoted['name'] = $urllib.quote($quoted['name'])
#silent $quoted['name'] = $six.moves.urllib.parse.quote($quoted['name'])
<td><a href="${infoURL % $quoted}">$result.name</a></td>
</tr>
#end for

6
www/kojiweb/taskinfo.chtml
View file

@ -1,6 +1,6 @@
#import koji
#from kojiweb import util
#import urllib
#import six.moves.urllib
#import cgi
#def printValue($key, $value, $sep=', ')
@ -424,9 +424,9 @@ $value
<th>Output</th>
<td>
#for $volume, $filename in $output
<a href="$pathinfo.task($task.id, volume=$volume)/$urllib.quote($filename)">$filename</a>
<a href="$pathinfo.task($task.id, volume=$volume)/$six.moves.urllib.parse.quote($filename)">$filename</a>
#if $filename.endswith('.log')
(<a href="getfile?taskID=$task.id&volume=$volume&name=$urllib.quote($filename)&offset=-4000">tail</a>)
(<a href="getfile?taskID=$task.id&volume=$volume&name=$six.moves.urllib.parse.quote($filename)&offset=-4000">tail</a>)
#end if
<br/>
#end for