From ab48fa80b255c1389251a61ab1a8e15e6d21169c Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Fri, 23 Jun 2017 11:34:25 -0400 Subject: [PATCH] move xmlrpc marshaller into lib and use i8 for large ints --- devtools/fakehub | 5 ++--- hub/kojihub.py | 3 +-- hub/kojixmlrpc.py | 46 ++------------------------------------------- koji/__init__.py | 48 ++++++++++++++++++++++++++++++++++++++++++++--- koji/daemon.py | 9 ++++----- 5 files changed, 54 insertions(+), 57 deletions(-) diff --git a/devtools/fakehub b/devtools/fakehub index fed749d5..4e4ba07b 100755 --- a/devtools/fakehub +++ b/devtools/fakehub @@ -7,7 +7,6 @@ import os import os.path import pprint import sys -import six.moves.xmlrpc_client from six.moves import cStringIO from six.moves.urllib.parse import quote @@ -58,7 +57,7 @@ def get_request(): else: args.append(nice_literal(s)) args = koji.encode_args(*args, **kwargs) - request = six.moves.xmlrpc_client.dumps(args, method, allow_none=1) + request = koji.dumps(args, method, allow_none=1) return request @@ -68,7 +67,7 @@ def start_response(status, headers): def parse_response(data): - p, u = six.moves.xmlrpc_client.getparser() + p, u = koji.getparser() for chunk in data: p.feed(chunk) p.close() diff --git a/hub/kojihub.py b/hub/kojihub.py index 7e575943..8c355b49 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -559,8 +559,7 @@ def make_task(method, arglist, **opts): raise koji.GenericError("invalid channel policy") # encode xmlrpc request - opts['request'] = xmlrpclib.dumps(tuple(arglist), methodname=method, - allow_none=1) + opts['request'] = koji.dumps(tuple(arglist), methodname=method, allow_none=1) opts['state'] = koji.TASK_STATES['FREE'] opts['method'] = method koji.plugin.run_callbacks('preTaskStateChange', attribute='state', old=None, new='FREE', info=opts) diff --git a/hub/kojixmlrpc.py b/hub/kojixmlrpc.py index 2a779801..29c24637 100644 --- a/hub/kojixmlrpc.py +++ b/hub/kojixmlrpc.py @@ -19,18 +19,14 @@ # Mike McLean from ConfigParser import RawConfigParser -import datetime import inspect import logging import os import sys import time import traceback -import types import pprint import resource -import xmlrpclib -from xmlrpclib import getparser, dumps, Fault import koji import koji.auth @@ -38,49 +34,11 @@ import koji.db import koji.plugin import koji.policy import koji.util +# import xmlrpclib functions from koji to use tweaked Marshaller +from koji import getparser, dumps, Fault from koji.context import context -# Workaround to allow xmlrpclib deal with iterators -class Marshaller(xmlrpclib.Marshaller): - - dispatch = xmlrpclib.Marshaller.dispatch.copy() - - def dump_generator(self, value, write): - dump = self.__dump - write("\n") - for v in value: - dump(v, write) - write("\n") - dispatch[types.GeneratorType] = dump_generator - - def dump_datetime(self, value, write): - # For backwards compatibility, we return datetime objects as strings - value = value.isoformat(' ') - self.dump_string(value, write) - dispatch[datetime.datetime] = dump_datetime - - MAXI8 = 2 ** 64 - 1 - MINI8 = -2 ** 64 - def dump_i8(self, value, write): - # python2's xmlrpclib doesn't support i8 extension for marshalling, - # but can unmarshall it correctly. - if value > Marshaller.MAXI8 or value < Marshaller.MINI8: - raise OverflowError, "long int exceeds XML-RPC limits" - elif value > xmlrpclib.MAXINT or value < xmlrpclib.MININT: - write("") - write(str(int(value))) - write("\n") - else: - write("") - write(str(int(value))) - write("\n") - dispatch[types.LongType] = dump_i8 - dispatch[types.IntType] = dump_i8 - -xmlrpclib.Marshaller = Marshaller - - class HandlerRegistry(object): """Track handlers for RPC calls""" diff --git a/koji/__init__.py b/koji/__init__.py index dffe9ed9..6e3f1c21 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -73,13 +73,55 @@ import struct import tempfile import time import traceback +import types from . import util import warnings -import six.moves.xmlrpc_client import xml.sax import xml.sax.handler -from six.moves.xmlrpc_client import loads, dumps, Fault import six.moves.urllib +from six.moves.xmlrpc_client import getparser, loads, dumps, Fault + +# Workaround to allow xmlrpclib deal with iterators and 64-bit ints +class Marshaller(six.moves.xmlrpc_client.Marshaller): + + dispatch = six.moves.xmlrpc_client.Marshaller.dispatch.copy() + + def dump_generator(self, value, write): + dump = self.__dump + write("\n") + for v in value: + dump(v, write) + write("\n") + dispatch[types.GeneratorType] = dump_generator + + def dump_datetime(self, value, write): + # For backwards compatibility, we return datetime objects as strings + value = value.isoformat(' ') + self.dump_string(value, write) + dispatch[datetime.datetime] = dump_datetime + + MAXI8 = 2 ** 64 - 1 + MINI8 = -2 ** 64 + def dump_i8(self, value, write): + # python2's xmlrpclib doesn't support i8 extension for marshalling, + # but can unmarshall it correctly. + if value > Marshaller.MAXI8 or value < Marshaller.MINI8: + raise OverflowError, "long int exceeds XML-RPC limits" + elif value > six.moves.xmlrpc_client.MAXINT or \ + value < six.moves.xmlrpc_client.MININT: + write("") + write(str(int(value))) + write("\n") + else: + write("") + write(str(int(value))) + write("\n") + dispatch[types.LongType] = dump_i8 + dispatch[types.IntType] = dump_i8 + +six.moves.xmlrpc_client.Marshaller = Marshaller +six.moves.xmlrpc_client.FastMarshaller = None + PROFILE_MODULES = {} # {module_name: module_instance} @@ -2390,7 +2432,7 @@ class ClientSession(object): return ret def _read_xmlrpc_response(self, response): - p, u = six.moves.xmlrpc_client.getparser() + p, u = getparser() for chunk in response.iter_content(8192): if self.opts.get('debug_xmlrpc', False): print("body: %r" % chunk) diff --git a/koji/daemon.py b/koji/daemon.py index 97064379..ba210a2c 100644 --- a/koji/daemon.py +++ b/koji/daemon.py @@ -34,7 +34,6 @@ import time import sys import traceback import errno -import xmlrpclib def incremental_upload(session, fname, fd, path, retries=5, logger=None): @@ -1209,12 +1208,12 @@ class TaskManager(object): try: response = (handler.run(),) # note that we wrap response in a singleton tuple - response = xmlrpclib.dumps(response, methodresponse=1, allow_none=1) + response = koji.dumps(response, methodresponse=1, allow_none=1) self.logger.info("RESPONSE: %r" % response) self.session.host.closeTask(handler.id, response) return - except xmlrpclib.Fault, fault: - response = xmlrpclib.dumps(fault) + except koji.Fault, fault: + response = koji.dumps(fault) tb = ''.join(traceback.format_exception(*sys.exc_info())).replace(r"\n", "\n") self.logger.warn("FAULT:\n%s" % tb) except (SystemExit, koji.tasks.ServerExit, KeyboardInterrupt): @@ -1233,7 +1232,7 @@ class TaskManager(object): if issubclass(e_class, koji.GenericError): #just pass it through tb = str(e) - response = xmlrpclib.dumps(xmlrpclib.Fault(faultCode, tb)) + response = koji.dumps(koji.Fault(faultCode, tb)) # if we get here, then we're handling an exception, so fail the task self.session.host.failTask(handler.id, response)