1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387 |
- # coding=utf-8
-
- import collections
- import csv
- import difflib
- import hashlib
- import inspect
- import itertools
- import json
- import operator
- import time
- from copy import deepcopy
-
-
- ###############################################################################
- ## The Motor ##
- ###############################################################################
-
- def regression_test(argsrc, tests, driver_settings, cleanup_hack=None,
- apply_hacks=None, on_next=None):
- """Perform regression test with argsets from `argsrc`.
-
- For each argset pulled from source, performs one comparison
- per driver pair in `tests`, which is list of tuples with
- comparison function and pair of test driver classes: `(operator,
- oracle_class, result_class)`. (The classes are assumed to
- be sub-classes of `hoover.BaseTestDriver`.)
-
- `driver_settings` is a dictionary supposed to hold environmental
- values for all the drivers, the keys having form "DriverName.
- settingName". Each driver is then instantiated with this
- dict, and gets a copy of the dict with settings only intended
- for itself (and the "DriverName" part stripped).
-
- If comparison fails, report is generated using `hoover.jsDiff()`,
- and along with affected arguments stored in `hoover.Tracker`
- instance, which is finally used as a return value. This instance
- then contains method for basic stats as well as method to format
- the final report and a helper method to export argument sets
- as a CSV files.
-
- Supports hacks, which are a data transformations performed by
- `hoover.TinyCase` class and are intended to avoid known bugs
- and anomalies (`apply_hacks`) or clean up data structures of
- irrelevant data (`cleanup_hack`, performed only if the comparison
- function provided along with driver pair is not "equals").
-
- A function can be provided as `on_next` argument, that will be
- called after pulling each argument set, with last argument set
- (or `None`) as first argument and current one as second argument.
- """
-
- # TODO: do not parse driver_settings thousands of times (use a view class?)
-
- on_next = on_next if on_next else lambda a, b: None
- apply_hacks = apply_hacks if apply_hacks else []
-
- tracker = Tracker()
- last_argset = None
-
- all_classes = set(reduce(lambda a, b: a+b,
- [triple[1:] for triple in tests]))
-
- counter = StatCounter()
-
- for argset in argsrc:
-
- on_start = time.time()
- on_next(argset, last_argset)
- counter.add('on_next', time.time() - on_start)
-
- ## load the data first, only once for each driver
- #
- data = {}
- for aclass in all_classes:
- try:
- aclass.check_values(argset)
- except NotImplementedError: # let them bail out
- counter.count_for(aclass, 'bailouts')
- pass
- else:
- data[aclass], duration, overhead = get_data_and_stats(
- aclass, argset, driver_settings)
- counter.count_for(aclass, 'calls')
- counter.add_for(aclass, 'duration', duration)
- counter.add_for(aclass, 'overhead', overhead)
-
- for match_op, oclass, rclass in tests:
-
- # skip test if one of classes bailed out on the argset
- if oclass not in data or rclass not in data:
- continue
-
- diff = None
-
- case = TinyCase({
- 'argset': argset,
- 'oracle': deepcopy(data[oclass]),
- 'result': deepcopy(data[rclass]),
- 'oname': oclass.__name__,
- 'rname': rclass.__name__
- })
-
- hacks_done = sum([case.hack(h) for h in apply_hacks])
- counter.add_for(oclass, 'ohacks', hacks_done)
- counter.add_for(rclass, 'rhacks', hacks_done)
- counter.add('hacks', hacks_done)
- counter.add('hacked_cases', (1 if hacks_done else 0))
-
- if not match_op(case['oracle'], case['result']):
-
- # try to clean up so that normally ignored items
- # do not clutter up the report
- if not match_op == operator.eq:
- case.hack(cleanup_hack)
- if match_op(case['oracle'], case['result']):
- raise RuntimeError("cleanup ate error")
-
- diff = jsDiff(dira=case['oracle'],
- dirb=case['result'],
- namea=case['oname'],
- nameb=case['rname'])
-
- tracker.update(diff, argset)
-
- counter.count('cases')
-
- tracker.argsets_done += 1
- last_argset = argset
-
- counter.count('argsets')
-
- tracker.driver_stats = counter.all_stats()
- return tracker
-
-
- def get_data_and_stats(driverClass, argset, driver_settings):
- """Run test with given driver"""
- start = time.time()
- d = driverClass()
- d.setup(driver_settings, only_own=True)
- d.run(argset)
- return (d.data, d.duration, time.time() - d.duration - start)
-
-
- def get_data(driverClass, argset, driver_settings):
- """Run test with given driver"""
- d = driverClass()
- d.setup(driver_settings, only_own=True)
- d.run(argset)
- return d.data
-
-
- ###############################################################################
- ## The Pattern ##
- ###############################################################################
-
- class _BaseRuleOp():
-
- def __init__(self, items, item_ok):
- self._items = items
- self._item_ok = item_ok
-
- def _eval(self, item):
- try: # it's a pattern! (recurse)
- return RuleOp.Match(item, self._item_ok)
- except ValueError: # no, it's something else...
- return self._item_ok(item)
-
- def __nonzero__(self):
- try:
- return self._match()
- except TypeError:
- raise ValueError("items must be an iterable: %r" % self._items)
-
-
- class RuleOp():
-
- class ALL(_BaseRuleOp):
-
- def _match(self):
- return all(self._eval(item) for item in self._items)
-
- class ANY(_BaseRuleOp):
-
- def _match(self):
- return any(self._eval(item) for item in self._items)
-
- @staticmethod
- def Match(pattern, item_ok):
- """Evaluate set of logically structured patterns using passed function.
-
- pattern has form of `(op, [item1, item2, ...])` where op can be any of
- pre-defined logical operators (`ALL`/`ANY`, I doubt you will ever need
- more) and item_ok is a function that will be used to evaluate each one
- in the list. In case an itemN is actually pattern as well, it will be
- recursed into, passing the item_ok on and on.
-
- Note that there is no data to evaluate "against", you can use closure
- if you need to do that.
- """
-
- try:
- op, items = pattern
- except TypeError:
- raise ValueError("pattern is not a tuple: %r" % pattern)
- try:
- assert issubclass(op, _BaseRuleOp)
- except TypeError:
- raise ValueError("invalid operator: %r" % op)
- except AssertionError:
- raise ValueError("invalid operator class: %s" % op.__name__)
- return bool(op(items, item_ok))
-
-
- ###############################################################################
- ## The Path ##
- ###############################################################################
-
- class DictPath():
- """Mixin that adds "path-like" behavior to the top dict of dicts.
-
- See TinyCase for description"""
-
- DIV = "/"
-
- class Path():
-
- def __init__(self, path, div):
- self.DIV = div
- self._path = path
-
- def _validate(self):
- try:
- assert self._path.startswith(self.DIV)
- except (AttributeError, AssertionError):
- raise ValueError("invalid path: %r" % self._path)
-
- def stripped(self):
- return self._path.lstrip(self.DIV)
-
- @classmethod
- def __s2path(cls, path):
- return cls.Path(path, cls.DIV)
-
- @classmethod
- def __err_path_not_found(cls, path):
- raise KeyError("path not found: %s" % path)
-
- @classmethod
- def __getitem(cls, dct, key):
- if cls.DIV in key:
- frag, rest = key.split(cls.DIV, 1)
- subdct = dct[frag]
- result = cls.__getitem(subdct, rest)
- else:
- result = dct[key]
- return result
-
- @classmethod
- def __setitem(cls, dct, key, value):
- if cls.DIV not in key:
- dct[key] = value
- else:
- frag, rest = key.split(cls.DIV, 1)
- subdct = dct[frag]
- cls.__setitem(subdct, rest, value)
-
- @classmethod
- def __delitem(cls, dct, key):
- if cls.DIV not in key:
- del dct[key]
- else:
- frag, rest = key.split(cls.DIV, 1)
- subdct = dct[frag]
- return cls.__delitem(subdct, rest)
-
- ## public methods
- #
-
- def getpath(self, path):
- try:
- return self.__getitem(self, self.__s2path(path).stripped())
- except (TypeError, KeyError):
- self.__err_path_not_found(path)
-
- def setpath(self, path, value):
- try:
- self.__setitem(self, self.__s2path(path).stripped(), value)
- except (TypeError, KeyError):
- self.__err_path_not_found(path)
-
- def delpath(self, path):
- try:
- self.__delitem(self, self.__s2path(path).stripped())
- except (TypeError, KeyError):
- self.__err_path_not_found(path)
-
- def ispath(self, path):
- try:
- self.getpath(path)
- return True
- except KeyError:
- return False
-
-
- ###############################################################################
- ## The Case ##
- ###############################################################################
-
- class TinyCase(dict, DictPath):
- """Abstraction of the smallest unit of testing.
-
- This class is intended to hold relevant data after the actual test
- and apply transformations (hacks) as defined by rules.
-
- The data form (self) is:
-
- {
- 'argset': {}, # argset as fed into `BaseTestDriver.run`
- 'oracle': {}, # data as returned from oracle driver's `run()`
- 'result': {}, # data as returned from result driver's `run()`
- 'oname': "", # name of oracle driver's class
- 'rname': "" # name of result driver's class
- }
-
- The transformation is done using the `TinyCase.hack()` method to which
- a list of rules is passed. Each rule is applied, and rules are expected
- to be in a following form:
-
- {
- 'drivers': [{}], # list of structures to match against self
- 'argsets': [{}], # -ditto-
- 'action_name': <Arg> # an action name with argument
- }
-
- For each of patterns ('drivers', argsets') present, match against self
- is done using function `hoover.dataMatch`, which is basically a recursive
- test if the pattern is a subset of the case. If none of results is
- negative (i.e. both patterns missing results in match), any known actions
- included in the rule are called. Along with action name a list or a dict
- providing necessary parameters is expected: this is simply passed as only
- parameter to corresponding method.
-
- Actions use specific way how to address elements in the structures
- saved in the oracle and result keys provided by `DictPath`, which makes
- it easy to define rules for arbitrarily complex dictionary structures.
- The format resembles to Unix path, where "directories" are dict
- keys and "root" is the `self` of the `TinyCase` instance:
-
- /oracle/temperature
- /result/stats/word_count
-
- Refer to each action's docstring for descriprion of their function
- as well as expected format of argument. The name of action as used
- in the reule is the name of method without leading 'a_'.
-
- Warning: All actions will silently ignore any paths that are invalid
- or leading to non-existent data!
- (This does not apply to a path leading to `None`.)
- """
-
- def a_exchange(self, action):
- """Exchange value A for value B.
-
- Expects a dict, where key is a tuple of two values `(a, b)` and
- value is a list of paths. For each key, it goes through the
- paths and if the value equals `a` it is set to `b`.
- """
- for (oldv, newv), paths in action.iteritems():
- for path in paths:
- try:
- curv = self.getpath(path)
- except KeyError:
- continue
- else:
- if curv == oldv:
- self.setpath(path, newv)
-
- def a_format_str(self, action):
- """Convert value to a string using format string.
-
- Expects a dict, where key is a format string, and value is a list
- of paths. For each record, the paths are traversed, and value is
- converted to string using the format string and the `%` operator.
-
- This is especially useful for floats which you may want to trim
- before comparison, since direct comparison of floats is unreliable
- on some architectures.
- """
- for fmt, paths in action.iteritems():
- for path in paths:
- if self.ispath(path):
- new = fmt % self.getpath(path)
- self.setpath(path, new)
-
- def a_even_up(self, action):
- """Even up structure of both dictionaries.
-
- Expects a list of two-element tuples `('/dict/a', '/dict/b')`
- containing pairs of path do simple dictionaries.
-
- Then the two dicts are altered to have same structure: if a key
- in dict "a" is missing in dict "b", it is set to `None` in "b" and
- vice-versa,
- """
- for patha, pathb in action:
- try:
- a = self.getpath(patha)
- b = self.getpath(pathb)
- except KeyError:
- continue
- else:
- for key in set(a.keys()) | set(b.keys()):
- if key in a and key in b:
- pass # nothing to do here
- elif key in a and a[key] is None:
- b[key] = None
- elif key in b and b[key] is None:
- a[key] = None
- else:
- pass # bailout: odd key but value is *not* None
-
- def a_remove(self, action):
- """Remove elements from structure.
-
- Expects a simple list of paths that are simply deleted fro, the
- structure.
- """
- for path in action:
- if self.ispath(path):
- self.delpath(path)
-
- def a_round(self, action):
- """Round a (presumably) float using tha `float()` built-in.
-
- Expects dict with precision (ndigits, after the dot) as a key and
- list of paths as value.
- """
- for ndigits, paths in action.iteritems():
- for path in paths:
- try:
- f = self.getpath(path)
- except KeyError:
- pass
- else:
- self.setpath(path, round(f, ndigits))
-
- known_actions = {'remove': a_remove,
- 'even_up': a_even_up,
- 'format_str': a_format_str,
- 'exchange': a_exchange,
- 'round': a_round}
-
- def hack(self, ruleset):
- """Apply action from each rule, if patterns match."""
-
- def driver_matches():
- if 'drivers' not in rule:
- return True
- else:
- return any(dataMatch(p, self)
- for p in rule['drivers'])
-
- def argset_matches():
- if 'argsets' not in rule:
- return True
- else:
- return any(dataMatch(p, self)
- for p in rule['argsets'])
-
- matched = False
- cls = self.__class__
- for rule in ruleset:
- if driver_matches() and argset_matches():
- matched = True
- for action_name in cls.known_actions:
- if action_name in rule:
- cls.known_actions[action_name](self, rule[action_name])
- return matched
-
-
- ###############################################################################
- ## Drivers ##
- ###############################################################################
-
- class DriverError(Exception):
- """Error encountered when obtaining driver data"""
-
- def __init__(self, message, driver):
- self.message = message
- self.driver = driver
-
- def __str__(self):
-
- result = ("\n\n"
- " type: %s\n"
- " message: %s\n"
- " driver: %s\n"
- " args: %s\n"
- " settings: %s\n"
- % (self.message.__class__.__name__,
- self.message,
- self.driver.__class__.__name__,
- self.driver._args,
- self.driver._settings))
-
- return result
-
-
- class DriverDataError(Exception):
- """Error encountered when decoding or normalizing driver data"""
-
- def __init__(self, exception, driver):
- self.exception = exception
- self.driver = driver
-
- def __str__(self):
-
- result = ("%s: %s\n"
- " class: %s\n"
- " args: %s\n"
- " data: %s\n"
- % (self.exception.__class__.__name__, self.exception,
- self.driver.__class__.__name__,
- json.dumps(self.driver._args, sort_keys=True, indent=4),
- json.dumps(self.driver.data, sort_keys=True, indent=4)))
- return result
-
-
- class BaseTestDriver(object):
- """Base class for test drivers used by `hoover.regression_test` and others.
-
- This class is used to create a test driver, which is an abstraction
- and encapsulation of the system being tested. Or, the driver in fact
- can be just a "mock" driver that provides data for comparison with
- a "real" driver.
-
- The minimum you need to create a working driver is to implement a working
- `self._get_data` method that sets `self.data`. Any exception from this
- method will be re-raised as DriverError with additional information.
-
- Also, you can set self.duration (in fractional seconds, as returned by
- standard time module) in the _get_data method, but if you don't, it is
- measured for you as time the method call took. This is useful if you
- need to fetch the data from some other driver or a gateway, and you
- have better mechanism to determine how long the action would take "in
- real life".
-
- For example, if we are testing a Java library using a Py4J gateway,
- we need to do some more conversions outside our testing code just to
- be able to use the data in our Python test. We don't want to include
- this in the "duration", since we are measuring the Java library, not the
- Py4J GW (or our ability to perform the conversions optimally). So we
- do our measurement within the Java machine and pass the result to the
- Python driver.
-
- Optionally, you can:
-
- * Make an __init__ and after calling base __init__, set
-
- * `self._mandatory_args`, a list of keys that need to be present
- in `args` argument to `run()`
-
- * and `self._mandatory_settings`, a list of keys that need to be
- present in the `settings` argument to `__init__`
-
- * implement methods
-
- * `_decode_data` and `_normalize_data`, which are intended to decode
- the data from any raw format it is received, and to prepare it
- for comparison in test,
-
- * and `_check_data`, to allow for early detection of failure,
-
- from which any exception is re-raised as a DriverDataError with
- some additional info
-
- * set "bailouts", a list of functions which, when passed "args"
- argument, return true to indicate that driver is not able to
- process these values (see below for explanation). If any of
- these functions returns true, NotImplementedError is raised.
-
- The expected workflow when using the driver is:
-
- # 1. sub-class hoover.BaseTestDriver
- # 2. prepare settings and args
- MyDriver.check_values(args) # optional, to force bailouts ASAP
- d = MyDriver()
- d.setup(settings)
- d.run(args)
- assert d.data, "no data" # evaluate the result...
- assert d.duration < 1 # duration of _get_data in seconds
-
- Note on bailouts: Typical strategy for which the driver is intended is
- that each possible combination of `args` is exhausted, and results from
- multiple drivers are compared to evaluate if driver, i.e. system in
- question is O.K.
-
- The bailouts mechanism is useful in cases, where for a certain system,
- a valid combination of arguments would bring the same result as another,
- so there is basically no value in testing both of them.
-
- Example might be a system that does not support a binary flag and
- behaves as if it was "on": you can simply make the test driver
- accept the option but "bail out" any time it is "off", therefore
- skipping the time-and-resource-consuming test.
- """
-
- bailouts = []
-
- ##
- # internal methods
- #
-
- def __init__(self):
- self.data = {}
- self.duration = None
- self._args = {}
- self._mandatory_args = []
- self._mandatory_settings = []
- self._settings = {}
- self._setup_ok = False
-
- def __check_mandatory(self):
- """validate before run()"""
- for key in self._mandatory_args:
- assert key in self._args, "missing arg: '%s'" % key
- for key in self._mandatory_settings:
- assert key in self._settings, "missing setting: '%s'" % key
-
- def __cleanup_data(self):
- """remove hidden data; e.g. what was only there for _check_data"""
- for key in self.data.keys():
- if key.startswith("_"):
- del self.data[key]
-
- ##
- # virtual methods
- #
-
- def _check_data(self):
- """Early check for failure"""
- pass
-
- def _decode_data(self):
- """Decode from raw data as brought by _get_data"""
- pass
-
- def _normalize_data(self):
- """Preare data for comparison (e.g. sort, split, trim...)"""
- pass
-
- ##
- # public methods
- #
-
- @classmethod
- def check_values(cls, args=None):
- """check args in advance before running or setting up anything"""
- for fn in cls.bailouts:
- if fn(args):
- raise NotImplementedError(inspect.getsource(fn))
-
- def setup(self, settings, only_own=False):
- """Load settings. only_own means that only settings that belong to us
- are loaded ("DriverClass.settingName", the first discriminating part
- is removed)"""
- if only_own:
- for ckey in settings.keys():
- driver_class_name, setting_name = ckey.split(".", 2)
- if self.__class__.__name__ == driver_class_name:
- self._settings[setting_name] = settings[ckey]
- else:
- self._settings = settings
- self._setup_ok = True
-
- def run(self, args):
- """validate, run and store data"""
-
- self._args = args
- assert self._setup_ok, "run() before setup()?"
- self.__class__.check_values(self._args)
- self.__check_mandatory()
- start = time.time()
- try:
- self._get_data() # run the test, i.e. obtain raw data
- except StandardError as e:
- raise DriverError(e, self)
- self.duration = (time.time() - start if self.duration is None
- else self.duration)
- try:
- self._decode_data() # decode raw data
- self._normalize_data() # normalize decoded data
- self._check_data() # perform arbitrarty checking
- except StandardError, e:
- raise DriverDataError(e, self)
- self.__cleanup_data() # cleanup (remove data['_*'])
-
-
- class MockDriverTrue(BaseTestDriver):
- """A simple mock driver, always returning True"""
-
- def _get_data(self, args):
- self.data = True
-
-
- ###############################################################################
- ## Helpers ##
- ###############################################################################
-
- class StatCounter(object):
- """A simple counter with formulas support."""
-
- def __init__(self):
- self.generic_stats = {}
- self.driver_stats = {}
- self.formulas = {}
- self._born = time.time()
-
- def _register(self, dname):
- self.driver_stats[dname] = {
- 'calls': 0,
- 'rhacks': 0,
- 'ohacks': 0,
- 'duration': 0,
- 'overhead': 0
- }
-
- ##
- ## Formulas. A lot of them.
- ##
-
- ## cumulative duration/overhead; just round to ms
- #
- self.add_formula(dname + '_overhead',
- lambda g, d: int(1000 * d[dname]['overhead']))
- self.add_formula(dname + '_duration',
- lambda g, d: int(1000 * d[dname]['duration']))
-
- ## average (per driver call) overhead/duration
- #
- self.add_formula(
- dname + '_overhead_per_call',
- lambda g, d: int(1000 * d[dname]['overhead'] / d[dname]['calls'])
- )
- self.add_formula(
- dname + '_duration_per_call',
- lambda g, d: int(1000 * d[dname]['duration'] / d[dname]['calls'])
- )
-
- ## grand totals in times: driver time, loop overhead
- #
- def gtotal_drivertime(g, d):
- driver_time = (sum(s['overhead'] for s in d.values())
- + sum(s['duration'] for s in d.values()))
- return int(1000 * driver_time)
-
- def gtotal_loop_overhead(g, d):
- driver_time = gtotal_drivertime(g, d)
- onnext_time = int(1000 * g['on_next'])
- age = int(1000 * (time.time() - self._born))
- return age - driver_time - onnext_time
-
- self.add_formula('gtotal_drivertime', gtotal_drivertime)
- self.add_formula('gtotal_loop_overhead', gtotal_loop_overhead)
- self.add_formula('gtotal_loop_onnext',
- lambda g, d: int(1000 * g['on_next']))
-
- ## average (per driver call) overhead/duration
- #
- self.add_formula(
- 'cases_hacked',
- lambda g, d: round(100 * float(g['hacked_cases']) / g['cases'], 2)
- )
-
- def _computed_stats(self):
- computed = dict.fromkeys(self.formulas.keys())
- for fname, fml in self.formulas.iteritems():
- try:
- v = fml(self.generic_stats, self.driver_stats)
- except ZeroDivisionError:
- v = None
- computed[fname] = v
- return computed
-
- def add_formula(self, vname, formula):
- """Add a function to work with generic_stats, driver_stats."""
- self.formulas[vname] = formula
-
- def add(self, vname, value):
- """Add a value to generic stat counter."""
- if vname in self.generic_stats:
- self.generic_stats[vname] += value
- else:
- self.generic_stats[vname] = value
-
- def add_for(self, dclass, vname, value):
- """Add a value to driver stat counter."""
- dname = dclass.__name__
- if dname not in self.driver_stats:
- self._register(dname)
- if vname in self.driver_stats[dname]:
- self.driver_stats[dname][vname] += value
- else:
- self.driver_stats[dname][vname] = value
-
- def count(self, vname):
- """Alias to add(vname, 1)"""
- self.add(vname, 1)
-
- def count_for(self, dclass, vname):
- """Alias to add_for(vname, 1)"""
- self.add_for(dclass, vname, 1)
-
- def all_stats(self):
- """Compute stats from formulas and add them to colledted data."""
- stats = self.generic_stats
- for dname, dstats in self.driver_stats.iteritems():
- for key, value in dstats.iteritems():
- stats[dname + "_" + key] = value
- stats.update(self._computed_stats())
- return stats
-
-
- class Tracker(dict):
- """Error tracker to allow for usable reports from huge regression tests.
-
- Best used as a result bearer from `regression_test`, this class keeps
- a simple in-memory "database" of errors seen during the regression
- test, and implements few methods to access the data.
-
- The basic usage is:
-
- 1. Instantiate (no parameters)
-
- 2. Each time you have a result of a test, you pass it to `update()`
- method along with the argument set (as a single object, typically
- a dict) that caused the error.
-
- If boolean value of the result is False, the object is thrown away
- and nothing happen. Otherwise, its string value is used as a key
- under which the argument set is saved.
-
- As you can see, the string is supposed to be ''as deterministic
- as possible'', i.e. it should provide as little information
- about the error as is necessary. Do not include any timestamps
- or "volatile" values.
-
- 3. At final stage, you can retrieve statistics as how many (distinct)
- errors have been recorded, what was the duration of the whole test,
- how many times `update()` was called, etc.
-
- 4. Optionally, you can also call `format_report()` to get a nicely
- formatted report with list of arguments for each error string.
-
- 5. Since in bigger tests, argument lists can grow really large,
- complete lists are not normally printed. Instead, you can use
- `write_stats_csv()`, which will create one CSV per each error,
- named as first 7 chars of its SHA1 (inspired by Git).
-
- Note that you need to pass an existing writable folder path.
- """
-
- ##
- # internal methods
- #
-
- def __init__(self):
- self._start = time.time()
- self._db = {}
- self.tests_done = 0
- self.tests_passed = 0
- self.argsets_done = 0
- self.driver_stats = {}
-
- def _csv_fname(self, errstr, prefix):
- """Format name of file for this error string"""
- return '%s/%s.csv' % (prefix, self._eid(errstr))
-
- def _eid(self, errstr):
- """Return EID for the error string (first 7 chars of SHA1)."""
- return hashlib.sha1(errstr).hexdigest()[:7]
-
- def _insert(self, errstr, argset):
- """Insert the argset into DB."""
- if not errstr in self._db:
- self._db[errstr] = []
- self._db[errstr].append(argset)
-
- def _format_error(self, errstr, max_aa=0):
- """Format single error for output."""
- argsets_affected = self._db[errstr]
- num_aa = len(argsets_affected)
-
- # trim if list is too long for Jenkins
- argsets_shown = argsets_affected
- if max_aa and (num_aa > max_aa):
- div = ["[...] not showing %s cases, see %s.csv for full list"
- % (num_aa - max_aa, self._eid(errstr))]
- argsets_shown = argsets_affected[0:max_aa] + div
-
- # format error
- formatted_aa = "\n".join([str(arg) for arg in argsets_shown])
- return ("~~~ ERROR FOUND (%s) ~~~~~~~~~~~~~~~~~~~~~~~~~\n"
- "--- error string: -----------------------------------\n%s\n"
- "--- argsets affected (%d) ---------------------------\n%s\n"
- % (self._eid(errstr), errstr, num_aa, formatted_aa))
-
- ##
- # public methods
- #
-
- def errors_found(self):
- """Return number of non-distinct errors in db."""
- return bool(self._db)
-
- def format_report(self, max_aa=0):
- """Return complete report formatted as string."""
- error_list = "\n".join([self._format_error(e, max_aa=max_aa)
- for e in self._db])
- return ("Found %(total_errors)s (%(distinct_errors)s distinct) errors"
- " in %(tests_done)s tests with %(argsets)s argsets"
- " (duration: %(time)ss):"
- % self.getstats()
- + "\n\n" + error_list)
-
- def getstats(self):
- """Return basic and driver stats
-
- argsets_done - this should must be raised by outer code,
- once per each unique argset
- tests_done - how many times Tracker.update() was called
- distinct_errors - how many distinct errors (same `str(error)`)
- were seen by Tracker.update()
- total_errors - how many times `Tracker.update()` saw an
- error, i.e. how many argsets are in DB
- time - how long since init (seconds)
- """
-
- def total_errors():
- return reduce(lambda x, y: x + len(y), self._db.values(), 0)
-
- stats = {
- "argsets": self.argsets_done,
- "tests_done": self.tests_done,
- "distinct_errors": len(self._db),
- "total_errors": total_errors(),
- "time": int(time.time() - self._start)
- }
- stats.update(self.driver_stats)
- return stats
-
- def update(self, error, argset):
- """Update tracker with test result.
-
- If `bool(error)` is true, it is considered error and argset
- is inserted to DB with `str(error)` as key. This allows for later
- sorting and analysis.
- """
- self.tests_done += 1
- if error:
- errstr = str(error)
- self._insert(errstr, argset)
-
- def write_stats_csv(self, fname):
- """Write stats to a simple one row (plus header) CSV."""
- stats = self.getstats()
- colnames = sorted(stats.keys())
- with open(fname, 'a') as fh:
- cw = csv.DictWriter(fh, colnames)
- cw.writerow(dict(zip(colnames, colnames))) # header
- cw.writerow(stats)
-
- def write_args_csv(self, prefix=''):
- """Write out a set of CSV files, one per distinctive error.
-
- Each CSV is named with error EID (first 7 chars of SHA1) and lists
- all argument sets affected by this error. This is supposed to make
- easier to further analyse impact and trigerring values of errors,
- perhaps using a table processor software."""
-
- def get_all_colnames():
- cn = {}
- for errstr, affected in self._db.iteritems():
- for argset in affected:
- cn.update(dict.fromkeys(argset.keys()))
- return sorted(cn.keys())
-
- all_colnames = get_all_colnames()
-
- for errstr in self._db:
- with open(self._csv_fname(errstr, prefix), 'a') as fh:
- cw = csv.DictWriter(fh, all_colnames)
- cw.writerow(dict(zip(all_colnames, all_colnames))) # header
- for argset in self._db[errstr]:
- cw.writerow(argset)
-
-
- ## ............................................................................
- ## dataMatch -- a data structure matcher
- ##
- #
-
- def dataMatch(pattern, data, rmax=10, _r=0):
- """Check if data structure matches a pattern data structure.
-
- Supports lists, dictionaries and scalars (int, float, string).
-
- For scalars, simple `==` is used. Lists are converted to sets and
- "to match" means "to have a matching subset (e.g. `[1, 2, 3, 4]`
- matches `[3, 2]`). Both lists and dictionaries are matched recursively.
- """
-
- def listMatch(pattern, data):
- """Match list-like objects"""
- assert all([hasattr(o, 'append') for o in [pattern, data]])
- results = []
- for pv in pattern:
- if any([dataMatch(pv, dv, _r=_r+1) for dv in data]):
- results.append(True)
- else:
- results.append(False)
- return all(results)
-
- def dictMatch(pattern, data):
- """Match dict-like objects"""
- assert all([hasattr(o, 'iteritems') for o in [pattern, data]])
- results = []
- try:
- for pk, pv in pattern.iteritems():
- results.append(dataMatch(pv, data[pk], _r=_r+1))
- except KeyError:
- results.append(False)
- return all(results)
-
- if _r == rmax:
- raise RuntimeError("recursion limit hit")
-
- result = None
- if pattern == data:
- result = True
- else:
- for handler in [dictMatch, listMatch]:
- try:
- result = handler(pattern, data)
- except AssertionError:
- continue
- return result
-
-
- def jsDump(data):
- """A human-readable JSON dump."""
- return json.dumps(data, sort_keys=True, indent=4,
- separators=(',', ': '))
-
-
- def jsDiff(dira, dirb, namea="A", nameb="B", chara="a", charb="b"):
- """JSON-based human-readable diff of two data structures.
-
- '''BETA''' version.
-
- jsDiff is based on unified diff of two human-readable JSON dumps except
- that instead of showing line numbers and context based on proximity to
- the changed lines, it prints only context important from the data
- structure point.
-
- The goal is to be able to quickly tell the story of what has changed
- where in the structure, no matter size and complexity of the data set.
-
- For example:
-
- a = {
- 'w': {1: 2, 3: 4},
- 'x': [1, 2, 3],
- 'y': [3, 1, 2]
- }
- b = {
- 'w': {1: 2, 3: 4},
- 'x': [1, 1, 3],
- 'y': [3, 1, 3]
- }
- print jsDiff(a, b)
-
- will output:
-
- aaa ~/A
- "x": [
- a 2,
- "y": [
- a 2
- bbb ~/B
- "x": [
- b 1,
- "y": [
- b 3
-
- Notice that the final output somehow resembles the traditional unified
- diff, so to avoid confusion, +/- is changed to a/b (the characters can
- be provided as well as the names A/B).
- """
-
- def compress(lines):
-
- def is_body(line):
- return line.startswith(("-", "+", " "))
-
- def is_diff(line):
- return line.startswith(("-", "+"))
-
- def is_diffA(line):
- return line.startswith("-")
-
- def is_diffB(line):
- return line.startswith("+")
-
- def is_context(line):
- return line.startswith(" ")
-
- def is_hdr(line):
- return line.startswith(("@@", "---", "+++"))
-
- def is_hdr_hunk(line):
- return line.startswith("@@")
-
- def is_hdr_A(line):
- return line.startswith("---")
-
- def is_hdr_B(line):
- return line.startswith("+++")
-
- class Level(object):
-
- def __init__(self, hint):
- self.hint = hint
- self.hinted = False
-
- def __str__(self):
- return str(self.hint)
-
- def get_hint(self):
- if not self.hinted:
- self.hinted = True
- return self.hint
-
- class ContextTracker(object):
-
- def __init__(self):
- self.trace = []
- self.last_line = None
- self.last_indent = -1
-
- def indent_of(self, line):
- meat = line[1:].lstrip(" ")
- ind = len(line) - len(meat) - 1
- return ind
-
- def check(self, line):
- indent = self.indent_of(line)
- if indent > self.last_indent:
- self.trace.append(Level(self.last_line))
- elif indent < self.last_indent:
- self.trace.pop()
- self.last_line = line
- self.last_indent = indent
-
- def get_hint(self):
- return self.trace[-1].get_hint()
-
- buffa = []
- buffb = []
- ct = ContextTracker()
-
- for line in lines:
-
- if is_hdr_hunk(line):
- continue
- elif is_hdr_A(line):
- line = line.replace("---", chara * 3, 1)
- buffa.insert(0, line)
- elif is_hdr_B(line):
- line = line.replace("+++", charb * 3, 1)
- buffb.insert(0, line)
-
- elif is_body(line):
-
- ct.check(line)
-
- if is_diff(line):
- hint = ct.get_hint()
- if hint:
- buffa.append(hint)
- buffb.append(hint)
-
- if is_diffA(line):
- line = line.replace("-", chara, 1)
- buffa.append(line)
-
- elif is_diffB(line):
- line = line.replace("+", charb, 1)
- buffb.append(line)
-
- else:
- raise AssertionError("difflib.unified_diff emited"
- " unknown format (%s chars):\n%s"
- % (len(line), line))
-
- return buffa + buffb
-
- dumpa = jsDump(dira)
- dumpb = jsDump(dirb)
- udiff = difflib.unified_diff(dumpa.split("\n"), dumpb.split("\n"),
- "~/" + namea, "~/" + nameb,
- n=10000, lineterm='')
-
- return "\n".join(compress([line for line in udiff]))
-
-
- #
- ## Cartman - create dict arguments from dicts of available values (iterables)
- # and a defined scheme
- #
-
- class Cartman(object):
- """Create argument sets from ranges (or ay iterators) of values.
-
- This class is to enable easy definition and generation of dictionary
- argument sets using Cartesian product. You only need to define:
-
- * structure of argument set (can be more than just flat dict)
-
- * ranges, or arbitrary iterators of values on each "leaf" of the
- argument set
-
- Since there is expectation that any argument can have any kind of values
- even another iterables, the pure logic "iterate it if you can"
- is insufficient. Instead, definition is divided in two parts:
-
- * scheme, which is a "prototype" of a final argument set, except
- that for each value that will change, a `Cartman.Iterable`
- sentinel is used. For each leaf that is constant, `Cartman.Scalar`
- is used
-
- * source, which has the same structure, except that where in scheme
- is `Iterable`, an iterable object is expected, whereas in places
- where `Scalar` is used, a value is assigned that does not change
- during iteration.
-
- Finally, when such instance is used in loop, argument sets are generated
- uising Cartesian product of each iterable found. This allows for
- relatively easy definition of complex scenarios.
-
- Consider this example:
-
- You have a system (wrapped up in test driver) that takes ''size''
- argument, that is supposed to be ''width'', ''height'' and ''depth'',
- each an integer ranging from 1 to 100, and ''color'' that can
- be "white", "black" or "yellow".
-
- For a test using all-combinations strategy, you will need to generate
- 100 * 100 * 100 * 3 argument sets, i.e. 3M tests.
-
- All you need to do is:
-
- scheme = {
- 'size': {
- 'width': Cartman.Iterable,
- 'height': Cartman.Iterable,
- 'depth': Cartman.Iterable,
- }
- 'color': Cartman.Iterable,
- }
-
- source = {
- 'size': {
- 'width': range(1, 100),
- 'height': range(1, 100),
- 'depth': range(1, 100),
- }
- 'color': ['white', 'black', 'yellow'],
- }
-
- c = Cartman(source, scheme)
-
- for argset in c:
- result = my_test(argset)
- # assert ...
-
- The main advantage is that you can separate the definition from
- the code, and you can keep yor iterators as big or as small as
- needed, and add / remove values.
-
- Also in case your parameters vary in structure over time, or from
- one test to another, it gets much easier to keep up with changes
- without much jumping through hoops.
-
- Note: `Cartman.Scalar` is provided mainly to make your definitions
- more readable. Following constructions are functionally equal:
-
- c = Cartman({'a': 1}, {'a': Cartman.Scalar})
- c = Cartman({'a': [1]}, {'a': Cartman.Iterable})
-
- In future, however, this might change, though, mainly in case
- optimization became possible based on what was used.
- """
-
-
- # TODO: support for arbitrary ordering (profile / nginx)
- # TODO: implement getstats and fmtstats
- # TODO: N-wise
-
- class _BaseMark(object):
- pass
-
- class Scalar(_BaseMark):
- pass
-
- class Iterable(_BaseMark):
- pass
-
- def __init__(self, source, scheme, recursion_limit=10, _r=0):
- self.source = source
- self.scheme = scheme
- self.recursion_limit = recursion_limit
- self._r = _r
- if self._r > self.recursion_limit:
- raise RuntimeError("recursion limit exceeded")
-
- # validate scheme + source and throw useful error
- scheme_ok = isinstance(self.scheme, collections.Mapping)
- source_ok = isinstance(self.source, collections.Mapping)
- if not scheme_ok:
- raise ValueError("scheme must be a mapping (e.g. dict)")
- elif scheme_ok and not source_ok:
- raise ValueError("scheme vs. source mismatch")
-
- def __deepcopy__(self, memo):
- return Cartman(deepcopy(self.source, memo),
- deepcopy(self.scheme, memo))
-
- def _is_mark(self, subscheme):
- try:
- return issubclass(subscheme, Cartman._BaseMark)
- except TypeError:
- return False
-
- def _means_scalar(self, subscheme):
- if self._is_mark(subscheme):
- return issubclass(subscheme, Cartman.Scalar)
-
- def _means_iterable(self, subscheme):
- if self._is_mark(subscheme):
- return issubclass(subscheme, Cartman.Iterable)
-
- def _get_iterable_for(self, key):
- subscheme = self.scheme[key]
- subsource = self.source[key]
- if self._means_scalar(subscheme):
- return [subsource]
- elif self._means_iterable(subscheme):
- return subsource
- else: # try to use it as scheme
- return iter(Cartman(subsource, subscheme, _r=self._r+1))
-
- def __iter__(self):
-
- names = []
- iterables = []
-
- keys = self.scheme.keys()
-
- for key in keys:
- try:
- iterables.append(self._get_iterable_for(key))
- except KeyError:
- pass # ignore that subsource mentioned by scheme is missing
- else:
- names.append(key)
-
- for values in itertools.product(*iterables):
- yield dict(zip(names, values))
-
- def getstats(self):
- return {}
-
- def fmtstats(self):
- return ""
|