Yi Kong | 7bf2a68 | 2019-04-18 17:30:49 -0700 | [diff] [blame^] | 1 | #!/usr/bin/env python |
| 2 | |
| 3 | from __future__ import print_function |
| 4 | |
| 5 | import yaml |
| 6 | # Try to use the C parser. |
| 7 | try: |
| 8 | from yaml import CLoader as Loader |
| 9 | except ImportError: |
| 10 | print("For faster parsing, you may want to install libYAML for PyYAML") |
| 11 | from yaml import Loader |
| 12 | |
| 13 | import cgi |
| 14 | from collections import defaultdict |
| 15 | import fnmatch |
| 16 | import functools |
| 17 | from multiprocessing import Lock |
| 18 | import os, os.path |
| 19 | import subprocess |
| 20 | try: |
| 21 | # The previously builtin function `intern()` was moved |
| 22 | # to the `sys` module in Python 3. |
| 23 | from sys import intern |
| 24 | except: |
| 25 | pass |
| 26 | |
| 27 | import re |
| 28 | |
| 29 | import optpmap |
| 30 | |
| 31 | try: |
| 32 | dict.iteritems |
| 33 | except AttributeError: |
| 34 | # Python 3 |
| 35 | def itervalues(d): |
| 36 | return iter(d.values()) |
| 37 | def iteritems(d): |
| 38 | return iter(d.items()) |
| 39 | else: |
| 40 | # Python 2 |
| 41 | def itervalues(d): |
| 42 | return d.itervalues() |
| 43 | def iteritems(d): |
| 44 | return d.iteritems() |
| 45 | |
| 46 | |
| 47 | def html_file_name(filename): |
| 48 | return filename.replace('/', '_').replace('#', '_') + ".html" |
| 49 | |
| 50 | |
| 51 | def make_link(File, Line): |
| 52 | return "\"{}#L{}\"".format(html_file_name(File), Line) |
| 53 | |
| 54 | |
| 55 | class Remark(yaml.YAMLObject): |
| 56 | # Work-around for http://pyyaml.org/ticket/154. |
| 57 | yaml_loader = Loader |
| 58 | |
| 59 | default_demangler = 'c++filt -n' |
| 60 | demangler_proc = None |
| 61 | |
| 62 | @classmethod |
| 63 | def set_demangler(cls, demangler): |
| 64 | cls.demangler_proc = subprocess.Popen(demangler.split(), stdin=subprocess.PIPE, stdout=subprocess.PIPE) |
| 65 | cls.demangler_lock = Lock() |
| 66 | |
| 67 | @classmethod |
| 68 | def demangle(cls, name): |
| 69 | with cls.demangler_lock: |
| 70 | cls.demangler_proc.stdin.write((name + '\n').encode('utf-8')) |
| 71 | cls.demangler_proc.stdin.flush() |
| 72 | return cls.demangler_proc.stdout.readline().rstrip().decode('utf-8') |
| 73 | |
| 74 | # Intern all strings since we have lot of duplication across filenames, |
| 75 | # remark text. |
| 76 | # |
| 77 | # Change Args from a list of dicts to a tuple of tuples. This saves |
| 78 | # memory in two ways. One, a small tuple is significantly smaller than a |
| 79 | # small dict. Two, using tuple instead of list allows Args to be directly |
| 80 | # used as part of the key (in Python only immutable types are hashable). |
| 81 | def _reduce_memory(self): |
| 82 | self.Pass = intern(self.Pass) |
| 83 | self.Name = intern(self.Name) |
| 84 | try: |
| 85 | # Can't intern unicode strings. |
| 86 | self.Function = intern(self.Function) |
| 87 | except: |
| 88 | pass |
| 89 | |
| 90 | def _reduce_memory_dict(old_dict): |
| 91 | new_dict = dict() |
| 92 | for (k, v) in iteritems(old_dict): |
| 93 | if type(k) is str: |
| 94 | k = intern(k) |
| 95 | |
| 96 | if type(v) is str: |
| 97 | v = intern(v) |
| 98 | elif type(v) is dict: |
| 99 | # This handles [{'Caller': ..., 'DebugLoc': { 'File': ... }}] |
| 100 | v = _reduce_memory_dict(v) |
| 101 | new_dict[k] = v |
| 102 | return tuple(new_dict.items()) |
| 103 | |
| 104 | self.Args = tuple([_reduce_memory_dict(arg_dict) for arg_dict in self.Args]) |
| 105 | |
| 106 | # The inverse operation of the dictonary-related memory optimization in |
| 107 | # _reduce_memory_dict. E.g. |
| 108 | # (('DebugLoc', (('File', ...) ... ))) -> [{'DebugLoc': {'File': ...} ....}] |
| 109 | def recover_yaml_structure(self): |
| 110 | def tuple_to_dict(t): |
| 111 | d = dict() |
| 112 | for (k, v) in t: |
| 113 | if type(v) is tuple: |
| 114 | v = tuple_to_dict(v) |
| 115 | d[k] = v |
| 116 | return d |
| 117 | |
| 118 | self.Args = [tuple_to_dict(arg_tuple) for arg_tuple in self.Args] |
| 119 | |
| 120 | def canonicalize(self): |
| 121 | if not hasattr(self, 'Hotness'): |
| 122 | self.Hotness = 0 |
| 123 | if not hasattr(self, 'Args'): |
| 124 | self.Args = [] |
| 125 | self._reduce_memory() |
| 126 | |
| 127 | @property |
| 128 | def File(self): |
| 129 | return self.DebugLoc['File'] |
| 130 | |
| 131 | @property |
| 132 | def Line(self): |
| 133 | return int(self.DebugLoc['Line']) |
| 134 | |
| 135 | @property |
| 136 | def Column(self): |
| 137 | return self.DebugLoc['Column'] |
| 138 | |
| 139 | @property |
| 140 | def DebugLocString(self): |
| 141 | return "{}:{}:{}".format(self.File, self.Line, self.Column) |
| 142 | |
| 143 | @property |
| 144 | def DemangledFunctionName(self): |
| 145 | return self.demangle(self.Function) |
| 146 | |
| 147 | @property |
| 148 | def Link(self): |
| 149 | return make_link(self.File, self.Line) |
| 150 | |
| 151 | def getArgString(self, mapping): |
| 152 | mapping = dict(list(mapping)) |
| 153 | dl = mapping.get('DebugLoc') |
| 154 | if dl: |
| 155 | del mapping['DebugLoc'] |
| 156 | |
| 157 | assert(len(mapping) == 1) |
| 158 | (key, value) = list(mapping.items())[0] |
| 159 | |
| 160 | if key == 'Caller' or key == 'Callee' or key == 'DirectCallee': |
| 161 | value = cgi.escape(self.demangle(value)) |
| 162 | |
| 163 | if dl and key != 'Caller': |
| 164 | dl_dict = dict(list(dl)) |
| 165 | return u"<a href={}>{}</a>".format( |
| 166 | make_link(dl_dict['File'], dl_dict['Line']), value) |
| 167 | else: |
| 168 | return value |
| 169 | |
| 170 | # Return a cached dictionary for the arguments. The key for each entry is |
| 171 | # the argument key (e.g. 'Callee' for inlining remarks. The value is a |
| 172 | # list containing the value (e.g. for 'Callee' the function) and |
| 173 | # optionally a DebugLoc. |
| 174 | def getArgDict(self): |
| 175 | if hasattr(self, 'ArgDict'): |
| 176 | return self.ArgDict |
| 177 | self.ArgDict = {} |
| 178 | for arg in self.Args: |
| 179 | if len(arg) == 2: |
| 180 | if arg[0][0] == 'DebugLoc': |
| 181 | dbgidx = 0 |
| 182 | else: |
| 183 | assert(arg[1][0] == 'DebugLoc') |
| 184 | dbgidx = 1 |
| 185 | |
| 186 | key = arg[1 - dbgidx][0] |
| 187 | entry = (arg[1 - dbgidx][1], arg[dbgidx][1]) |
| 188 | else: |
| 189 | arg = arg[0] |
| 190 | key = arg[0] |
| 191 | entry = (arg[1], ) |
| 192 | |
| 193 | self.ArgDict[key] = entry |
| 194 | return self.ArgDict |
| 195 | |
| 196 | def getDiffPrefix(self): |
| 197 | if hasattr(self, 'Added'): |
| 198 | if self.Added: |
| 199 | return '+' |
| 200 | else: |
| 201 | return '-' |
| 202 | return '' |
| 203 | |
| 204 | @property |
| 205 | def PassWithDiffPrefix(self): |
| 206 | return self.getDiffPrefix() + self.Pass |
| 207 | |
| 208 | @property |
| 209 | def message(self): |
| 210 | # Args is a list of mappings (dictionaries) |
| 211 | values = [self.getArgString(mapping) for mapping in self.Args] |
| 212 | return "".join(values) |
| 213 | |
| 214 | @property |
| 215 | def RelativeHotness(self): |
| 216 | if self.max_hotness: |
| 217 | return "{0:.2f}%".format(self.Hotness * 100. / self.max_hotness) |
| 218 | else: |
| 219 | return '' |
| 220 | |
| 221 | @property |
| 222 | def key(self): |
| 223 | return (self.__class__, self.PassWithDiffPrefix, self.Name, self.File, |
| 224 | self.Line, self.Column, self.Function, self.Args) |
| 225 | |
| 226 | def __hash__(self): |
| 227 | return hash(self.key) |
| 228 | |
| 229 | def __eq__(self, other): |
| 230 | return self.key == other.key |
| 231 | |
| 232 | def __repr__(self): |
| 233 | return str(self.key) |
| 234 | |
| 235 | |
| 236 | class Analysis(Remark): |
| 237 | yaml_tag = '!Analysis' |
| 238 | |
| 239 | @property |
| 240 | def color(self): |
| 241 | return "white" |
| 242 | |
| 243 | |
| 244 | class AnalysisFPCommute(Analysis): |
| 245 | yaml_tag = '!AnalysisFPCommute' |
| 246 | |
| 247 | |
| 248 | class AnalysisAliasing(Analysis): |
| 249 | yaml_tag = '!AnalysisAliasing' |
| 250 | |
| 251 | |
| 252 | class Passed(Remark): |
| 253 | yaml_tag = '!Passed' |
| 254 | |
| 255 | @property |
| 256 | def color(self): |
| 257 | return "green" |
| 258 | |
| 259 | |
| 260 | class Missed(Remark): |
| 261 | yaml_tag = '!Missed' |
| 262 | |
| 263 | @property |
| 264 | def color(self): |
| 265 | return "red" |
| 266 | |
| 267 | |
| 268 | def get_remarks(input_file, filter_): |
| 269 | max_hotness = 0 |
| 270 | all_remarks = dict() |
| 271 | file_remarks = defaultdict(functools.partial(defaultdict, list)) |
| 272 | |
| 273 | with open(input_file) as f: |
| 274 | docs = yaml.load_all(f, Loader=Loader) |
| 275 | |
| 276 | filter_e = re.compile(filter_) |
| 277 | for remark in docs: |
| 278 | remark.canonicalize() |
| 279 | # Avoid remarks withoug debug location or if they are duplicated |
| 280 | if not hasattr(remark, 'DebugLoc') or remark.key in all_remarks: |
| 281 | continue |
| 282 | |
| 283 | if filter_ and not filter_e.search(remark.Pass): |
| 284 | continue |
| 285 | |
| 286 | all_remarks[remark.key] = remark |
| 287 | |
| 288 | file_remarks[remark.File][remark.Line].append(remark) |
| 289 | |
| 290 | # If we're reading a back a diff yaml file, max_hotness is already |
| 291 | # captured which may actually be less than the max hotness found |
| 292 | # in the file. |
| 293 | if hasattr(remark, 'max_hotness'): |
| 294 | max_hotness = remark.max_hotness |
| 295 | max_hotness = max(max_hotness, remark.Hotness) |
| 296 | |
| 297 | return max_hotness, all_remarks, file_remarks |
| 298 | |
| 299 | |
| 300 | def gather_results(filenames, num_jobs, should_print_progress, filter_): |
| 301 | if should_print_progress: |
| 302 | print('Reading YAML files...') |
| 303 | if not Remark.demangler_proc: |
| 304 | Remark.set_demangler(Remark.default_demangler) |
| 305 | remarks = optpmap.pmap( |
| 306 | get_remarks, filenames, num_jobs, should_print_progress, filter_) |
| 307 | max_hotness = max(entry[0] for entry in remarks) |
| 308 | |
| 309 | def merge_file_remarks(file_remarks_job, all_remarks, merged): |
| 310 | for filename, d in iteritems(file_remarks_job): |
| 311 | for line, remarks in iteritems(d): |
| 312 | for remark in remarks: |
| 313 | # Bring max_hotness into the remarks so that |
| 314 | # RelativeHotness does not depend on an external global. |
| 315 | remark.max_hotness = max_hotness |
| 316 | if remark.key not in all_remarks: |
| 317 | merged[filename][line].append(remark) |
| 318 | |
| 319 | all_remarks = dict() |
| 320 | file_remarks = defaultdict(functools.partial(defaultdict, list)) |
| 321 | for _, all_remarks_job, file_remarks_job in remarks: |
| 322 | merge_file_remarks(file_remarks_job, all_remarks, file_remarks) |
| 323 | all_remarks.update(all_remarks_job) |
| 324 | |
| 325 | return all_remarks, file_remarks, max_hotness != 0 |
| 326 | |
| 327 | |
| 328 | def find_opt_files(*dirs_or_files): |
| 329 | all = [] |
| 330 | for dir_or_file in dirs_or_files: |
| 331 | if os.path.isfile(dir_or_file): |
| 332 | all.append(dir_or_file) |
| 333 | else: |
| 334 | for dir, subdirs, files in os.walk(dir_or_file): |
| 335 | # Exclude mounted directories and symlinks (os.walk default). |
| 336 | subdirs[:] = [d for d in subdirs |
| 337 | if not os.path.ismount(os.path.join(dir, d))] |
| 338 | for file in files: |
| 339 | if fnmatch.fnmatch(file, "*.opt.yaml*"): |
| 340 | all.append(os.path.join(dir, file)) |
| 341 | return all |