API tools faq
paste
Advertisement
bos

tracing patch POC

bos
Feb 12th, 2017
212
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Diff 19.76 KB | None | 0 0
raw download clone embed print report
  1. diff --git a/mercurial/changegroup.py b/mercurial/changegroup.py
  2. --- a/mercurial/changegroup.py
  3. +++ b/mercurial/changegroup.py
  4. @@ -26,6 +26,7 @@ from . import (
  5.      error,
  6.      mdiff,
  7.      phases,
  8. +    tracing,
  9.      util,
  10.  )
  11.  
  12. @@ -298,6 +299,7 @@ class cg1unpacker(object):
  13.                  trp = weakref.proxy(tr)
  14.                  # pull off the changeset group
  15.                  repo.ui.status(_("adding changesets\n"))
  16. +                tracing.event('adding changesets', 'apply', 'B')
  17.                  clstart = len(cl)
  18.                  class prog(object):
  19.                      def __init__(self, step, total):
  20. @@ -324,11 +326,13 @@ class cg1unpacker(object):
  21.                  clend = len(cl)
  22.                  changesets = clend - clstart
  23.                  repo.ui.progress(_('changesets'), None)
  24. +                tracing.event('adding changesets', 'apply', 'E')
  25.                  self.callback = None
  26.  
  27.                  # pull off the manifest group
  28.                  repo.ui.status(_("adding manifests\n"))
  29. -                self._unpackmanifests(repo, revmap, trp, prog, changesets)
  30. +                with tracing.duration('adding manifests', 'apply'):
  31. +                    self._unpackmanifests(repo, revmap, trp, prog, changesets)
  32.  
  33.                  needfiles = {}
  34.                  if repo.ui.configbool('server', 'validate', default=False):
  35. @@ -344,10 +348,11 @@ class cg1unpacker(object):
  36.  
  37.                  # process the files
  38.                  repo.ui.status(_("adding file changes\n"))
  39. -                newrevs, newfiles = _addchangegroupfiles(
  40. -                    repo, self, revmap, trp, efiles, needfiles)
  41. -                revisions += newrevs
  42. -                files += newfiles
  43. +                with tracing.duration('adding file changes', 'apply'):
  44. +                    newrevs, newfiles = _addchangegroupfiles(
  45. +                        repo, self, revmap, trp, efiles, needfiles)
  46. +                    revisions += newrevs
  47. +                    files += newfiles
  48.  
  49.                  dh = 0
  50.                  if oldheads:
  51. @@ -376,6 +381,7 @@ class cg1unpacker(object):
  52.                          hookargs['node_last'] = hex(cl.node(clend - 1))
  53.                      repo.hook('pretxnchangegroup', throw=True, **hookargs)
  54.  
  55. +                tracing.event('phases', 'apply', 'B')
  56.                  added = [cl.node(r) for r in xrange(clstart, clend)]
  57.                  publishing = repo.publishing()
  58.                  if srctype in ('push', 'serve'):
  59. @@ -401,6 +407,7 @@ class cg1unpacker(object):
  60.                      #
  61.                      # strip should not touch boundary at all
  62.                      phases.retractboundary(repo, tr, targetphase, added)
  63. +                tracing.event('phases', 'apply', 'E')
  64.  
  65.                  if changesets > 0:
  66.                      if srctype != 'strip':
  67. @@ -409,7 +416,8 @@ class cg1unpacker(object):
  68.                          # In other case we can safely update cache on
  69.                          # disk.
  70.                          repo.ui.debug('updating the branch cache\n')
  71. -                        branchmap.updatecache(repo.filtered('served'))
  72. +                        with tracing.duration('updating branch cache', 'apply'):
  73. +                            branchmap.updatecache(repo.filtered('served'))
  74.  
  75.                      def runhooks():
  76.                          # These hooks run when the lock releases, not when the
  77. diff --git a/mercurial/commands.py b/mercurial/commands.py
  78. --- a/mercurial/commands.py
  79. +++ b/mercurial/commands.py
  80. @@ -65,6 +65,7 @@ from . import (
  81.      streamclone,
  82.      templatekw,
  83.      templater,
  84. +    tracing,
  85.      ui as uimod,
  86.      util,
  87.  )
  88. @@ -4597,8 +4598,9 @@ def log(ui, repo, *pats, **opts):
  89.      if opts.get('graph'):
  90.          return cmdutil.graphlog(ui, repo, *pats, **opts)
  91.  
  92. -    revs, expr, filematcher = cmdutil.getlogrevs(repo, pats, opts)
  93. -    limit = cmdutil.loglimit(opts)
  94. +    with tracing.duration('getlogrevs', 'log'):
  95. +        revs, expr, filematcher = cmdutil.getlogrevs(repo, pats, opts)
  96. +        limit = cmdutil.loglimit(opts)
  97.      count = 0
  98.  
  99.      getrenamed = None
  100. @@ -4609,6 +4611,7 @@ def log(ui, repo, *pats, **opts):
  101.          getrenamed = templatekw.getrenamedfn(repo, endrev=endrev)
  102.  
  103.      displayer = cmdutil.show_changeset(ui, repo, opts, buffered=True)
  104. +    tracing.event('displayer', 'log', 'B')
  105.      for rev in revs:
  106.          if count == limit:
  107.              break
  108. @@ -4627,6 +4630,7 @@ def log(ui, repo, *pats, **opts):
  109.          displayer.show(ctx, copies=copies, matchfn=revmatchfn)
  110.          if displayer.flush(ctx):
  111.              count += 1
  112. +    tracing.event('displayer', 'log', 'E')
  113.  
  114.      displayer.close()
  115.  
  116. diff --git a/mercurial/dirstate.py b/mercurial/dirstate.py
  117. --- a/mercurial/dirstate.py
  118. +++ b/mercurial/dirstate.py
  119. @@ -23,6 +23,7 @@ from . import (
  120.      pathutil,
  121.      pycompat,
  122.      scmutil,
  123. +    tracing,
  124.      util,
  125.  )
  126.  
  127. @@ -393,6 +394,7 @@ class dirstate(object):
  128.          self._pendingmode = mode
  129.          return fp
  130.  
  131. +    @tracing.call
  132.      def _read(self):
  133.          self._map = {}
  134.          self._copymap = {}
  135. @@ -938,6 +940,7 @@ class dirstate(object):
  136.  
  137.          return results, dirsfound, dirsnotfound
  138.  
  139. +    @tracing.call
  140.      def walk(self, match, subrepos, unknown, ignored, full=True):
  141.          '''
  142.          Walk recursively through the directory tree, finding all files
  143. @@ -1100,6 +1103,7 @@ class dirstate(object):
  144.                      results[nf()] = st
  145.          return results
  146.  
  147. +    @tracing.call
  148.      def status(self, match, subrepos, ignored, clean, unknown):
  149.          '''Determine the status of the working copy relative to the
  150.          dirstate and return a pair of (unsure, status), where status is of type
  151. diff --git a/mercurial/dispatch.py b/mercurial/dispatch.py
  152. --- a/mercurial/dispatch.py
  153. +++ b/mercurial/dispatch.py
  154. @@ -19,7 +19,6 @@ import sys
  155.  import time
  156.  import traceback
  157.  
  158. -
  159.  from .i18n import _
  160.  
  161.  from . import (
  162. @@ -42,6 +41,7 @@ from . import (
  163.      templatefilters,
  164.      templatekw,
  165.      templater,
  166. +    tracing,
  167.      ui as uimod,
  168.      util,
  169.  )
  170. @@ -652,12 +652,18 @@ def _dispatch(req):
  171.      if cwd:
  172.          os.chdir(cwd[-1])
  173.  
  174. +    tracefile = _earlygetopt(['--trace'], args)
  175. +    if tracefile:
  176. +        tracing.setup(tracing.tracing(open(tracefile[-1], 'w')))
  177. +    tracing.event('startup', 'startup', 'B')
  178. +
  179.      rpath = _earlygetopt(["-R", "--repository", "--repo"], args)
  180.      path, lui = _getlocal(ui, rpath)
  181.  
  182.      # Configure extensions in phases: uisetup, extsetup, cmdtable, and
  183.      # reposetup. Programs like TortoiseHg will call _dispatch several
  184.      # times so we keep track of configured extensions in _loaded.
  185. +    tracing.event('load extensions', 'extensions,startup', 'B')
  186.      extensions.loadall(lui)
  187.      exts = [ext for ext in extensions.extensions() if ext[0] not in _loaded]
  188.      # Propagate any changes to lui.__class__ by extensions
  189. @@ -671,9 +677,11 @@ def _dispatch(req):
  190.              if extraobj is not None:
  191.                  getattr(loadermod, loadername)(ui, name, extraobj)
  192.          _loaded.add(name)
  193. +    tracing.event('load extensions', 'extensions,startup', 'E')
  194.  
  195.      # (reposetup is handled in hg.repository)
  196.  
  197. +    tracing.event('aliases', 'extensions,startup', 'B')
  198.      # Side-effect of accessing is debugcommands module is guaranteed to be
  199.      # imported and commands.table is populated.
  200.      debugcommands.command
  201. @@ -683,6 +691,7 @@ def _dispatch(req):
  202.      # All aliases and commands are completely defined, now.
  203.      # Check abbreviation/ambiguity of shell alias.
  204.      shellaliasfn = _checkshellalias(lui, ui, args)
  205. +    tracing.event('aliases', 'extensions,startup', 'E')
  206.      if shellaliasfn:
  207.          with profiling.maybeprofile(lui):
  208.              return shellaliasfn()
  209. @@ -692,6 +701,8 @@ def _dispatch(req):
  210.      if fallback:
  211.          encoding.fallbackencoding = fallback
  212.  
  213. +    tracing.event('options', 'startup', 'B')
  214. +
  215.      fullargs = args
  216.      cmd, func, args, options, cmdoptions = _parse(lui, args)
  217.  
  218. @@ -755,6 +766,9 @@ def _dispatch(req):
  219.      elif not cmd:
  220.          return commands.help_(ui, 'shortlist')
  221.  
  222. +    tracing.event('options', 'startup', 'E')
  223. +    tracing.event('reposetup', 'startup', 'B')
  224. +
  225.      with profiling.maybeprofile(lui):
  226.          repo = None
  227.          cmdpats = args[:]
  228. @@ -802,14 +816,19 @@ def _dispatch(req):
  229.          elif rpath:
  230.              ui.warn(_("warning: --repository ignored\n"))
  231.  
  232. +        tracing.event('reposetup', 'startup', 'E')
  233. +
  234.          msg = ' '.join(' ' in a and repr(a) or a for a in fullargs)
  235.          ui.log("command", '%s\n', msg)
  236.          strcmdopt = pycompat.strkwargs(cmdoptions)
  237.          d = lambda: util.checksignature(func)(ui, *args, **strcmdopt)
  238.          try:
  239. -            return runcommand(lui, repo, cmd, fullargs, ui, options, d,
  240. -                              cmdpats, cmdoptions)
  241. +            tracing.event('startup', 'startup', 'E')
  242. +            with tracing.duration('run command ' + cmd, 'execution'):
  243. +                return runcommand(lui, repo, cmd, fullargs, ui, options, d,
  244. +                                  cmdpats, cmdoptions)
  245.          finally:
  246. +            tracing.close()
  247.              if repo and repo != req.repo:
  248.                  repo.close()
  249.  
  250. diff --git a/mercurial/extensions.py b/mercurial/extensions.py
  251. --- a/mercurial/extensions.py
  252. +++ b/mercurial/extensions.py
  253. @@ -19,6 +19,7 @@ from . import (
  254.      cmdutil,
  255.      error,
  256.      pycompat,
  257. +    tracing,
  258.      util,
  259.  )
  260.  
  261. @@ -163,7 +164,8 @@ def loadall(ui):
  262.                  _disabledextensions[name] = path[1:]
  263.                  continue
  264.          try:
  265. -            load(ui, name, path)
  266. +            with tracing.duration('load ' + name, 'extensions,startup'):
  267. +                load(ui, name, path)
  268.          except KeyboardInterrupt:
  269.              raise
  270.          except Exception as inst:
  271. diff --git a/mercurial/hook.py b/mercurial/hook.py
  272. --- a/mercurial/hook.py
  273. +++ b/mercurial/hook.py
  274. @@ -17,6 +17,7 @@ from . import (
  275.      error,
  276.      extensions,
  277.      pycompat,
  278. +    tracing,
  279.      util,
  280.  )
  281.  
  282. @@ -188,19 +189,20 @@ def redirect(state):
  283.      _redirect = state
  284.  
  285.  def hook(ui, repo, name, throw=False, **args):
  286. -    if not ui.callhooks:
  287. -        return False
  288. +    with tracing.duration(name, 'hook'):
  289. +        if not ui.callhooks:
  290. +            return False
  291.  
  292. -    hooks = []
  293. -    for hname, cmd in _allhooks(ui):
  294. -        if hname.split('.')[0] == name and cmd:
  295. -            hooks.append((hname, cmd))
  296. +        hooks = []
  297. +        for hname, cmd in _allhooks(ui):
  298. +            if hname.split('.')[0] == name and cmd:
  299. +                hooks.append((hname, cmd))
  300.  
  301. -    res = runhooks(ui, repo, name, hooks, throw=throw, **args)
  302. -    r = False
  303. -    for hname, cmd in hooks:
  304. -        r = res[hname][0] or r
  305. -    return r
  306. +        res = runhooks(ui, repo, name, hooks, throw=throw, **args)
  307. +        r = False
  308. +        for hname, cmd in hooks:
  309. +            r = res[hname][0] or r
  310. +        return r
  311.  
  312.  def runhooks(ui, repo, name, hooks, throw=False, **args):
  313.      res = {}
  314. diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
  315. --- a/mercurial/localrepo.py
  316. +++ b/mercurial/localrepo.py
  317. @@ -54,6 +54,7 @@ from . import (
  318.      store,
  319.      subrepo,
  320.      tags as tagsmod,
  321. +    tracing,
  322.      transaction,
  323.      util,
  324.  )
  325. @@ -507,6 +508,7 @@ class localrepository(object):
  326.          return store
  327.  
  328.      @storecache('00changelog.i')
  329. +    @tracing.call
  330.      def changelog(self):
  331.          c = changelog.changelog(self.svfs)
  332.          if 'HG_PENDING' in encoding.environ:
  333. @@ -522,10 +524,12 @@ class localrepository(object):
  334.          return manifest.manifestrevlog(self.svfs)
  335.  
  336.      @storecache('00manifest.i')
  337. +    @tracing.call
  338.      def manifestlog(self):
  339.          return manifest.manifestlog(self.svfs, self)
  340.  
  341.      @repofilecache('dirstate')
  342. +    @tracing.call
  343.      def dirstate(self):
  344.          return dirstate.dirstate(self.vfs, self.ui, self.root,
  345.                                   self._dirstatevalidate)
  346. @@ -1040,6 +1044,7 @@ class localrepository(object):
  347.          idbase = "%.40f#%f" % (random.random(), time.time())
  348.          txnid = 'TXN:' + hashlib.sha1(idbase).hexdigest()
  349.          self.hook('pretxnopen', throw=True, txnname=desc, txnid=txnid)
  350. +        tracing.event('txn-' + desc, 'transaction', 'B')
  351.  
  352.          self._writejournal(desc)
  353.          renames = [(vfs, x, undoname(x)) for vfs, x in self._journalfiles()]
  354. @@ -1095,6 +1100,7 @@ class localrepository(object):
  355.              def hook():
  356.                  reporef().hook('txnclose', throw=False, txnname=desc,
  357.                                 **hookargs)
  358. +                tracing.event('txn-' + desc, 'transaction', 'E')
  359.              reporef()._afterlock(hook)
  360.          tr.addfinalize('txnclose-hook', txnclosehook)
  361.          def txnaborthook(tr2):
  362. @@ -1102,6 +1108,7 @@ class localrepository(object):
  363.              """
  364.              reporef().hook('txnabort', throw=False, txnname=desc,
  365.                             **tr2.hookargs)
  366. +            tracing.event('txn-' + desc, 'transaction', 'E')
  367.          tr.addabort('txnabort-hook', txnaborthook)
  368.          # avoid eager cache invalidation. in-memory data should be identical
  369.          # to stored data if transaction has no error.
  370. diff --git a/mercurial/match.py b/mercurial/match.py
  371. --- a/mercurial/match.py
  372. +++ b/mercurial/match.py
  373. @@ -15,11 +15,13 @@ from .i18n import _
  374.  from . import (
  375.      error,
  376.      pathutil,
  377. +    tracing,
  378.      util,
  379.  )
  380.  
  381.  propertycache = util.propertycache
  382.  
  383. +@tracing.call
  384.  def _rematcher(regex):
  385.      '''compile the regexp with the best available regexp engine and return a
  386.      matcher function'''
  387. @@ -550,6 +552,7 @@ def _regex(kind, pat, globsuffix):
  388.          return '.*' + pat
  389.      return _globre(pat) + globsuffix
  390.  
  391. +@tracing.call
  392.  def _buildmatch(ctx, kindpats, globsuffix, listsubrepos, root):
  393.      '''Return regexp string and a matcher function for kindpats.
  394.      globsuffix is appended to the regexp of globs.'''
  395. diff --git a/mercurial/obsolete.py b/mercurial/obsolete.py
  396. --- a/mercurial/obsolete.py
  397. +++ b/mercurial/obsolete.py
  398. @@ -79,6 +79,7 @@ from . import (
  399.      node,
  400.      parsers,
  401.      phases,
  402. +    tracing,
  403.      util,
  404.  )
  405.  
  406. @@ -636,13 +637,15 @@ class obsstore(object):
  407.          return self.add(transaction, markers)
  408.  
  409.      @propertycache
  410. +    @tracing.call
  411.      def _all(self):
  412.          data = self.svfs.tryread('obsstore')
  413.          if not data:
  414.              return []
  415.          self._version, markers = _readmarkers(data)
  416.          markers = list(markers)
  417. -        _checkinvalidmarkers(markers)
  418. +        with tracing.duration('check invalid markers', 'repo'):
  419. +            _checkinvalidmarkers(markers)
  420.          return markers
  421.  
  422.      @propertycache
  423. diff --git a/mercurial/tracing.py b/mercurial/tracing.py
  424. new file mode 100644
  425. --- /dev/null
  426. +++ b/mercurial/tracing.py
  427. @@ -0,0 +1,122 @@
  428. +# tracing.py - performance tracing support
  429. +#
  430. +# Copyright (C) 2016 Facebook, Inc. <bryano@fb.com>
  431. +#
  432. +# This software may be used and distributed according to the terms of the
  433. +# GNU General Public License version 2 or any later version.
  434. +
  435. +from __future__ import absolute_import
  436. +
  437. +import contextlib
  438. +import inspect
  439. +import json
  440. +import os
  441. +
  442. +from . import (
  443. +    util,
  444. +    )
  445. +
  446. +_encode = None
  447. +
  448. +class tracing(object):
  449. +    def __init__(self, fp):
  450. +        global _encode
  451. +        if _encode is None:
  452. +            _encode = json.encoder.encode_basestring_ascii
  453. +        self.lastcount = self.starttime = util.timer()
  454. +        self.logfp = fp
  455. +        self.pid = os.getpid()
  456. +        self.logfp.write('[\n{"name":"tracing", "cat":"tracing", "ph":"B", '
  457. +                         '"pid":%d, "ts":0}' % (self.pid,))
  458. +        self.threshold = 1e-3
  459. +
  460. +    @contextlib.contextmanager
  461. +    def duration(self, name, category, **kwargs):
  462. +        start = util.timer()
  463. +        try:
  464. +            yield
  465. +        finally:
  466. +            end = util.timer()
  467. +            if end - start >= self.threshold:
  468. +                if kwargs:
  469. +                    args = ',\n "args":' + json.dumps(kwargs)
  470. +                else:
  471. +                    args = ''
  472. +                self.logfp.write(',\n{"name":%s, "cat":%s,\n "ph":"X", '
  473. +                                 '"pid":%d, "ts":%r, "dur":%r%s}' %
  474. +                                 (_encode(name), _encode(category), self.pid,
  475. +                                  (start - self.starttime) * 1e6,
  476. +                                  (end - start) * 1e6, args))
  477. +
  478. +    def event(self, name, category, phase, timestamp=None, **kwargs):
  479. +        if timestamp is None:
  480. +            timestamp = util.timer()
  481. +        if kwargs:
  482. +            args = ',\n "args":' + json.dumps(kwargs)
  483. +        else:
  484. +            args = ''
  485. +        self.logfp.write(',\n{"name":%s, "cat":%s,\n "ph":"%c", "pid":%d, '
  486. +                         '"ts":%r%s}' %
  487. +                         (_encode(name), _encode(category), phase, self.pid,
  488. +                          (timestamp - self.starttime) * 1e6, args))
  489. +
  490. +    def progress(self, name, **kwargs):
  491. +        timestamp = util.timer()
  492. +        if timestamp - self.lastcount < 0.05:
  493. +            return
  494. +        self.lastcount = timestamp
  495. +        self.logfp.write(',\n{"name":%s, "ph":"C", "pid":%d, "ts":%r,'
  496. +                         '\n "args":%s}' %
  497. +                         (_encode(name), self.pid,
  498. +                          (timestamp - self.starttime) * 1e6,
  499. +                          json.dumps(kwargs)))
  500. +
  501. +    def forked(self):
  502. +        self.pid = os.getpid()
  503. +
  504. +    def close(self):
  505. +        timestamp = util.timer()
  506. +        self.logfp.write(',\n{"name":"tracing", "cat":"tracing",\n "ph":"E", '
  507. +                         '"pid":%d, "ts":%r}\n]\n' %
  508. +                         (self.pid, (timestamp - self.starttime) * 1e6))
  509. +        self.logfp.close()
  510. +
  511. +@contextlib.contextmanager
  512. +def duration(name, category):
  513. +    yield
  514. +
  515. +def event(name, category, phase, timestamp=None, **kwargs):
  516. +    pass
  517. +
  518. +def progress(name, **kwargs):
  519. +    pass
  520. +
  521. +def forked():
  522. +    pass
  523. +
  524. +def close():
  525. +    pass
  526. +
  527. +def setup(tr):
  528. +    global duration, event, count, forked, close
  529. +    duration = tr.duration
  530. +    event = tr.event
  531. +    count = tr.count
  532. +    forked = tr.forked
  533. +    close = tr.close
  534. +
  535. +def call(func):
  536. +    funcname = [None]
  537. +    def wrapped(*args, **kwargs):
  538. +        fn = funcname[0]
  539. +        if fn is None:
  540. +            if inspect.ismethod(func):
  541. +                fn = funcname[0] = '%s %s.%s' % (func.__name__,
  542. +                                                 func.__class__.__module__,
  543. +                                                 func.__class__.__name__)
  544. +            else:
  545. +                fn = funcname[0] = '%s %s' % (func.__name__, func.__module__)
  546. +        with duration(fn, 'call'):
  547. +            return func(*args, **kwargs)
  548. +    wrapped.__name__ = func.__name__
  549. +    return wrapped
  550. diff --git a/mercurial/ui.py b/mercurial/ui.py
  551. --- a/mercurial/ui.py
  552. +++ b/mercurial/ui.py
  553. @@ -29,6 +29,7 @@ from . import (
  554.      progress,
  555.      pycompat,
  556.      scmutil,
  557. +    tracing,
  558.      util,
  559.  )
  560.  
  561. @@ -1150,7 +1151,10 @@ class ui(object):
  562.          if self._progbar is not None:
  563.              self._progbar.progress(topic, pos, item=item, unit=unit,
  564.                                     total=total)
  565. -        if pos is None or not self.configbool('progress', 'debug'):
  566. +        if pos is None:
  567. +            return
  568. +        tracing.progress(topic, pos=pos)
  569. +        if not self.configbool('progress', 'debug'):
  570.              return
  571.  
  572.          if unit:
  573. diff --git a/mercurial/util.py b/mercurial/util.py
  574. --- a/mercurial/util.py
  575. +++ b/mercurial/util.py
  576. @@ -1203,8 +1203,10 @@ def checkwinfilename(path):
  577.  
  578.  if pycompat.osname == 'nt':
  579.      checkosfilename = checkwinfilename
  580. +    timer = time.clock
  581.  else:
  582.      checkosfilename = platform.checkosfilename
  583. +    timer = time.time
  584.  
  585.  def makelock(info, pathname):
  586.      try:
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!