D12399: perf-util: add a `compare-discovery-case` script

[marmoute (Pierre-Yves David)]

marmoute created this revision.
Herald added a reviewer: hg-reviewers.
Herald added a subscriber: mercurial-patches.

REVISION SUMMARY
  This script run the same discovery case using multiple variants of the algorithm
  and report differences in behavior, especially regarding the numbers of roundtrip.

REPOSITORY
  rHG Mercurial

BRANCH
  default

REVISION DETAIL
  https://phab.mercurial-scm.org/D12399

AFFECTED FILES
  contrib/perf-utils/compare-discovery-case

CHANGE DETAILS

diff --git a/contrib/perf-utils/compare-discovery-case b/contrib/perf-utils/compare-discovery-case
new file mode 100644
--- /dev/null
+++ b/contrib/perf-utils/compare-discovery-case
@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+# compare various algorithm variants for a givent case
+#
+#  search-discovery-case REPO LOCAL_CASE REMOTE_CASE
+#
+# The description for the case input use the same format at the ouput of
+# search-discovery-case
+
+import json
+import os
+import subprocess
+import sys
+
+this_script = os.path.abspath(sys.argv[0])
+script_name = os.path.basename(this_script)
+this_dir = os.path.dirname(this_script)
+hg_dir = os.path.join(this_dir, '..', '..')
+HG_REPO = os.path.normpath(hg_dir)
+HG_BIN = os.path.join(HG_REPO, 'hg')
+
+
+SUBSET_PATH = os.path.join(HG_REPO, 'contrib', 'perf-utils', 'subsetmaker.py')
+
+CMD_BASE = (
+    HG_BIN,
+    'debugdiscovery',
+    '--template',
+    'json',
+    '--config',
+    'extensions.subset=%s' % SUBSET_PATH,
+)
+
+# --old
+# --nonheads
+#
+# devel.discovery.exchange-heads=True
+# devel.discovery.grow-sample=True
+# devel.discovery.grow-sample.dynamic=True
+
+VARIANTS = {
+    'tree-discovery': ('--old',),
+    'set-discovery-basic': (
+        '--config',
+        'devel.discovery.exchange-heads=no',
+        '--config',
+        'devel.discovery.grow-sample=no',
+        '--config',
+        'devel.discovery.grow-sample.dynamic=no',
+        '--config',
+        'devel.discovery.randomize=yes',
+    ),
+    'set-discovery-heads': (
+        '--config',
+        'devel.discovery.exchange-heads=yes',
+        '--config',
+        'devel.discovery.grow-sample=no',
+        '--config',
+        'devel.discovery.grow-sample.dynamic=no',
+        '--config',
+        'devel.discovery.randomize=yes',
+    ),
+    'set-discovery-grow-sample': (
+        '--config',
+        'devel.discovery.exchange-heads=yes',
+        '--config',
+        'devel.discovery.grow-sample=yes',
+        '--config',
+        'devel.discovery.grow-sample.dynamic=no',
+        '--config',
+        'devel.discovery.randomize=yes',
+    ),
+    'set-discovery-dynamic-sample': (
+        '--config',
+        'devel.discovery.exchange-heads=yes',
+        '--config',
+        'devel.discovery.grow-sample=yes',
+        '--config',
+        'devel.discovery.grow-sample.dynamic=yes',
+        '--config',
+        'devel.discovery.randomize=yes',
+    ),
+    'set-discovery-default': (
+        '--config',
+        'devel.discovery.randomize=yes',
+    ),
+}
+
+VARIANTS_KEYS = [
+    'tree-discovery',
+    'set-discovery-basic',
+    'set-discovery-heads',
+    'set-discovery-grow-sample',
+    'set-discovery-dynamic-sample',
+    'set-discovery-default',
+]
+
+assert set(VARIANTS.keys()) == set(VARIANTS_KEYS)
+
+
+def format_case(case):
+    return '-'.join(str(s) for s in case)
+
+
+def to_revsets(case):
+    t = case[0]
+    if t == 'scratch':
+        return 'not scratch(all(), %d, "%d")' % (case[1], case[2])
+    elif t == 'randomantichain':
+        return '::randomantichain(all(), "%d")' % case[1]
+    elif t == 'rev':
+        return '::%d' % case[1]
+    else:
+        assert False
+
+
+def compare(repo, local_case, remote_case):
+    case = (repo, local_case, remote_case)
+    for variant in VARIANTS_KEYS:
+        res = process(case, VARIANTS[variant])
+        revs = res["nb-revs"]
+        local_heads = res["nb-head-local"]
+        common_heads = res["nb-common-heads"]
+        roundtrips = res["total-roundtrips"]
+        queries = res["total-queries"]
+        if 'tree-discovery' in variant:
+            print(
+                repo,
+                format_case(local_case),
+                format_case(remote_case),
+                variant,
+                roundtrips,
+                queries,
+                revs,
+                local_heads,
+                common_heads,
+            )
+        else:
+            undecided_common = res["nb-ini_und-common"]
+            undecided_missing = res["nb-ini_und-missing"]
+            undecided = undecided_common + undecided_missing
+            print(
+                repo,
+                format_case(local_case),
+                format_case(remote_case),
+                variant,
+                roundtrips,
+                queries,
+                revs,
+                local_heads,
+                common_heads,
+                undecided,
+                undecided_common,
+                undecided_missing,
+            )
+    return 0
+
+
+def process(case, variant):
+    (repo, left, right) = case
+    cmd = list(CMD_BASE)
+    cmd.append('-R')
+    cmd.append(repo)
+    cmd.append('--local-as-revs')
+    cmd.append(to_revsets(left))
+    cmd.append('--remote-as-revs')
+    cmd.append(to_revsets(right))
+    cmd.extend(variant)
+    s = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+    out, err = s.communicate()
+    return json.loads(out)[0]
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 4:
+        usage = f'USAGE: {script_name} REPO LOCAL_CASE REMOTE_CASE'
+        print(usage, file=sys.stderr)
+        sys.exit(128)
+    repo = sys.argv[1]
+    local_case = sys.argv[2].split('-')
+    local_case = (local_case[0],) + tuple(int(x) for x in local_case[1:])
+    remote_case = sys.argv[3].split('-')
+    remote_case = (remote_case[0],) + tuple(int(x) for x in remote_case[1:])
+    sys.exit(compare(repo, local_case, remote_case))



To: marmoute, #hg-reviewers
Cc: mercurial-patches, mercurial-devel