D10026: changegroup: add v4 changegroup for revlog v2 exchange
Alphare (Raphaël Gomès)
phabricator at mercurial-scm.org
Fri Feb 19 11:18:35 UTC 2021
Alphare created this revision.
Herald added a reviewer: indygreg.
Herald added a reviewer: hg-reviewers.
Herald added a subscriber: mercurial-patches.
REPOSITORY
rHG Mercurial
BRANCH
default
REVISION DETAIL
https://phab.mercurial-scm.org/D10026
AFFECTED FILES
hgext/remotefilelog/remotefilelog.py
hgext/remotefilelog/shallowbundle.py
hgext/sqlitestore.py
mercurial/bundlerepo.py
mercurial/changegroup.py
mercurial/debugcommands.py
mercurial/exchange.py
mercurial/exchangev2.py
mercurial/localrepo.py
mercurial/revlog.py
tests/test-check-interfaces.py
tests/test-revlog-raw.py
CHANGE DETAILS
diff --git a/tests/test-revlog-raw.py b/tests/test-revlog-raw.py
--- a/tests/test-revlog-raw.py
+++ b/tests/test-revlog-raw.py
@@ -147,6 +147,7 @@
b'flags': rlog.flags(r),
b'deltabase': rlog.node(deltaparent),
b'delta': rlog.revdiff(deltaparent, r),
+ b'sidedata': rlog.sidedata(r),
}
def deltaiter(self):
@@ -159,10 +160,11 @@
deltabase = chunkdata[b'deltabase']
delta = chunkdata[b'delta']
flags = chunkdata[b'flags']
+ sidedata = chunkdata[b'sidedata']
chain = node
- yield (node, p1, p2, cs, deltabase, delta, flags)
+ yield (node, p1, p2, cs, deltabase, delta, flags, sidedata)
def linkmap(lnode):
return rlog.rev(lnode)
diff --git a/tests/test-check-interfaces.py b/tests/test-check-interfaces.py
--- a/tests/test-check-interfaces.py
+++ b/tests/test-check-interfaces.py
@@ -272,6 +272,7 @@
flags=b'',
baserevisionsize=None,
revision=b'',
+ sidedata=b'',
delta=None,
)
checkzobject(rd)
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -2512,7 +2512,7 @@
deltacomputer = deltautil.deltacomputer(self)
# loop through our set of deltas
for data in deltas:
- node, p1, p2, linknode, deltabase, delta, flags = data
+ node, p1, p2, linknode, deltabase, delta, flags, sidedata = data
link = linkmapper(linknode)
flags = flags or REVIDX_DEFAULT_FLAGS
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -313,7 +313,13 @@
)
def getbundle(
- self, source, heads=None, common=None, bundlecaps=None, **kwargs
+ self,
+ source,
+ heads=None,
+ common=None,
+ bundlecaps=None,
+ remote_sidedata=None,
+ **kwargs
):
chunks = exchange.getbundlechunks(
self._repo,
@@ -321,6 +327,7 @@
heads=heads,
common=common,
bundlecaps=bundlecaps,
+ remote_sidedata=remote_sidedata,
**kwargs
)[1]
cb = util.chunkbuffer(chunks)
diff --git a/mercurial/exchangev2.py b/mercurial/exchangev2.py
--- a/mercurial/exchangev2.py
+++ b/mercurial/exchangev2.py
@@ -416,6 +416,8 @@
mdiff.trivialdiffheader(len(data)) + data,
# Flags not yet supported.
0,
+ # Sidedata not yet supported
+ {},
)
cl.addgroup(
@@ -495,6 +497,8 @@
delta,
# Flags not yet supported.
0,
+ # Sidedata not yet supported.
+ {},
)
progress.increment()
@@ -620,6 +624,8 @@
delta,
# Flags not yet supported.
0,
+ # Sidedata not yet supported.
+ {},
)
progress.increment()
@@ -718,6 +724,8 @@
delta,
# Flags not yet supported.
0,
+ # Sidedata not yet supported.
+ {},
)
progress.increment()
diff --git a/mercurial/exchange.py b/mercurial/exchange.py
--- a/mercurial/exchange.py
+++ b/mercurial/exchange.py
@@ -2249,7 +2249,13 @@
def getbundlechunks(
- repo, source, heads=None, common=None, bundlecaps=None, **kwargs
+ repo,
+ source,
+ heads=None,
+ common=None,
+ bundlecaps=None,
+ remote_sidedata=None,
+ **kwargs
):
"""Return chunks constituting a bundle's raw data.
@@ -2279,7 +2285,12 @@
return (
info,
changegroup.makestream(
- repo, outgoing, b'01', source, bundlecaps=bundlecaps
+ repo,
+ outgoing,
+ b'01',
+ source,
+ bundlecaps=bundlecaps,
+ remote_sidedata=remote_sidedata,
),
)
@@ -2303,6 +2314,7 @@
source,
bundlecaps=bundlecaps,
b2caps=b2caps,
+ remote_sidedata=remote_sidedata,
**pycompat.strkwargs(kwargs)
)
@@ -2325,6 +2337,7 @@
b2caps=None,
heads=None,
common=None,
+ remote_sidedata=None,
**kwargs
):
"""add a changegroup part to the requested bundle"""
@@ -2355,7 +2368,13 @@
matcher = None
cgstream = changegroup.makestream(
- repo, outgoing, version, source, bundlecaps=bundlecaps, matcher=matcher
+ repo,
+ outgoing,
+ version,
+ source,
+ bundlecaps=bundlecaps,
+ matcher=matcher,
+ remote_sidedata=remote_sidedata,
)
part = bundler.newpart(b'changegroup', data=cgstream)
diff --git a/mercurial/debugcommands.py b/mercurial/debugcommands.py
--- a/mercurial/debugcommands.py
+++ b/mercurial/debugcommands.py
@@ -346,7 +346,7 @@
def showchunks(named):
ui.write(b"\n%s%s\n" % (indent_string, named))
for deltadata in gen.deltaiter():
- node, p1, p2, cs, deltabase, delta, flags = deltadata
+ node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata
ui.write(
b"%s%s %s %s %s %s %d\n"
% (
@@ -372,7 +372,7 @@
raise error.Abort(_(b'use debugbundle2 for this file'))
gen.changelogheader()
for deltadata in gen.deltaiter():
- node, p1, p2, cs, deltabase, delta, flags = deltadata
+ node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata
ui.write(b"%s%s\n" % (indent_string, hex(node)))
diff --git a/mercurial/changegroup.py b/mercurial/changegroup.py
--- a/mercurial/changegroup.py
+++ b/mercurial/changegroup.py
@@ -32,10 +32,12 @@
)
from .interfaces import repository
+from .revlogutils import sidedata as sidedatamod
_CHANGEGROUPV1_DELTA_HEADER = struct.Struct(b"20s20s20s20s")
_CHANGEGROUPV2_DELTA_HEADER = struct.Struct(b"20s20s20s20s20s")
_CHANGEGROUPV3_DELTA_HEADER = struct.Struct(b">20s20s20s20s20sH")
+_CHANGEGROUPV4_DELTA_HEADER = struct.Struct(b">20s20s20s20s20sHi")
LFS_REQUIREMENT = b'lfs'
@@ -202,7 +204,9 @@
header = self.deltaheader.unpack(headerdata)
delta = readexactly(self._stream, l - self.deltaheadersize)
node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode)
- return (node, p1, p2, cs, deltabase, delta, flags)
+ # cg4 forward-compat
+ sidedata = {}
+ return (node, p1, p2, cs, deltabase, delta, flags, sidedata)
def getchunks(self):
"""returns all the chunks contains in the bundle
@@ -552,6 +556,35 @@
raise error.Abort(_(b"received dir revlog group is empty"))
+class cg4unpacker(cg3unpacker):
+ """Unpacker for cg4 streams.
+
+ cg4 streams add support for exchanging sidedata.
+ """
+
+ version = b'04'
+
+ def deltachunk(self, prevnode):
+ res = super(cg4unpacker, self).deltachunk(prevnode)
+ if not res:
+ return res
+
+ (node, p1, p2, cs, deltabase, delta, flags, _sidedata) = res
+
+ sidedata_length_raw = readexactly(self._stream, 4)
+ (sidedata_length,) = struct.unpack(b">l", sidedata_length_raw)
+
+ sidedata = {}
+
+ assert sidedata_length == 0 or sidedata_length >= 4, sidedata_length
+ if sidedata_length > 0:
+ sidedata_length = sidedata_length - 4
+ sidedata_raw = readexactly(self._stream, sidedata_length)
+ sidedata = sidedatamod.deserialize_sidedata(sidedata_raw)
+
+ return node, p1, p2, cs, deltabase, delta, flags, sidedata
+
+
class headerlessfixup(object):
def __init__(self, fh, h):
self._h = h
@@ -861,6 +894,7 @@
shallow=False,
ellipsisroots=None,
fullnodes=None,
+ remote_sidedata=None,
):
"""Given a source repo, construct a bundler.
@@ -893,6 +927,8 @@
nodes. We store this rather than the set of nodes that should be
ellipsis because for very large histories we expect this to be
significantly smaller.
+
+ remote_sidedata is the set of sidedata categories wanted by the remote.
"""
assert oldmatcher
assert matcher
@@ -988,7 +1024,7 @@
for tree, deltas in it:
if tree:
- assert self.version == b'03'
+ assert self.version in (b'03', b'04')
chunk = _fileheader(tree)
size += len(chunk)
yield chunk
@@ -1394,6 +1430,7 @@
shallow=False,
ellipsisroots=None,
fullnodes=None,
+ remote_sidedata=None,
):
builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
d.node, d.p1node, d.p2node, d.linknode
@@ -1424,6 +1461,7 @@
shallow=False,
ellipsisroots=None,
fullnodes=None,
+ remote_sidedata=None,
):
builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
d.node, d.p1node, d.p2node, d.basenode, d.linknode
@@ -1453,6 +1491,7 @@
shallow=False,
ellipsisroots=None,
fullnodes=None,
+ remote_sidedata=None,
):
builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags
@@ -1473,12 +1512,45 @@
)
+def _makecg4packer(
+ repo,
+ oldmatcher,
+ matcher,
+ bundlecaps,
+ ellipses=False,
+ shallow=False,
+ ellipsisroots=None,
+ fullnodes=None,
+ remote_sidedata=None,
+):
+ builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
+ d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags
+ )
+
+ return cgpacker(
+ repo,
+ oldmatcher,
+ matcher,
+ b'04',
+ builddeltaheader=builddeltaheader,
+ manifestsend=closechunk(),
+ bundlecaps=bundlecaps,
+ ellipses=ellipses,
+ shallow=shallow,
+ ellipsisroots=ellipsisroots,
+ fullnodes=fullnodes,
+ remote_sidedata=remote_sidedata,
+ )
+
+
_packermap = {
b'01': (_makecg1packer, cg1unpacker),
# cg2 adds support for exchanging generaldelta
b'02': (_makecg2packer, cg2unpacker),
# cg3 adds support for exchanging revlog flags and treemanifests
b'03': (_makecg3packer, cg3unpacker),
+ # ch4 adds support for exchanging sidedata
+ b'04': (_makecg4packer, cg4unpacker),
}
@@ -1498,11 +1570,9 @@
#
# (or even to push subset of history)
needv03 = True
- if b'exp-sidedata-flag' in repo.requirements:
- needv03 = True
- # don't attempt to use 01/02 until we do sidedata cleaning
- versions.discard(b'01')
- versions.discard(b'02')
+ has_revlogv2 = requirements.REVLOGV2_REQUIREMENT in repo.requirements
+ if not has_revlogv2:
+ versions.discard(b'04')
if not needv03:
versions.discard(b'03')
return versions
@@ -1565,6 +1635,7 @@
shallow=False,
ellipsisroots=None,
fullnodes=None,
+ remote_sidedata=None,
):
assert version in supportedoutgoingversions(repo)
@@ -1601,6 +1672,7 @@
shallow=shallow,
ellipsisroots=ellipsisroots,
fullnodes=fullnodes,
+ remote_sidedata=remote_sidedata,
)
@@ -1644,8 +1716,15 @@
fastpath=False,
bundlecaps=None,
matcher=None,
+ remote_sidedata=None,
):
- bundler = getbundler(version, repo, bundlecaps=bundlecaps, matcher=matcher)
+ bundler = getbundler(
+ version,
+ repo,
+ bundlecaps=bundlecaps,
+ matcher=matcher,
+ remote_sidedata=remote_sidedata,
+ )
repo = repo.unfiltered()
commonrevs = outgoing.common
diff --git a/mercurial/bundlerepo.py b/mercurial/bundlerepo.py
--- a/mercurial/bundlerepo.py
+++ b/mercurial/bundlerepo.py
@@ -61,7 +61,7 @@
self.repotiprev = n - 1
self.bundlerevs = set() # used by 'bundle()' revset expression
for deltadata in cgunpacker.deltaiter():
- node, p1, p2, cs, deltabase, delta, flags = deltadata
+ node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata
size = len(delta)
start = cgunpacker.tell() - size
diff --git a/hgext/sqlitestore.py b/hgext/sqlitestore.py
--- a/hgext/sqlitestore.py
+++ b/hgext/sqlitestore.py
@@ -681,7 +681,16 @@
):
empty = True
- for node, p1, p2, linknode, deltabase, delta, wireflags in deltas:
+ for (
+ node,
+ p1,
+ p2,
+ linknode,
+ deltabase,
+ delta,
+ wireflags,
+ sidedata,
+ ) in deltas:
storeflags = 0
if wireflags & repository.REVISION_FLAG_CENSORED:
diff --git a/hgext/remotefilelog/shallowbundle.py b/hgext/remotefilelog/shallowbundle.py
--- a/hgext/remotefilelog/shallowbundle.py
+++ b/hgext/remotefilelog/shallowbundle.py
@@ -67,7 +67,7 @@
shallowcg1packer, self, nodelist, rlog, lookup, units=units
)
- def generatefiles(self, changedfiles, *args):
+ def generatefiles(self, changedfiles, *args, **kwargs):
try:
linknodes, commonrevs, source = args
except ValueError:
@@ -92,7 +92,9 @@
[f for f in changedfiles if not repo.shallowmatch(f)]
)
- return super(shallowcg1packer, self).generatefiles(changedfiles, *args)
+ return super(shallowcg1packer, self).generatefiles(
+ changedfiles, *args, **kwargs
+ )
def shouldaddfilegroups(self, source):
repo = self._repo
@@ -176,9 +178,11 @@
repo.shallowmatch = original
-def addchangegroupfiles(orig, repo, source, revmap, trp, expectedfiles, *args):
+def addchangegroupfiles(
+ orig, repo, source, revmap, trp, expectedfiles, *args, **kwargs
+):
if not shallowutil.isenabled(repo):
- return orig(repo, source, revmap, trp, expectedfiles, *args)
+ return orig(repo, source, revmap, trp, expectedfiles, *args, **kwargs)
newfiles = 0
visited = set()
@@ -272,7 +276,7 @@
revisiondata = revisiondatas[(f, node)]
# revisiondata: (node, p1, p2, cs, deltabase, delta, flags)
- node, p1, p2, linknode, deltabase, delta, flags = revisiondata
+ node, p1, p2, linknode, deltabase, delta, flags, sidedata = revisiondata
if not available(f, node, f, deltabase):
continue
diff --git a/hgext/remotefilelog/remotefilelog.py b/hgext/remotefilelog/remotefilelog.py
--- a/hgext/remotefilelog/remotefilelog.py
+++ b/hgext/remotefilelog/remotefilelog.py
@@ -306,6 +306,7 @@
assumehaveparentrevisions=False,
deltaprevious=False,
deltamode=None,
+ sidedata_helpers=None,
):
# we don't use any of these parameters here
del nodesorder, revisiondata, assumehaveparentrevisions, deltaprevious
@@ -333,6 +334,8 @@
baserevisionsize=None,
revision=revision,
delta=delta,
+ # Sidedata is not supported yet
+ sidedata=None,
)
def revdiff(self, node1, node2):
To: Alphare, indygreg, #hg-reviewers
Cc: mercurial-patches, mercurial-devel
More information about the Mercurial-devel
mailing list