D10032: sidedata-exchange: rewrite sidedata on-the-fly whenever possible
Alphare (Raphaël Gomès)
phabricator at mercurial-scm.org
Fri Feb 19 11:18:33 UTC 2021
Alphare created this revision.
Herald added a reviewer: indygreg.
Herald added a reviewer: hg-reviewers.
Herald added a subscriber: mercurial-patches.
REVISION SUMMARY
When a A exchanges with B, the difference of their supported sidedata categories
is made, and the responsibility is always with the client to generated it:
- If A pushes to B and B requires category `foo` that A does not have, A will need to generate it when sending it to B.
- If A pulls from B and A needs category `foo`, it will generate `foo` before the end of the transaction.
- Any category that is not required is removed. If peers are not compatible, abort.
It is forbidden to rewrite sidedata for a rev that already has sidedata, since
that would introduce unreachable (garbage) data in the data file, something
we're not prepared for yet.
REPOSITORY
rHG Mercurial
BRANCH
default
REVISION DETAIL
https://phab.mercurial-scm.org/D10032
AFFECTED FILES
mercurial/changegroup.py
mercurial/revlog.py
tests/test-copies-in-changeset.t
tests/test-sidedata-exchange.t
tests/testlib/ext-sidedata-2.py
tests/testlib/ext-sidedata-3.py
tests/testlib/ext-sidedata-4.py
tests/testlib/ext-sidedata-5.py
tests/testlib/ext-sidedata.py
CHANGE DETAILS
diff --git a/tests/testlib/ext-sidedata.py b/tests/testlib/ext-sidedata.py
--- a/tests/testlib/ext-sidedata.py
+++ b/tests/testlib/ext-sidedata.py
@@ -1,6 +1,6 @@
# ext-sidedata.py - small extension to test the sidedata logic
#
-# Copyright 2019 Pierre-Yves David <pierre-yves.david at octobus.net)
+# Copyright 2019 Pierre-Yves David <pierre-yves.david at octobus.net>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
@@ -47,11 +47,12 @@
if self.version & 0xFFFF != 2:
return text, sd
if nodeorrev != nullrev and nodeorrev != nullid:
- if len(text) != struct.unpack('>I', sd[sidedata.SD_TEST1])[0]:
+ cat1 = sd.get(sidedata.SD_TEST1)
+ if cat1 is not None and len(text) != struct.unpack('>I', cat1)[0]:
raise RuntimeError('text size mismatch')
- expected = sd[sidedata.SD_TEST2]
+ expected = sd.get(sidedata.SD_TEST2)
got = hashlib.sha256(text).digest()
- if got != expected:
+ if expected is not None and got != expected:
raise RuntimeError('sha256 mismatch')
return text, sd
@@ -86,3 +87,10 @@
extensions.wrapfunction(
upgrade_engine, 'getsidedatacompanion', wrapgetsidedatacompanion
)
+
+
+def reposetup(ui, repo):
+ # We don't register sidedata computers because we don't care within these
+ # tests
+ repo.register_wanted_sidedata(sidedata.SD_TEST1)
+ repo.register_wanted_sidedata(sidedata.SD_TEST2)
diff --git a/tests/testlib/ext-sidedata-5.py b/tests/testlib/ext-sidedata-5.py
new file mode 100644
--- /dev/null
+++ b/tests/testlib/ext-sidedata-5.py
@@ -0,0 +1,81 @@
+# coding: utf8
+# ext-sidedata-5.py - small extension to test (differently still) the sidedata
+# logic
+#
+# Simulates a server for a simple sidedata exchange.
+#
+# Copyright 2021 Raphaël Gomès <rgomes at octobus.net>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+from __future__ import absolute_import
+
+import hashlib
+import struct
+
+from mercurial import (
+ extensions,
+ revlog,
+)
+
+
+from mercurial.revlogutils import sidedata as sidedatamod
+
+
+def compute_sidedata_1(repo, revlog, rev, sidedata, text=None):
+ sidedata = sidedata.copy()
+ if text is None:
+ text = revlog.revision(rev)
+ sidedata[sidedatamod.SD_TEST1] = struct.pack('>I', len(text))
+ return sidedata
+
+
+def compute_sidedata_2(repo, revlog, rev, sidedata, text=None):
+ sidedata = sidedata.copy()
+ if text is None:
+ text = revlog.revision(rev)
+ sha256 = hashlib.sha256(text).digest()
+ sidedata[sidedatamod.SD_TEST2] = struct.pack('>32s', sha256)
+ return sidedata
+
+
+def reposetup(ui, repo):
+ # Sidedata keys happen to be the same as the categories, easier for testing.
+ for kind in (b'changelog', b'manifest', b'filelog'):
+ repo.register_sidedata_computer(
+ kind,
+ sidedatamod.SD_TEST1,
+ (sidedatamod.SD_TEST1,),
+ compute_sidedata_1,
+ )
+ repo.register_sidedata_computer(
+ kind,
+ sidedatamod.SD_TEST2,
+ (sidedatamod.SD_TEST2,),
+ compute_sidedata_2,
+ )
+
+ # We don't register sidedata computers because we don't care within these
+ # tests
+ repo.register_wanted_sidedata(sidedatamod.SD_TEST1)
+ repo.register_wanted_sidedata(sidedatamod.SD_TEST2)
+
+
+def wrapaddrevision(
+ orig, self, text, transaction, link, p1, p2, *args, **kwargs
+):
+ if kwargs.get('sidedata') is None:
+ kwargs['sidedata'] = {}
+ sd = kwargs['sidedata']
+ ## let's store some arbitrary data just for testing
+ # text length
+ sd[sidedatamod.SD_TEST1] = struct.pack('>I', len(text))
+ # and sha2 hashes
+ sha256 = hashlib.sha256(text).digest()
+ sd[sidedatamod.SD_TEST2] = struct.pack('>32s', sha256)
+ return orig(self, text, transaction, link, p1, p2, *args, **kwargs)
+
+
+def extsetup(ui):
+ extensions.wrapfunction(revlog.revlog, 'addrevision', wrapaddrevision)
diff --git a/tests/testlib/ext-sidedata.py b/tests/testlib/ext-sidedata-4.py
copy from tests/testlib/ext-sidedata.py
copy to tests/testlib/ext-sidedata-4.py
--- a/tests/testlib/ext-sidedata.py
+++ b/tests/testlib/ext-sidedata-4.py
@@ -1,88 +1,19 @@
-# ext-sidedata.py - small extension to test the sidedata logic
+# coding: utf8
+# ext-sidedata-4.py - small extension to test (differently still) the sidedata
+# logic
#
-# Copyright 2019 Pierre-Yves David <pierre-yves.david at octobus.net)
+# Simulates a server for a complex sidedata exchange.
+#
+# Copyright 2021 Raphaël Gomès <rgomes at octobus.net>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
from __future__ import absolute_import
-import hashlib
-import struct
-
-from mercurial.node import (
- nullid,
- nullrev,
-)
-from mercurial import (
- extensions,
- requirements,
- revlog,
-)
-
-from mercurial.upgrade_utils import engine as upgrade_engine
-
from mercurial.revlogutils import sidedata
-def wrapaddrevision(
- orig, self, text, transaction, link, p1, p2, *args, **kwargs
-):
- if kwargs.get('sidedata') is None:
- kwargs['sidedata'] = {}
- sd = kwargs['sidedata']
- ## let's store some arbitrary data just for testing
- # text length
- sd[sidedata.SD_TEST1] = struct.pack('>I', len(text))
- # and sha2 hashes
- sha256 = hashlib.sha256(text).digest()
- sd[sidedata.SD_TEST2] = struct.pack('>32s', sha256)
- return orig(self, text, transaction, link, p1, p2, *args, **kwargs)
-
-
-def wrap_revisiondata(orig, self, nodeorrev, *args, **kwargs):
- text, sd = orig(self, nodeorrev, *args, **kwargs)
- if getattr(self, 'sidedatanocheck', False):
- return text, sd
- if self.version & 0xFFFF != 2:
- return text, sd
- if nodeorrev != nullrev and nodeorrev != nullid:
- if len(text) != struct.unpack('>I', sd[sidedata.SD_TEST1])[0]:
- raise RuntimeError('text size mismatch')
- expected = sd[sidedata.SD_TEST2]
- got = hashlib.sha256(text).digest()
- if got != expected:
- raise RuntimeError('sha256 mismatch')
- return text, sd
-
-
-def wrapgetsidedatacompanion(orig, srcrepo, dstrepo):
- sidedatacompanion = orig(srcrepo, dstrepo)
- addedreqs = dstrepo.requirements - srcrepo.requirements
- if requirements.SIDEDATA_REQUIREMENT in addedreqs:
- assert sidedatacompanion is None # deal with composition later
-
- def sidedatacompanion(revlog, rev):
- update = {}
- revlog.sidedatanocheck = True
- try:
- text = revlog.revision(rev)
- finally:
- del revlog.sidedatanocheck
- ## let's store some arbitrary data just for testing
- # text length
- update[sidedata.SD_TEST1] = struct.pack('>I', len(text))
- # and sha2 hashes
- sha256 = hashlib.sha256(text).digest()
- update[sidedata.SD_TEST2] = struct.pack('>32s', sha256)
- return False, (), update, 0, 0
-
- return sidedatacompanion
-
-
-def extsetup(ui):
- extensions.wrapfunction(revlog.revlog, 'addrevision', wrapaddrevision)
- extensions.wrapfunction(revlog.revlog, '_revisiondata', wrap_revisiondata)
- extensions.wrapfunction(
- upgrade_engine, 'getsidedatacompanion', wrapgetsidedatacompanion
- )
+def reposetup(ui, repo):
+ repo.register_wanted_sidedata(sidedata.SD_TEST2)
+ repo.register_wanted_sidedata(sidedata.SD_TEST3)
diff --git a/tests/testlib/ext-sidedata.py b/tests/testlib/ext-sidedata-3.py
copy from tests/testlib/ext-sidedata.py
copy to tests/testlib/ext-sidedata-3.py
--- a/tests/testlib/ext-sidedata.py
+++ b/tests/testlib/ext-sidedata-3.py
@@ -1,6 +1,10 @@
-# ext-sidedata.py - small extension to test the sidedata logic
+# coding: utf8
+# ext-sidedata-3.py - small extension to test (differently still) the sidedata
+# logic
#
-# Copyright 2019 Pierre-Yves David <pierre-yves.david at octobus.net)
+# Simulates a client for a complex sidedata exchange.
+#
+# Copyright 2021 Raphaël Gomès <rgomes at octobus.net>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
@@ -10,19 +14,38 @@
import hashlib
import struct
-from mercurial.node import (
- nullid,
- nullrev,
-)
from mercurial import (
extensions,
- requirements,
revlog,
)
-from mercurial.upgrade_utils import engine as upgrade_engine
+from mercurial.revlogutils import sidedata as sidedatamod
+
+
+def compute_sidedata_1(repo, revlog, rev, sidedata, text=None):
+ sidedata = sidedata.copy()
+ if text is None:
+ text = revlog.revision(rev)
+ sidedata[sidedatamod.SD_TEST1] = struct.pack('>I', len(text))
+ return sidedata
+
-from mercurial.revlogutils import sidedata
+def compute_sidedata_2(repo, revlog, rev, sidedata, text=None):
+ sidedata = sidedata.copy()
+ if text is None:
+ text = revlog.revision(rev)
+ sha256 = hashlib.sha256(text).digest()
+ sidedata[sidedatamod.SD_TEST2] = struct.pack('>32s', sha256)
+ return sidedata
+
+
+def compute_sidedata_3(repo, revlog, rev, sidedata, text=None):
+ sidedata = sidedata.copy()
+ if text is None:
+ text = revlog.revision(rev)
+ sha384 = hashlib.sha384(text).digest()
+ sidedata[sidedatamod.SD_TEST3] = struct.pack('>48s', sha384)
+ return sidedata
def wrapaddrevision(
@@ -31,58 +54,35 @@
if kwargs.get('sidedata') is None:
kwargs['sidedata'] = {}
sd = kwargs['sidedata']
- ## let's store some arbitrary data just for testing
- # text length
- sd[sidedata.SD_TEST1] = struct.pack('>I', len(text))
- # and sha2 hashes
- sha256 = hashlib.sha256(text).digest()
- sd[sidedata.SD_TEST2] = struct.pack('>32s', sha256)
+ sd = compute_sidedata_1(None, self, None, sd, text=text)
+ kwargs['sidedata'] = compute_sidedata_2(None, self, None, sd, text=text)
return orig(self, text, transaction, link, p1, p2, *args, **kwargs)
-def wrap_revisiondata(orig, self, nodeorrev, *args, **kwargs):
- text, sd = orig(self, nodeorrev, *args, **kwargs)
- if getattr(self, 'sidedatanocheck', False):
- return text, sd
- if self.version & 0xFFFF != 2:
- return text, sd
- if nodeorrev != nullrev and nodeorrev != nullid:
- if len(text) != struct.unpack('>I', sd[sidedata.SD_TEST1])[0]:
- raise RuntimeError('text size mismatch')
- expected = sd[sidedata.SD_TEST2]
- got = hashlib.sha256(text).digest()
- if got != expected:
- raise RuntimeError('sha256 mismatch')
- return text, sd
-
-
-def wrapgetsidedatacompanion(orig, srcrepo, dstrepo):
- sidedatacompanion = orig(srcrepo, dstrepo)
- addedreqs = dstrepo.requirements - srcrepo.requirements
- if requirements.SIDEDATA_REQUIREMENT in addedreqs:
- assert sidedatacompanion is None # deal with composition later
-
- def sidedatacompanion(revlog, rev):
- update = {}
- revlog.sidedatanocheck = True
- try:
- text = revlog.revision(rev)
- finally:
- del revlog.sidedatanocheck
- ## let's store some arbitrary data just for testing
- # text length
- update[sidedata.SD_TEST1] = struct.pack('>I', len(text))
- # and sha2 hashes
- sha256 = hashlib.sha256(text).digest()
- update[sidedata.SD_TEST2] = struct.pack('>32s', sha256)
- return False, (), update, 0, 0
-
- return sidedatacompanion
-
-
def extsetup(ui):
extensions.wrapfunction(revlog.revlog, 'addrevision', wrapaddrevision)
- extensions.wrapfunction(revlog.revlog, '_revisiondata', wrap_revisiondata)
- extensions.wrapfunction(
- upgrade_engine, 'getsidedatacompanion', wrapgetsidedatacompanion
- )
+
+
+def reposetup(ui, repo):
+ # Sidedata keys happen to be the same as the categories, easier for testing.
+ for kind in (b'changelog', b'manifest', b'filelog'):
+ repo.register_sidedata_computer(
+ kind,
+ sidedatamod.SD_TEST1,
+ (sidedatamod.SD_TEST1,),
+ compute_sidedata_1,
+ )
+ repo.register_sidedata_computer(
+ kind,
+ sidedatamod.SD_TEST2,
+ (sidedatamod.SD_TEST2,),
+ compute_sidedata_2,
+ )
+ repo.register_sidedata_computer(
+ kind,
+ sidedatamod.SD_TEST3,
+ (sidedatamod.SD_TEST3,),
+ compute_sidedata_3,
+ )
+ repo.register_wanted_sidedata(sidedatamod.SD_TEST1)
+ repo.register_wanted_sidedata(sidedatamod.SD_TEST2)
diff --git a/tests/testlib/ext-sidedata.py b/tests/testlib/ext-sidedata-2.py
copy from tests/testlib/ext-sidedata.py
copy to tests/testlib/ext-sidedata-2.py
--- a/tests/testlib/ext-sidedata.py
+++ b/tests/testlib/ext-sidedata-2.py
@@ -1,6 +1,9 @@
-# ext-sidedata.py - small extension to test the sidedata logic
+# coding: utf8
+# ext-sidedata-2.py - small extension to test (differently) the sidedata logic
#
-# Copyright 2019 Pierre-Yves David <pierre-yves.david at octobus.net)
+# Simulates a client for a complex sidedata exchange.
+#
+# Copyright 2021 Raphaël Gomès <rgomes at octobus.net>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
@@ -10,79 +13,38 @@
import hashlib
import struct
-from mercurial.node import (
- nullid,
- nullrev,
-)
-from mercurial import (
- extensions,
- requirements,
- revlog,
-)
-
-from mercurial.upgrade_utils import engine as upgrade_engine
-
-from mercurial.revlogutils import sidedata
+from mercurial.revlogutils import sidedata as sidedatamod
-def wrapaddrevision(
- orig, self, text, transaction, link, p1, p2, *args, **kwargs
-):
- if kwargs.get('sidedata') is None:
- kwargs['sidedata'] = {}
- sd = kwargs['sidedata']
- ## let's store some arbitrary data just for testing
- # text length
- sd[sidedata.SD_TEST1] = struct.pack('>I', len(text))
- # and sha2 hashes
- sha256 = hashlib.sha256(text).digest()
- sd[sidedata.SD_TEST2] = struct.pack('>32s', sha256)
- return orig(self, text, transaction, link, p1, p2, *args, **kwargs)
+def compute_sidedata_1(repo, revlog, rev, sidedata, text=None):
+ sidedata = sidedata.copy()
+ if text is None:
+ text = revlog.revision(rev)
+ sidedata[sidedatamod.SD_TEST1] = struct.pack('>I', len(text))
+ return sidedata
-def wrap_revisiondata(orig, self, nodeorrev, *args, **kwargs):
- text, sd = orig(self, nodeorrev, *args, **kwargs)
- if getattr(self, 'sidedatanocheck', False):
- return text, sd
- if self.version & 0xFFFF != 2:
- return text, sd
- if nodeorrev != nullrev and nodeorrev != nullid:
- if len(text) != struct.unpack('>I', sd[sidedata.SD_TEST1])[0]:
- raise RuntimeError('text size mismatch')
- expected = sd[sidedata.SD_TEST2]
- got = hashlib.sha256(text).digest()
- if got != expected:
- raise RuntimeError('sha256 mismatch')
- return text, sd
+def compute_sidedata_2(repo, revlog, rev, sidedata, text=None):
+ sidedata = sidedata.copy()
+ if text is None:
+ text = revlog.revision(rev)
+ sha256 = hashlib.sha256(text).digest()
+ sidedata[sidedatamod.SD_TEST2] = struct.pack('>32s', sha256)
+ return sidedata
-def wrapgetsidedatacompanion(orig, srcrepo, dstrepo):
- sidedatacompanion = orig(srcrepo, dstrepo)
- addedreqs = dstrepo.requirements - srcrepo.requirements
- if requirements.SIDEDATA_REQUIREMENT in addedreqs:
- assert sidedatacompanion is None # deal with composition later
-
- def sidedatacompanion(revlog, rev):
- update = {}
- revlog.sidedatanocheck = True
- try:
- text = revlog.revision(rev)
- finally:
- del revlog.sidedatanocheck
- ## let's store some arbitrary data just for testing
- # text length
- update[sidedata.SD_TEST1] = struct.pack('>I', len(text))
- # and sha2 hashes
- sha256 = hashlib.sha256(text).digest()
- update[sidedata.SD_TEST2] = struct.pack('>32s', sha256)
- return False, (), update, 0, 0
-
- return sidedatacompanion
-
-
-def extsetup(ui):
- extensions.wrapfunction(revlog.revlog, 'addrevision', wrapaddrevision)
- extensions.wrapfunction(revlog.revlog, '_revisiondata', wrap_revisiondata)
- extensions.wrapfunction(
- upgrade_engine, 'getsidedatacompanion', wrapgetsidedatacompanion
- )
+def reposetup(ui, repo):
+ # Sidedata keys happen to be the same as the categories, easier for testing.
+ for kind in (b'changelog', b'manifest', b'filelog'):
+ repo.register_sidedata_computer(
+ kind,
+ sidedatamod.SD_TEST1,
+ (sidedatamod.SD_TEST1,),
+ compute_sidedata_1,
+ )
+ repo.register_sidedata_computer(
+ kind,
+ sidedatamod.SD_TEST2,
+ (sidedatamod.SD_TEST2,),
+ compute_sidedata_2,
+ )
diff --git a/tests/test-sidedata-exchange.t b/tests/test-sidedata-exchange.t
new file mode 100644
--- /dev/null
+++ b/tests/test-sidedata-exchange.t
@@ -0,0 +1,473 @@
+===========================
+Tests for sidedata exchange
+===========================
+
+Check simple exchange behavior
+==============================
+
+Pusher and pushed have sidedata enabled
+---------------------------------------
+
+ $ hg init sidedata-source --config format.exp-use-side-data=yes
+ $ cat << EOF >> sidedata-source/.hg/hgrc
+ > [extensions]
+ > testsidedata=$TESTDIR/testlib/ext-sidedata-5.py
+ > EOF
+ $ hg init sidedata-target --config format.exp-use-side-data=yes
+ $ cat << EOF >> sidedata-target/.hg/hgrc
+ > [extensions]
+ > testsidedata=$TESTDIR/testlib/ext-sidedata-5.py
+ > EOF
+ $ cd sidedata-source
+ $ echo a > a
+ $ echo b > b
+ $ echo c > c
+ $ hg commit -Am "initial"
+ adding a
+ adding b
+ adding c
+ $ echo aa > a
+ $ hg commit -m "other"
+ $ hg push -r . ../sidedata-target
+ pushing to ../sidedata-target
+ searching for changes
+ adding changesets
+ adding manifests
+ adding file changes
+ added 2 changesets with 4 changes to 3 files
+ $ hg -R ../sidedata-target debugsidedata -c 0
+ 2 sidedata entries
+ entry-0001 size 4
+ entry-0002 size 32
+ $ hg -R ../sidedata-target debugsidedata -c 1 -v
+ 2 sidedata entries
+ entry-0001 size 4
+ '\x00\x00\x00:'
+ entry-0002 size 32
+ '\xa3\xee4v\x99\x85$\x9f\x1f\x8dKe\x0f\xc3\x9d-\xc9\xb5%[\x15=h\xe9\xf2O\xb5\xd9\x1f*\xff\xe5'
+ $ hg -R ../sidedata-target debugsidedata -m 0
+ 2 sidedata entries
+ entry-0001 size 4
+ entry-0002 size 32
+ $ hg -R ../sidedata-target debugsidedata -m 1 -v
+ 2 sidedata entries
+ entry-0001 size 4
+ '\x00\x00\x00\x81'
+ entry-0002 size 32
+ '-bL\xc5\xa4uu"#\xac\x1b`,\xc0\xbc\x9d\xf5\xac\xf0\x1d\x89)2\xf8N\xb1\x14m\xce\xd7\xbc\xae'
+ $ hg -R ../sidedata-target debugsidedata a 0
+ 2 sidedata entries
+ entry-0001 size 4
+ entry-0002 size 32
+ $ hg -R ../sidedata-target debugsidedata a 1 -v
+ 2 sidedata entries
+ entry-0001 size 4
+ '\x00\x00\x00\x03'
+ entry-0002 size 32
+ '\xd9\xcd\x81UvL5C\xf1\x0f\xad\x8aH\rt17Fo\x8dU!<\x8e\xae\xfc\xd1/\x06\xd4:\x80'
+ $ cd ..
+
+Puller and pulled have sidedata enabled
+---------------------------------------
+
+ $ rm -rf sidedata-source sidedata-target
+ $ hg init sidedata-source --config format.exp-use-side-data=yes
+ $ cat << EOF >> sidedata-source/.hg/hgrc
+ > [extensions]
+ > testsidedata=$TESTDIR/testlib/ext-sidedata-5.py
+ > EOF
+ $ hg init sidedata-target --config format.exp-use-side-data=yes
+ $ cat << EOF >> sidedata-target/.hg/hgrc
+ > [extensions]
+ > testsidedata=$TESTDIR/testlib/ext-sidedata-5.py
+ > EOF
+ $ cd sidedata-source
+ $ echo a > a
+ $ echo b > b
+ $ echo c > c
+ $ hg commit -Am "initial"
+ adding a
+ adding b
+ adding c
+ $ echo aa > a
+ $ hg commit -m "other"
+ $ hg pull -R ../sidedata-target ../sidedata-source
+ pulling from ../sidedata-source
+ requesting all changes
+ adding changesets
+ adding manifests
+ adding file changes
+ added 2 changesets with 4 changes to 3 files
+ new changesets 05da661850d7:7ec8b4049447
+ (run 'hg update' to get a working copy)
+ $ hg -R ../sidedata-target debugsidedata -c 0
+ 2 sidedata entries
+ entry-0001 size 4
+ entry-0002 size 32
+ $ hg -R ../sidedata-target debugsidedata -c 1 -v
+ 2 sidedata entries
+ entry-0001 size 4
+ '\x00\x00\x00:'
+ entry-0002 size 32
+ '\xa3\xee4v\x99\x85$\x9f\x1f\x8dKe\x0f\xc3\x9d-\xc9\xb5%[\x15=h\xe9\xf2O\xb5\xd9\x1f*\xff\xe5'
+ $ hg -R ../sidedata-target debugsidedata -m 0
+ 2 sidedata entries
+ entry-0001 size 4
+ entry-0002 size 32
+ $ hg -R ../sidedata-target debugsidedata -m 1 -v
+ 2 sidedata entries
+ entry-0001 size 4
+ '\x00\x00\x00\x81'
+ entry-0002 size 32
+ '-bL\xc5\xa4uu"#\xac\x1b`,\xc0\xbc\x9d\xf5\xac\xf0\x1d\x89)2\xf8N\xb1\x14m\xce\xd7\xbc\xae'
+ $ hg -R ../sidedata-target debugsidedata a 0
+ 2 sidedata entries
+ entry-0001 size 4
+ entry-0002 size 32
+ $ hg -R ../sidedata-target debugsidedata a 1 -v
+ 2 sidedata entries
+ entry-0001 size 4
+ '\x00\x00\x00\x03'
+ entry-0002 size 32
+ '\xd9\xcd\x81UvL5C\xf1\x0f\xad\x8aH\rt17Fo\x8dU!<\x8e\xae\xfc\xd1/\x06\xd4:\x80'
+ $ cd ..
+
+Now on to asymmetric configs.
+
+Pusher has sidedata enabled, pushed does not
+--------------------------------------------
+
+ $ rm -rf sidedata-source sidedata-target
+ $ hg init sidedata-source --config format.exp-use-side-data=yes
+ $ cat << EOF >> sidedata-source/.hg/hgrc
+ > [extensions]
+ > testsidedata=$TESTDIR/testlib/ext-sidedata-5.py
+ > EOF
+ $ hg init sidedata-target --config format.exp-use-side-data=no
+ $ cd sidedata-source
+ $ echo a > a
+ $ echo b > b
+ $ echo c > c
+ $ hg commit -Am "initial"
+ adding a
+ adding b
+ adding c
+ $ echo aa > a
+ $ hg commit -m "other"
+ $ hg push -r . ../sidedata-target --traceback
+ pushing to ../sidedata-target
+ searching for changes
+ adding changesets
+ adding manifests
+ adding file changes
+ added 2 changesets with 4 changes to 3 files
+ $ hg -R ../sidedata-target log -G
+ o changeset: 1:7ec8b4049447
+ | tag: tip
+ | user: test
+ | date: Thu Jan 01 00:00:00 1970 +0000
+ | summary: other
+ |
+ o changeset: 0:05da661850d7
+ user: test
+ date: Thu Jan 01 00:00:00 1970 +0000
+ summary: initial
+
+
+ $ hg -R ../sidedata-target debugsidedata -c 0
+ $ hg -R ../sidedata-target debugsidedata -c 1 -v
+ $ hg -R ../sidedata-target debugsidedata -m 0
+ $ hg -R ../sidedata-target debugsidedata -m 1 -v
+ $ hg -R ../sidedata-target debugsidedata a 0
+ $ hg -R ../sidedata-target debugsidedata a 1 -v
+ $ cd ..
+
+Pulled has sidedata enabled, puller does not
+--------------------------------------------
+
+ $ rm -rf sidedata-source sidedata-target
+ $ hg init sidedata-source --config format.exp-use-side-data=yes
+ $ cat << EOF >> sidedata-source/.hg/hgrc
+ > [extensions]
+ > testsidedata=$TESTDIR/testlib/ext-sidedata-5.py
+ > EOF
+ $ hg init sidedata-target --config format.exp-use-side-data=no
+ $ cd sidedata-source
+ $ echo a > a
+ $ echo b > b
+ $ echo c > c
+ $ hg commit -Am "initial"
+ adding a
+ adding b
+ adding c
+ $ echo aa > a
+ $ hg commit -m "other"
+ $ hg pull -R ../sidedata-target ../sidedata-source
+ pulling from ../sidedata-source
+ requesting all changes
+ adding changesets
+ adding manifests
+ adding file changes
+ added 2 changesets with 4 changes to 3 files
+ new changesets 05da661850d7:7ec8b4049447
+ (run 'hg update' to get a working copy)
+ $ hg -R ../sidedata-target log -G
+ o changeset: 1:7ec8b4049447
+ | tag: tip
+ | user: test
+ | date: Thu Jan 01 00:00:00 1970 +0000
+ | summary: other
+ |
+ o changeset: 0:05da661850d7
+ user: test
+ date: Thu Jan 01 00:00:00 1970 +0000
+ summary: initial
+
+
+ $ hg -R ../sidedata-target debugsidedata -c 0
+ $ hg -R ../sidedata-target debugsidedata -c 1 -v
+ $ hg -R ../sidedata-target debugsidedata -m 0
+ $ hg -R ../sidedata-target debugsidedata -m 1 -v
+ $ hg -R ../sidedata-target debugsidedata a 0
+ $ hg -R ../sidedata-target debugsidedata a 1 -v
+ $ cd ..
+
+
+Check sidedata exchange with on-the-fly generation and removal
+==============================================================
+
+(Push) Target has strict superset of the source
+-----------------------------------------------
+
+ $ hg init source-repo --config format.exp-use-side-data=yes
+ $ hg init target-repo --config format.exp-use-side-data=yes
+ $ cat << EOF >> target-repo/.hg/hgrc
+ > [extensions]
+ > testsidedata=$TESTDIR/testlib/ext-sidedata.py
+ > EOF
+ $ cd source-repo
+ $ echo aaa > a
+ $ hg add a
+ $ hg commit -m a
+ $ echo aaa > b
+ $ hg add b
+ $ hg commit -m b
+ $ echo xxx >> a
+ $ hg commit -m aa
+
+No sidedata is generated in the source
+ $ hg debugsidedata -c 0
+
+Check that sidedata capabilities are advertised
+ $ hg debugcapabilities ../target-repo | grep sidedata
+ exp-wanted-sidedata=1,2
+
+We expect the client to abort the push since it's not capable of generating
+what the server is asking
+ $ hg push -r . ../target-repo
+ pushing to ../target-repo
+ abort: cannot push: required sidedata category not supported by this client: '1'
+ [255]
+
+Add the required capabilities
+ $ cat << EOF >> .hg/hgrc
+ > [extensions]
+ > testsidedata2=$TESTDIR/testlib/ext-sidedata-2.py
+ > EOF
+
+We expect the target to have sidedata that was generated by the source on push
+ $ hg push -r . ../target-repo
+ pushing to ../target-repo
+ searching for changes
+ adding changesets
+ adding manifests
+ adding file changes
+ added 3 changesets with 3 changes to 2 files
+ $ cd ../target-repo
+ $ hg debugsidedata -c 0
+ 2 sidedata entries
+ entry-0001 size 4
+ entry-0002 size 32
+ $ hg debugsidedata -c 1 -v
+ 2 sidedata entries
+ entry-0001 size 4
+ '\x00\x00\x006'
+ entry-0002 size 32
+ '\x98\t\xf9\xc4v\xf0\xc5P\x90\xf7wRf\xe8\xe27e\xfc\xc1\x93\xa4\x96\xd0\x1d\x97\xaaG\x1d\xd7t\xfa\xde'
+ $ hg debugsidedata -m 2
+ 2 sidedata entries
+ entry-0001 size 4
+ entry-0002 size 32
+ $ hg debugsidedata a 1
+ 2 sidedata entries
+ entry-0001 size 4
+ entry-0002 size 32
+ $ cd ..
+
+(Push) Difference is not subset/superset
+----------------------------------------
+
+Source has one in common, one missing and one more sidedata category with the
+target.
+
+ $ rm -rf source-repo target-repo
+ $ hg init source-repo --config format.exp-use-side-data=yes
+ $ cat << EOF >> source-repo/.hg/hgrc
+ > [extensions]
+ > testsidedata3=$TESTDIR/testlib/ext-sidedata-3.py
+ > EOF
+ $ hg init target-repo --config format.exp-use-side-data=yes
+ $ cat << EOF >> target-repo/.hg/hgrc
+ > [extensions]
+ > testsidedata4=$TESTDIR/testlib/ext-sidedata-4.py
+ > EOF
+ $ cd source-repo
+ $ echo aaa > a
+ $ hg add a
+ $ hg commit -m a
+ $ echo aaa > b
+ $ hg add b
+ $ hg commit -m b
+ $ echo xxx >> a
+ $ hg commit -m aa
+
+Check that sidedata capabilities are advertised
+ $ hg debugcapabilities . | grep sidedata
+ exp-wanted-sidedata=1,2
+ $ hg debugcapabilities ../target-repo | grep sidedata
+ exp-wanted-sidedata=2,3
+
+Sidedata is generated in the source, but only the right categories (entry-0001 and entry-0002)
+ $ hg debugsidedata -c 0
+ 2 sidedata entries
+ entry-0001 size 4
+ entry-0002 size 32
+ $ hg debugsidedata -c 1 -v
+ 2 sidedata entries
+ entry-0001 size 4
+ '\x00\x00\x006'
+ entry-0002 size 32
+ '\x98\t\xf9\xc4v\xf0\xc5P\x90\xf7wRf\xe8\xe27e\xfc\xc1\x93\xa4\x96\xd0\x1d\x97\xaaG\x1d\xd7t\xfa\xde'
+ $ hg debugsidedata -m 2
+ 2 sidedata entries
+ entry-0001 size 4
+ entry-0002 size 32
+ $ hg debugsidedata a 1
+ 2 sidedata entries
+ entry-0001 size 4
+ entry-0002 size 32
+
+
+We expect the target to have sidedata that was generated by the source on push,
+and also removed the sidedata categories that are not supported by the target.
+Namely, we expect entry-0002 (only exchanged) and entry-0003 (generated),
+but not entry-0001.
+
+ $ hg push -r . ../target-repo --traceback
+ pushing to ../target-repo
+ searching for changes
+ adding changesets
+ adding manifests
+ adding file changes
+ added 3 changesets with 3 changes to 2 files
+ $ cd ../target-repo
+ $ hg log -G
+ o changeset: 2:40f977031323
+ | tag: tip
+ | user: test
+ | date: Thu Jan 01 00:00:00 1970 +0000
+ | summary: aa
+ |
+ o changeset: 1:2707720c6597
+ | user: test
+ | date: Thu Jan 01 00:00:00 1970 +0000
+ | summary: b
+ |
+ o changeset: 0:7049e48789d7
+ user: test
+ date: Thu Jan 01 00:00:00 1970 +0000
+ summary: a
+
+ $ hg debugsidedata -c 0
+ 2 sidedata entries
+ entry-0002 size 32
+ entry-0003 size 48
+ $ hg debugsidedata -c 1 -v
+ 2 sidedata entries
+ entry-0002 size 32
+ '\x98\t\xf9\xc4v\xf0\xc5P\x90\xf7wRf\xe8\xe27e\xfc\xc1\x93\xa4\x96\xd0\x1d\x97\xaaG\x1d\xd7t\xfa\xde'
+ entry-0003 size 48
+ '\x87\xcf\xdfI/\xb5\xed\xeaC\xc1\xf0S\xf3X\x1c\xcc\x00m\xee\xe6#\xc1\xe3\xcaB8Fk\x82e\xfc\xc01\xf6\xb7\xb9\xb3([\xf6D\xa6\xcf\x9b\xea\x11{\x08'
+ $ hg debugsidedata -m 2
+ 2 sidedata entries
+ entry-0002 size 32
+ entry-0003 size 48
+ $ hg debugsidedata a 1
+ 2 sidedata entries
+ entry-0002 size 32
+ entry-0003 size 48
+ $ cd ..
+
+(Pull) Target has strict superset of the source
+-----------------------------------------------
+
+ $ rm -rf source-repo target-repo
+ $ hg init source-repo --config format.exp-use-side-data=yes
+ $ hg init target-repo --config format.exp-use-side-data=yes
+ $ cat << EOF >> target-repo/.hg/hgrc
+ > [extensions]
+ > testsidedata=$TESTDIR/testlib/ext-sidedata.py
+ > EOF
+ $ cd source-repo
+ $ echo aaa > a
+ $ hg add a
+ $ hg commit -m a
+ $ echo aaa > b
+ $ hg add b
+ $ hg commit -m b
+ $ echo xxx >> a
+ $ hg commit -m aa
+
+No sidedata is generated in the source
+ $ hg debugsidedata -c 0
+
+Check that sidedata capabilities are advertised
+ $ hg debugcapabilities ../target-repo | grep sidedata
+ exp-wanted-sidedata=1,2
+
+ $ cd ../target-repo
+
+Add the required capabilities
+ $ cat << EOF >> .hg/hgrc
+ > [extensions]
+ > testsidedata2=$TESTDIR/testlib/ext-sidedata-2.py
+ > EOF
+
+We expect the target to have sidedata that it generated on-the-fly during pull
+ $ hg pull -r . ../source-repo --traceback
+ pulling from ../source-repo
+ adding changesets
+ adding manifests
+ adding file changes
+ added 3 changesets with 3 changes to 2 files
+ new changesets 7049e48789d7:40f977031323
+ (run 'hg update' to get a working copy)
+ $ hg debugsidedata -c 0 --traceback
+ 2 sidedata entries
+ entry-0001 size 4
+ entry-0002 size 32
+ $ hg debugsidedata -c 1 -v --traceback
+ 2 sidedata entries
+ entry-0001 size 4
+ '\x00\x00\x006'
+ entry-0002 size 32
+ '\x98\t\xf9\xc4v\xf0\xc5P\x90\xf7wRf\xe8\xe27e\xfc\xc1\x93\xa4\x96\xd0\x1d\x97\xaaG\x1d\xd7t\xfa\xde'
+ $ hg debugsidedata -m 2
+ 2 sidedata entries
+ entry-0001 size 4
+ entry-0002 size 32
+ $ hg debugsidedata a 1
+ 2 sidedata entries
+ entry-0001 size 4
+ entry-0002 size 32
+ $ cd ..
diff --git a/tests/test-copies-in-changeset.t b/tests/test-copies-in-changeset.t
--- a/tests/test-copies-in-changeset.t
+++ b/tests/test-copies-in-changeset.t
@@ -273,13 +273,12 @@
$ hg ci --amend -m 'copy a to j, v2'
saved backup bundle to $TESTTMP/repo/.hg/strip-backup/*-*-amend.hg (glob)
$ hg debugsidedata -c -v -- -1
- 1 sidedata entries (missing-correct-output !)
- entry-0014 size 24 (missing-correct-output !)
- '\x00\x00\x00\x02\x00\x00\x00\x00\x01\x00\x00\x00\x00\x06\x00\x00\x00\x02\x00\x00\x00\x00aj' (missing-correct-output !)
+ 1 sidedata entries
+ entry-0014 size 24
+ '\x00\x00\x00\x02\x00\x00\x00\x00\x01\x00\x00\x00\x00\x06\x00\x00\x00\x02\x00\x00\x00\x00aj'
#endif
$ hg showcopies --config experimental.copies.read-from=filelog-only
- a -> j (sidedata missing-correct-output !)
- a -> j (no-sidedata !)
+ a -> j
The entries should be written to extras even if they're empty (so the client
won't have to fall back to reading from filelogs)
$ echo x >> j
@@ -357,8 +356,7 @@
saved backup bundle to $TESTTMP/rebase-rename/.hg/strip-backup/*-*-rebase.hg (glob)
$ hg st --change . --copies
A b
- a (sidedata missing-correct-output !)
- a (no-sidedata !)
+ a
R a
$ cd ..
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -3190,3 +3190,58 @@
)
return d
+
+ def rewrite_sidedata(self, helpers, startrev, endrev, kind):
+ if self.version & 0xFFFF != REVLOGV2:
+ return
+ # inline are not yet supported because they suffer from an issue when
+ # rewriting them (since it's not an append-only operation).
+ # See issue6485.
+ assert not self._inline
+ if not helpers[1] and not helpers[2]:
+ # Nothing to generate or remove
+ return
+
+ if kind not in (b"changelog", b"manifest", b"filelog"):
+ raise error.ProgrammingError(b"Unexpected revlog kind '%s'", kind)
+
+ new_entries = []
+ # append the new sidedata
+ with self._datafp(b'a+') as fp:
+ # Maybe this bug still exists, see revlog._writeentry
+ fp.seek(0, os.SEEK_END)
+ current_offset = fp.tell()
+ for rev in range(startrev, endrev + 1):
+ entry = self.index[rev]
+ new_sidedata = storageutil.run_sidedata_helpers(
+ store=self,
+ sidedata_helpers=helpers,
+ sidedata={},
+ rev=rev,
+ kind=kind,
+ )
+
+ serialized_sidedata = sidedatautil.serialize_sidedata(
+ new_sidedata
+ )
+ if entry[10] != 0 or entry[11] != 0:
+ # rewriting entries that already have sidedata is not
+ # supported yet, because it introduces garbage data in the
+ # revlog.
+ msg = "Rewriting existing sidedata is not supported yet"
+ raise error.Abort(msg)
+ entry = entry[:10]
+ entry += (current_offset, len(serialized_sidedata))
+
+ fp.write(serialized_sidedata)
+ new_entries.append(entry)
+ current_offset += len(serialized_sidedata)
+
+ # rewrite the new index entries
+ with self._indexfp(b'w+') as fp:
+ fp.seek(startrev * self._io.size)
+ for i, entry in enumerate(new_entries):
+ rev = startrev + i
+ self.index.replace(rev, entry)
+ packed = self._io.packentry(entry, self.node, self.version, rev)
+ fp.write(packed)
diff --git a/mercurial/changegroup.py b/mercurial/changegroup.py
--- a/mercurial/changegroup.py
+++ b/mercurial/changegroup.py
@@ -7,6 +7,7 @@
from __future__ import absolute_import
+import collections
import os
import struct
import weakref
@@ -253,7 +254,7 @@
pos = next
yield closechunk()
- def _unpackmanifests(self, repo, revmap, trp, prog):
+ def _unpackmanifests(self, repo, revmap, trp, prog, addrevisioncb=None):
self.callback = prog.increment
# no need to check for empty manifest group here:
# if the result of the merge of 1 and 2 is the same in 3 and 4,
@@ -261,7 +262,12 @@
# be empty during the pull
self.manifestheader()
deltas = self.deltaiter()
- repo.manifestlog.getstorage(b'').addgroup(deltas, revmap, trp)
+ repo.manifestlog.getstorage(b'').addgroup(
+ deltas,
+ revmap,
+ trp,
+ addrevisioncb=addrevisioncb,
+ )
prog.complete()
self.callback = None
@@ -367,6 +373,13 @@
efilesset = None
self.callback = None
+ # Keep track of the (non-changelog) revlogs we've updated and their
+ # range of new revisions for sidedata rewrite.
+ # TODO do something more efficient than keeping the reference to
+ # the revlogs, especially memory-wise.
+ touched_manifests = {}
+ touched_filelogs = {}
+
# pull off the manifest group
repo.ui.status(_(b"adding manifests\n"))
# We know that we'll never have more manifests than we had
@@ -374,7 +387,24 @@
progress = repo.ui.makeprogress(
_(b'manifests'), unit=_(b'chunks'), total=changesets
)
- self._unpackmanifests(repo, revmap, trp, progress)
+ on_manifest_rev = None
+ if sidedata_helpers and b'manifest' in sidedata_helpers[1]:
+
+ def on_manifest_rev(manifest, rev):
+ range = touched_manifests.get(manifest)
+ if not range:
+ touched_manifests[manifest] = (rev, rev)
+ else:
+ assert rev == range[1] + 1
+ touched_manifests[manifest] = (range[0], rev)
+
+ self._unpackmanifests(
+ repo,
+ revmap,
+ trp,
+ progress,
+ addrevisioncb=on_manifest_rev,
+ )
needfiles = {}
if repo.ui.configbool(b'server', b'validate'):
@@ -388,12 +418,43 @@
for f, n in pycompat.iteritems(mfest):
needfiles.setdefault(f, set()).add(n)
+ on_filelog_rev = None
+ if sidedata_helpers and b'filelog' in sidedata_helpers[1]:
+
+ def on_filelog_rev(filelog, rev):
+ range = touched_filelogs.get(filelog)
+ if not range:
+ touched_filelogs[filelog] = (rev, rev)
+ else:
+ assert rev == range[1] + 1
+ touched_filelogs[filelog] = (range[0], rev)
+
# process the files
repo.ui.status(_(b"adding file changes\n"))
newrevs, newfiles = _addchangegroupfiles(
- repo, self, revmap, trp, efiles, needfiles
+ repo,
+ self,
+ revmap,
+ trp,
+ efiles,
+ needfiles,
+ addrevisioncb=on_filelog_rev,
)
+ if sidedata_helpers:
+ if b'changelog' in sidedata_helpers[1]:
+ cl.rewrite_sidedata(
+ sidedata_helpers, clstart, clend - 1, b"changelog"
+ )
+ for mf, (startrev, endrev) in touched_manifests.items():
+ mf.rewrite_sidedata(
+ sidedata_helpers, startrev, endrev, b"manifest"
+ )
+ for fl, (startrev, endrev) in touched_filelogs.items():
+ fl.rewrite_sidedata(
+ sidedata_helpers, startrev, endrev, b"filelog"
+ )
+
# making sure the value exists
tr.changes.setdefault(b'changegroup-count-changesets', 0)
tr.changes.setdefault(b'changegroup-count-revisions', 0)
@@ -557,14 +618,18 @@
node, p1, p2, deltabase, cs, flags = headertuple
return node, p1, p2, deltabase, cs, flags
- def _unpackmanifests(self, repo, revmap, trp, prog):
- super(cg3unpacker, self)._unpackmanifests(repo, revmap, trp, prog)
+ def _unpackmanifests(self, repo, revmap, trp, prog, addrevisioncb=None):
+ super(cg3unpacker, self)._unpackmanifests(
+ repo, revmap, trp, prog, addrevisioncb=addrevisioncb
+ )
for chunkdata in iter(self.filelogheader, {}):
# If we get here, there are directory manifests in the changegroup
d = chunkdata[b"filename"]
repo.ui.debug(b"adding %s revisions\n" % d)
deltas = self.deltaiter()
- if not repo.manifestlog.getstorage(d).addgroup(deltas, revmap, trp):
+ if not repo.manifestlog.getstorage(d).addgroup(
+ deltas, revmap, trp, addrevisioncb=addrevisioncb
+ ):
raise error.Abort(_(b"received dir revlog group is empty"))
@@ -1783,7 +1848,15 @@
return bundler.generate(commonrevs, csets, fastpathlinkrev, source)
-def _addchangegroupfiles(repo, source, revmap, trp, expectedfiles, needfiles):
+def _addchangegroupfiles(
+ repo,
+ source,
+ revmap,
+ trp,
+ expectedfiles,
+ needfiles,
+ addrevisioncb=None,
+):
revisions = 0
files = 0
progress = repo.ui.makeprogress(
@@ -1798,7 +1871,13 @@
o = len(fl)
try:
deltas = source.deltaiter()
- if not fl.addgroup(deltas, revmap, trp):
+ added = fl.addgroup(
+ deltas,
+ revmap,
+ trp,
+ addrevisioncb=addrevisioncb,
+ )
+ if not added:
raise error.Abort(_(b"received file revlog group is empty"))
except error.CensoredBaseError as e:
raise error.Abort(_(b"received delta base is censored: %s") % e)
To: Alphare, indygreg, #hg-reviewers
Cc: mercurial-patches, mercurial-devel
More information about the Mercurial-devel
mailing list