[Request] [+++- ] D12096: stream-clone: allow to change persistent-nodemap format during stream clone
marmoute (Pierre-Yves David)
phabricator at mercurial-scm.org
Thu Jan 27 23:26:30 UTC 2022
marmoute created this revision.
Herald added a reviewer: hg-reviewers.
Herald added a subscriber: mercurial-patches.
REVISION SUMMARY
Persistent nodemap affect the store format. However it is fairly isolated and
fast to general locally. So not making it a fixed part of the stream clone is
useful.
This allow clients without persistent-nodemap support (default for client
without Rust enabled, or simply older client). So it make it possible to enable
persistent nodemap on client, where it can provide a massive boost. without too
much consequence.
To do so, we stop using it in the advertisement requirements for streaming and
let the client add/remove the necessary file depending of its configuration.
We still send the files as it seems like a small save to not regenerate them.
In addition, the way we match them will overlap with the changelog-v2/revlog-v2
so we can't simply skip the associated patterns.
REPOSITORY
rHG Mercurial
BRANCH
default
REVISION DETAIL
https://phab.mercurial-scm.org/D12096
AFFECTED FILES
mercurial/requirements.py
mercurial/revlogutils/nodemap.py
mercurial/streamclone.py
tests/test-bundle.t
tests/test-clone-stream-format.t
tests/test-clone-stream.t
tests/test-clonebundles.t
tests/test-debugcommands.t
tests/test-stream-bundle-v2.t
CHANGE DETAILS
diff --git a/tests/test-stream-bundle-v2.t b/tests/test-stream-bundle-v2.t
--- a/tests/test-stream-bundle-v2.t
+++ b/tests/test-stream-bundle-v2.t
@@ -47,11 +47,11 @@
Stream params: {}
stream2 -- {bytecount: 1693, filecount: 11, requirements: generaldelta%2Crevlogv1%2Csparserevlog} (mandatory: True) (no-zstd !)
stream2 -- {bytecount: 1693, filecount: 11, requirements: generaldelta%2Crevlog-compression-zstd%2Crevlogv1%2Csparserevlog} (mandatory: True) (zstd no-rust !)
- stream2 -- {bytecount: 1693, filecount: 11, requirements: generaldelta%2Cpersistent-nodemap%2Crevlog-compression-zstd%2Crevlogv1%2Csparserevlog} (mandatory: True) (rust !)
+ stream2 -- {bytecount: 1693, filecount: 11, requirements: generaldelta%2Crevlog-compression-zstd%2Crevlogv1%2Csparserevlog} (mandatory: True) (rust !)
$ hg debugbundle --spec bundle.hg
none-v2;stream=v2;requirements%3Dgeneraldelta%2Crevlogv1%2Csparserevlog (no-zstd !)
none-v2;stream=v2;requirements%3Dgeneraldelta%2Crevlog-compression-zstd%2Crevlogv1%2Csparserevlog (zstd no-rust !)
- none-v2;stream=v2;requirements%3Dgeneraldelta%2Cpersistent-nodemap%2Crevlog-compression-zstd%2Crevlogv1%2Csparserevlog (rust !)
+ none-v2;stream=v2;requirements%3Dgeneraldelta%2Crevlog-compression-zstd%2Crevlogv1%2Csparserevlog (rust !)
Test that we can apply the bundle as a stream clone bundle
diff --git a/tests/test-debugcommands.t b/tests/test-debugcommands.t
--- a/tests/test-debugcommands.t
+++ b/tests/test-debugcommands.t
@@ -657,8 +657,8 @@
devel-peer-request: pairs: 81 bytes
sending hello command
sending between command
- remote: 487
- remote: capabilities: batch branchmap $USUAL_BUNDLE2_CAPS$ changegroupsubset getbundle known lookup protocaps pushkey streamreqs=generaldelta,persistent-nodemap,revlog-compression-zstd,revlogv1,sparserevlog unbundle=HG10GZ,HG10BZ,HG10UN unbundlehash
+ remote: 468
+ remote: capabilities: batch branchmap $USUAL_BUNDLE2_CAPS$ changegroupsubset getbundle known lookup protocaps pushkey streamreqs=generaldelta,revlog-compression-zstd,revlogv1,sparserevlog unbundle=HG10GZ,HG10BZ,HG10UN unbundlehash
remote: 1
devel-peer-request: protocaps
devel-peer-request: caps: * bytes (glob)
diff --git a/tests/test-clonebundles.t b/tests/test-clonebundles.t
--- a/tests/test-clonebundles.t
+++ b/tests/test-clonebundles.t
@@ -281,7 +281,7 @@
writing 613 bytes for 4 files
bundle requirements: generaldelta, revlogv1, sparserevlog (no-rust no-zstd !)
bundle requirements: generaldelta, revlog-compression-zstd, revlogv1, sparserevlog (no-rust zstd !)
- bundle requirements: generaldelta, persistent-nodemap, revlog-compression-zstd, revlogv1, sparserevlog (rust !)
+ bundle requirements: generaldelta, revlog-compression-zstd, revlogv1, sparserevlog (rust !)
No bundle spec should work
diff --git a/tests/test-clone-stream.t b/tests/test-clone-stream.t
--- a/tests/test-clone-stream.t
+++ b/tests/test-clone-stream.t
@@ -338,23 +338,23 @@
#endif
#if zstd rust no-dirstate-v2
$ f --size --hex --bytes 256 body
- body: size=116331
+ body: size=116310
0000: 04 6e 6f 6e 65 48 47 32 30 00 00 00 00 00 00 00 |.noneHG20.......|
- 0010: 91 07 53 54 52 45 41 4d 32 00 00 00 00 03 00 09 |..STREAM2.......|
- 0020: 06 09 04 0c 55 62 79 74 65 63 6f 75 6e 74 31 30 |....Ubytecount10|
+ 0010: 7c 07 53 54 52 45 41 4d 32 00 00 00 00 03 00 09 ||.STREAM2.......|
+ 0020: 06 09 04 0c 40 62 79 74 65 63 6f 75 6e 74 31 30 |.... at bytecount10|
0030: 31 32 37 36 66 69 6c 65 63 6f 75 6e 74 31 30 39 |1276filecount109|
0040: 33 72 65 71 75 69 72 65 6d 65 6e 74 73 67 65 6e |3requirementsgen|
- 0050: 65 72 61 6c 64 65 6c 74 61 25 32 43 70 65 72 73 |eraldelta%2Cpers|
- 0060: 69 73 74 65 6e 74 2d 6e 6f 64 65 6d 61 70 25 32 |istent-nodemap%2|
- 0070: 43 72 65 76 6c 6f 67 2d 63 6f 6d 70 72 65 73 73 |Crevlog-compress|
- 0080: 69 6f 6e 2d 7a 73 74 64 25 32 43 72 65 76 6c 6f |ion-zstd%2Crevlo|
- 0090: 67 76 31 25 32 43 73 70 61 72 73 65 72 65 76 6c |gv1%2Csparserevl|
- 00a0: 6f 67 00 00 80 00 73 08 42 64 61 74 61 2f 30 2e |og....s.Bdata/0.|
- 00b0: 69 00 03 00 01 00 00 00 00 00 00 00 02 00 00 00 |i...............|
- 00c0: 01 00 00 00 00 00 00 00 01 ff ff ff ff ff ff ff |................|
- 00d0: ff 80 29 63 a0 49 d3 23 87 bf ce fe 56 67 92 67 |..)c.I.#....Vg.g|
- 00e0: 2c 69 d1 ec 39 00 00 00 00 00 00 00 00 00 00 00 |,i..9...........|
- 00f0: 00 75 30 73 26 45 64 61 74 61 2f 30 30 63 68 61 |.u0s&Edata/00cha|
+ 0050: 65 72 61 6c 64 65 6c 74 61 25 32 43 72 65 76 6c |eraldelta%2Crevl|
+ 0060: 6f 67 2d 63 6f 6d 70 72 65 73 73 69 6f 6e 2d 7a |og-compression-z|
+ 0070: 73 74 64 25 32 43 72 65 76 6c 6f 67 76 31 25 32 |std%2Crevlogv1%2|
+ 0080: 43 73 70 61 72 73 65 72 65 76 6c 6f 67 00 00 80 |Csparserevlog...|
+ 0090: 00 73 08 42 64 61 74 61 2f 30 2e 69 00 03 00 01 |.s.Bdata/0.i....|
+ 00a0: 00 00 00 00 00 00 00 02 00 00 00 01 00 00 00 00 |................|
+ 00b0: 00 00 00 01 ff ff ff ff ff ff ff ff 80 29 63 a0 |.............)c.|
+ 00c0: 49 d3 23 87 bf ce fe 56 67 92 67 2c 69 d1 ec 39 |I.#....Vg.g,i..9|
+ 00d0: 00 00 00 00 00 00 00 00 00 00 00 00 75 30 73 26 |............u0s&|
+ 00e0: 45 64 61 74 61 2f 30 30 63 68 61 6e 67 65 6c 6f |Edata/00changelo|
+ 00f0: 67 2d 61 62 33 34 39 31 38 30 61 30 34 30 35 30 |g-ab349180a04050|
#endif
#if zstd dirstate-v2
$ f --size --hex --bytes 256 body
diff --git a/tests/test-clone-stream-format.t b/tests/test-clone-stream-format.t
--- a/tests/test-clone-stream-format.t
+++ b/tests/test-clone-stream-format.t
@@ -4,6 +4,11 @@
#testcases stream-legacy stream-bundle2
+ $ cat << EOF >> $HGRCPATH
+ > [storage]
+ > revlog.persistent-nodemap.slow-path=allow
+ > EOF
+
#if stream-legacy
$ cat << EOF >> $HGRCPATH
> [server]
@@ -13,7 +18,7 @@
Initialize repository
- $ hg init server --config format.use-share-safe=yes
+ $ hg init server --config format.use-share-safe=yes --config format.use-persistent-nodemap=yes
$ cd server
$ sh $TESTDIR/testlib/stream_clone_setup.sh
adding 00changelog-ab349180a0405010.nd
@@ -345,3 +350,86 @@
$ killdaemons.py
+
+
+Test streaming from/to repository without a persistent-nodemap
+==============================================================
+
+persistent nodemap affects revlog, but they are easy to generate locally, so we allow it to be changed over a stream clone
+
+ $ rm hg-*.pid errors-*.txt
+ $ hg clone --pull --config format.use-persistent-nodemap=no server server-no-persistent-nodemap
+ requesting all changes
+ adding changesets
+ adding manifests
+ adding file changes
+ added 5004 changesets with 1088 changes to 1088 files (+1 heads)
+ new changesets 96ee1d7354c4:06ddac466af5
+ updating to branch default
+ 0 files updated, 0 files merged, 0 files removed, 0 files unresolved
+ $ hg verify -R server-no-persistent-nodemap
+ checking changesets
+ checking manifests
+ crosschecking files in changesets and manifests
+ checking files
+ checked 5004 changesets with 1088 changes to 1088 files
+ $ hg -R server serve -p $HGPORT -d --pid-file=hg-1.pid --error errors-1.txt
+ $ cat hg-1.pid > $DAEMON_PIDS
+ $ hg -R server-no-persistent-nodemap serve -p $HGPORT2 -d --pid-file=hg-2.pid --error errors-2.txt
+ $ cat hg-2.pid >> $DAEMON_PIDS
+ $ hg debugrequires -R server | grep persistent-nodemap
+ persistent-nodemap
+ $ hg debugrequires -R server-no-persistent-nodemap | grep persistent-nodemap
+ [1]
+ $ ls -1 server/.hg/store/00changelog*
+ server/.hg/store/00changelog-*.nd (glob)
+ server/.hg/store/00changelog.d
+ server/.hg/store/00changelog.i
+ server/.hg/store/00changelog.n
+ $ ls -1 server-no-persistent-nodemap/.hg/store/00changelog*
+ server-no-persistent-nodemap/.hg/store/00changelog.d
+ server-no-persistent-nodemap/.hg/store/00changelog.i
+
+persistent-nodemap â no-persistent-nodemap cloning
+
+ $ hg clone --quiet --stream -U http://localhost:$HGPORT clone-remove-persistent-nodemap --config format.use-persistent-nodemap=no
+ $ cat errors-1.txt
+ $ hg -R clone-remove-persistent-nodemap verify
+ checking changesets
+ checking manifests
+ crosschecking files in changesets and manifests
+ checking files
+ checked 5004 changesets with 1088 changes to 1088 files
+ $ hg debugrequires -R clone-remove-persistent-nodemap | grep persistent-nodemap
+ [1]
+
+The persistent-nodemap files should no longer exists
+
+ $ ls -1 clone-remove-persistent-nodemap/.hg/store/00changelog*
+ clone-remove-persistent-nodemap/.hg/store/00changelog.d
+ clone-remove-persistent-nodemap/.hg/store/00changelog.i
+
+
+no-persistent-nodemap â persistent-nodemap cloning
+
+ $ hg clone --quiet --stream -U http://localhost:$HGPORT2 clone-add-persistent-nodemap --config format.use-persistent-nodemap=yes
+ $ cat errors-2.txt
+ $ hg -R clone-add-persistent-nodemap verify
+ checking changesets
+ checking manifests
+ crosschecking files in changesets and manifests
+ checking files
+ checked 5004 changesets with 1088 changes to 1088 files
+ $ hg debugrequires -R clone-add-persistent-nodemap | grep persistent-nodemap
+ persistent-nodemap
+
+The persistent-nodemap files should exists
+
+ $ ls -1 clone-add-persistent-nodemap/.hg/store/00changelog*
+ clone-add-persistent-nodemap/.hg/store/00changelog-*.nd (glob)
+ clone-add-persistent-nodemap/.hg/store/00changelog.d
+ clone-add-persistent-nodemap/.hg/store/00changelog.i
+ clone-add-persistent-nodemap/.hg/store/00changelog.n
+
+
+ $ killdaemons.py
diff --git a/tests/test-bundle.t b/tests/test-bundle.t
--- a/tests/test-bundle.t
+++ b/tests/test-bundle.t
@@ -297,16 +297,16 @@
$ hg -R test debugcreatestreamclonebundle packed.hg
writing 2665 bytes for 6 files
- bundle requirements: generaldelta, persistent-nodemap, revlog-compression-zstd, revlogv1, sparserevlog
+ bundle requirements: generaldelta, revlog-compression-zstd, revlogv1, sparserevlog
$ f -B 64 --size --sha1 --hexdump packed.hg
- packed.hg: size=2884, sha1=b0c868701f8a9fe44daf094b2f5bf661cf90c789
+ packed.hg: size=2865, sha1=353d10311f4befa195d9a1ca4b8e26518115c702
0000: 48 47 53 31 55 4e 00 00 00 00 00 00 00 06 00 00 |HGS1UN..........|
- 0010: 00 00 00 00 0a 69 00 4e 67 65 6e 65 72 61 6c 64 |.....i.Ngenerald|
- 0020: 65 6c 74 61 2c 70 65 72 73 69 73 74 65 6e 74 2d |elta,persistent-|
- 0030: 6e 6f 64 65 6d 61 70 2c 72 65 76 6c 6f 67 2d 63 |nodemap,revlog-c|
+ 0010: 00 00 00 00 0a 69 00 3b 67 65 6e 65 72 61 6c 64 |.....i.;generald|
+ 0020: 65 6c 74 61 2c 72 65 76 6c 6f 67 2d 63 6f 6d 70 |elta,revlog-comp|
+ 0030: 72 65 73 73 69 6f 6e 2d 7a 73 74 64 2c 72 65 76 |ression-zstd,rev|
$ hg debugbundle --spec packed.hg
- none-packed1;requirements%3Dgeneraldelta%2Cpersistent-nodemap%2Crevlog-compression-zstd%2Crevlogv1%2Csparserevlog
+ none-packed1;requirements%3Dgeneraldelta%2Crevlog-compression-zstd%2Crevlogv1%2Csparserevlog
#endif
#if reporevlogstore no-rust zstd
@@ -357,17 +357,17 @@
$ hg -R testnongd debugcreatestreamclonebundle packednongd.hg
writing 301 bytes for 3 files
- bundle requirements: persistent-nodemap, revlog-compression-zstd, revlogv1
+ bundle requirements: revlog-compression-zstd, revlogv1
$ f -B 64 --size --sha1 --hexdump packednongd.hg
- packednongd.hg: size=426, sha1=79563ccd6ef779bcfe62a4da64f89a1b308e92e0
+ packednongd.hg: size=407, sha1=0b8714422b785ba8eb98c916b41ffd5fb994c9b5
0000: 48 47 53 31 55 4e 00 00 00 00 00 00 00 03 00 00 |HGS1UN..........|
- 0010: 00 00 00 00 01 2d 00 34 70 65 72 73 69 73 74 65 |.....-.4persiste|
- 0020: 6e 74 2d 6e 6f 64 65 6d 61 70 2c 72 65 76 6c 6f |nt-nodemap,revlo|
- 0030: 67 2d 63 6f 6d 70 72 65 73 73 69 6f 6e 2d 7a 73 |g-compression-zs|
+ 0010: 00 00 00 00 01 2d 00 21 72 65 76 6c 6f 67 2d 63 |.....-.!revlog-c|
+ 0020: 6f 6d 70 72 65 73 73 69 6f 6e 2d 7a 73 74 64 2c |ompression-zstd,|
+ 0030: 72 65 76 6c 6f 67 76 31 00 64 61 74 61 2f 66 6f |revlogv1.data/fo|
$ hg debugbundle --spec packednongd.hg
- none-packed1;requirements%3Dpersistent-nodemap%2Crevlog-compression-zstd%2Crevlogv1
+ none-packed1;requirements%3Drevlog-compression-zstd%2Crevlogv1
#endif
@@ -427,7 +427,7 @@
$ hg -R testsecret debugcreatestreamclonebundle packedsecret.hg
(warning: stream clone bundle will contain secret revisions)
writing 301 bytes for 3 files
- bundle requirements: generaldelta, persistent-nodemap, revlog-compression-zstd, revlogv1, sparserevlog
+ bundle requirements: generaldelta, revlog-compression-zstd, revlogv1, sparserevlog
#endif
diff --git a/mercurial/streamclone.py b/mercurial/streamclone.py
--- a/mercurial/streamclone.py
+++ b/mercurial/streamclone.py
@@ -27,6 +27,9 @@
store,
util,
)
+from .revlogutils import (
+ nodemap,
+)
from .utils import (
stringutil,
)
@@ -216,6 +219,7 @@
repo.ui, repo.requirements, repo.features
)
scmutil.writereporequirements(repo)
+ nodemap.post_stream_cleanup(repo)
if rbranchmap:
repo._branchcaches.replace(repo, rbranchmap)
@@ -510,6 +514,7 @@
)
consumev1(repo, fp, filecount, bytecount)
+ nodemap.post_stream_cleanup(repo)
class streamcloneapplier(object):
@@ -826,6 +831,7 @@
repo.ui, repo.requirements, repo.features
)
scmutil.writereporequirements(repo)
+ nodemap.post_stream_cleanup(repo)
def _copy_files(src_vfs_map, dst_vfs_map, entries, progress):
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -16,6 +16,7 @@
from .. import (
error,
+ requirements,
util,
)
from . import docket as docket_mod
@@ -34,6 +35,19 @@
pass
+def post_stream_cleanup(repo):
+ """The stream clone might needs to remove some file if persisten nodemap
+ was dropped while stream cloning
+ """
+ if requirements.REVLOGV1_REQUIREMENT not in repo.requirements:
+ return
+ if requirements.NODEMAP_REQUIREMENT in repo.requirements:
+ return
+ unfi = repo.unfiltered()
+ delete_nodemap(None, unfi, unfi.changelog)
+ delete_nodemap(None, repo, unfi.manifestlog._rootstore._revlog)
+
+
def persisted_data(revlog):
"""read the nodemap for a revlog from disk"""
if revlog._nodemap_file is None:
diff --git a/mercurial/requirements.py b/mercurial/requirements.py
--- a/mercurial/requirements.py
+++ b/mercurial/requirements.py
@@ -111,7 +111,6 @@
COPIESSDC_REQUIREMENT,
GENERALDELTA_REQUIREMENT,
INTERNAL_PHASE_REQUIREMENT,
- NODEMAP_REQUIREMENT,
REVLOG_COMPRESSION_ZSTD,
REVLOGV1_REQUIREMENT,
REVLOGV2_REQUIREMENT,
To: marmoute, #hg-reviewers
Cc: mercurial-patches, mercurial-devel
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mercurial-scm.org/pipermail/mercurial-patches/attachments/20220127/50c83ba8/attachment-0001.html>
More information about the Mercurial-patches
mailing list