diff --git a/README.asciidoc b/README.asciidoc index dee5b4d..29a85b1 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -27,7 +27,9 @@ to be appropriately merged upstream): * adds a 'git-hg-helper' script than can aid in the git-hg interaction workflow * provides enhanced bidirectional git-hg safety * avoids clutter of `refs/hg/...` by keeping these implementation details really private -* more robust and efficient fetching +* more robust and efficient fetching, especially so when fetching or cloning from multiple + Mercurial clones which will only process changesets not yet fetched from elsewhere + (as opposed to processing everything all over again) See sections below or sidemarked notes for more details. **** @@ -207,6 +209,24 @@ If somehow your workflow relies on having these in the old place: % git config --global remote-hg.show-private-refs true -------------------------------------- +More importantly, a significantly more efficient workflow is achieved using +one set of shared marks files for all remotes (which also forces a local repo +to use an internal proxy clone). +The practical consequence is that fetching from a newly added remote hg repo +does not require another (lengthy) complete import +(as the original clone) but will only fetch additional changesets (if any). +The same goes for subsequent fetching from any hg remote; what was fetched +and imported from some remote need not be imported again from another. +Operating in this shared mode also has the added advantage +of correctly pushing after a `strip` on a remote. +This shared-marks-files behaviour is the default on a fresh repo clone. It can +also be enabled on an existing one by the following setting. +-------------------------------------- +% git config --global remote-hg.shared-marks true +-------------------------------------- +Note, however, that one should then perform a fetch from each relevant remote +to fully complete the conversion (prior to subsequent pushing). + === Helper Commands === Beyond that, a 'git-hg-helper' script has been added that can aid in the git-hg diff --git a/doc/git-remote-hg.txt b/doc/git-remote-hg.txt index 9c290b2..9ae2953 100644 --- a/doc/git-remote-hg.txt +++ b/doc/git-remote-hg.txt @@ -84,6 +84,16 @@ maintained not so visibly. If that, however, would be preferred: % git config --global remote-hg.show-private-refs true -------------------------------------- +Use of shared marks files is the default in a new repo, but can also be enabled +for an existing repo: + +-------------------------------------- +% git config --global remote-hg.shared-marks true +-------------------------------------- + +Note that one should perform a fetch from each remote to properly complete the +conversion to shared marks files. + NOTES ----- @@ -170,6 +180,14 @@ mark is essentially a plain number. `marks-hg` similarly contains a (JSON) base mapping between such mark and hg revision hash. Together they provide for a (consistent) view of the synchronization state of things. +When operating with shared-marks files, the `marks-git` and `marks-hg` files +are shared among all repos. As such, they are then found in the `.git/hg` +directory (rather than a repo subdirectory). +As there is really only one hg repository +(the shared storage "union bag" in `.git/hg/.hg`), only 1 set of marks files +should track the mapping between commit hash and revision hash. +Each individual remote then only adds some metadata (e.g regarding heads). + Upon a fetch, the helper uses the `marks-hg` file to decide what is already present and what not. The required parts are then retrieved from Mercurial and turned into a `git-fast-import` stream as expected by `import` capability of diff --git a/git-hg-helper b/git-hg-helper index e8fb003..5699fe7 100755 --- a/git-hg-helper +++ b/git-hg-helper @@ -128,7 +128,7 @@ class GitHgRepo: remotehg = import_sibling('remotehg', 'git-remote-hg') for r in self.get_hg_repos(): try: - hgpath = os.path.join(self.gitdir, 'hg', r) + hgpath = remotehg.select_marks_dir(r, self.gitdir, False) m = remotehg.Marks(os.path.join(hgpath, 'marks-hg'), None) mark = m.from_rev(rev) m = GitMarks(os.path.join(hgpath, 'marks-git')) @@ -151,6 +151,9 @@ class GitHgRepo: # skip the shared repo if r == '.hg': continue + # only dirs + if not os.path.isdir(os.path.join(shared_path, r)): + continue local_path = os.path.join(shared_path, r, 'clone') local_hg = os.path.join(local_path, '.hg') if not os.path.exists(local_hg): @@ -415,7 +418,7 @@ class GcCommand(SubCommand): for remote in args: if not remote in hg_repos: self.usage('%s is not a valid hg remote' % (remote)) - hgpath = os.path.join(self.githgrepo.gitdir, 'hg', remote) + hgpath = remotehg.select_marks_dir(remote, self.githgrepo.gitdir, False) print "Loading hg marks ..." hgm = remotehg.Marks(os.path.join(hgpath, 'marks-hg'), None) print "Loading git marks ..." diff --git a/git-remote-hg b/git-remote-hg index 7f721ee..4a19a68 100755 --- a/git-remote-hg +++ b/git-remote-hg @@ -1574,6 +1574,86 @@ def select_private_refs(alias): # keep private implementation refs really private return 'hg/%s/refs' % alias +def select_marks_dir(alias, gitdir, migrate): + dirname = os.path.join(gitdir, 'hg', alias) + private_gm = os.path.join(dirname, 'marks-git') + shared_dir = os.path.join(gitdir, 'hg') + shared_gm = os.path.join(shared_dir, 'marks-git') + shared_hgm = os.path.join(shared_dir, 'marks-hg') + # not good in either case + if os.path.exists(private_gm) and os.path.exists(shared_gm): + die('found both %s and %s' % (private_gm, shared_gm)) + # retrieve setting + shared_marks = get_config('remote-hg.shared-marks').strip() + # standardize to True, False or None + shared_marks = get_config_bool('remote-hg.shared-marks', False) \ + if shared_marks else None + # if no specific setting, select one here (favouring shared for new installation) + if shared_marks == None: + # select one automagically, favouring shared for new installation + if os.path.exists(private_gm): + shared_marks = False + elif not os.path.exists(shared_gm) and os.path.exists(shared_dir): + # not a fresh clone, but no shared setup, so use private + shared_marks = False + else: + shared_marks = True + # only migrate with explicit setting + migrate = False + # otherwise, there is not much to decide + # but a migration from one setup to the other might be needed + l = os.listdir(shared_dir) if os.path.exists(shared_dir) and migrate else [] + if shared_marks and migrate: + # make sure all local ones are cleaned up + # use one of these (preferably origin) to seed the shared + seen_file = False + while l: + d = l.pop() + gitm = os.path.join(shared_dir, d, 'marks-git') + hgm = os.path.join(shared_dir, d, 'marks-hg') + # move marks to shared if no such yet (if origin or last one in line) + if os.path.exists(gitm) and os.path.exists(hgm) and \ + not os.path.exists(shared_gm) and not os.path.exists(shared_gm) and \ + (d == 'origin' or not l): + warn('using marks of remote %s as shared marks' % (d)) + seen_file = True + os.rename(gitm, shared_gm) + os.rename(hgm, shared_hgm) + for p in (gitm, hgm): + if os.path.exists(p): + seen_file = True + os.remove(p) + # all private marks removed, should have shared + if seen_file and (not os.path.exists(shared_gm) or not os.path.exists(shared_gm)): + die('migration to shared marks failed; perform fetch to recover') + elif migrate: + if not os.path.exists(shared_gm) or not os.path.exists(shared_hgm): + l = [] + for d in l: + if not os.path.isdir(os.path.join(shared_dir, d)) or d == '.hg': + continue + gitm = os.path.join(shared_dir, d, 'marks-git') + hgm = os.path.join(shared_dir, d, 'marks-hg') + for p in ((shared_gm, gitm), (shared_hgm, hgm)): + if os.path.exists(p[0]): + shutil.copyfile(p[0], p[1]) + # try to run helper gc + # only really important for a local repo (without proxy) + warn('seeded marks of %s with shared; performing gc' % d) + try: + subprocess.check_call(['git-hg-helper', 'gc', '--check-hg', d], + stdout=sys.stderr) + except: + warn('gc for %s failed' % d) + for p in (shared_gm, shared_hgm): + if os.path.exists(p): + os.remove(p) + if shared_marks: + # force proxy for local repo + os.environ['GIT_REMOTE_HG_TEST_REMOTE'] = 'y' + return shared_dir + return dirname + def main(args): global prefix, gitdir, dirname, branches, bmarks global marks, blob_marks, parsed_refs @@ -1641,12 +1721,12 @@ def main(args): warn('various enhanced features might fail in subtle ways') prefix = select_private_refs(alias) + marksdir = select_marks_dir(alias, gitdir, True) repo = get_repo(url, alias) if not is_tmp: fix_path(alias, peer or repo, url) - marksdir = dirname marks_path = os.path.join(marksdir, 'marks-hg') marks = Marks(marks_path, repo) diff --git a/test/bidi.t b/test/bidi.t index 08e1cc3..3228ce3 100755 --- a/test/bidi.t +++ b/test/bidi.t @@ -51,7 +51,7 @@ hg_push () { } hg_log () { - hg -R $1 log --graph --debug + hg -R $1 log --debug } setup () { @@ -204,8 +204,9 @@ test_expect_success 'hg branch' ' : Back to the common revision && (cd hgrepo && hg checkout default) && - hg_log hgrepo > expected && - hg_log hgrepo2 > actual && + # fetch does not affect phase, but pushing now does + hg_log hgrepo | grep -v phase > expected && + hg_log hgrepo2 | grep -v phase > actual && test_cmp expected actual ' @@ -232,10 +233,12 @@ test_expect_success 'hg tags' ' ) && hg_push hgrepo gitrepo && - hg_clone gitrepo hgrepo2 && + # pushing a fetched tag is a problem ... + { hg_clone gitrepo hgrepo2 || true ; } && - hg_log hgrepo > expected && - hg_log hgrepo2 > actual && + # fetch does not affect phase, but pushing now does + hg_log hgrepo | grep -v phase > expected && + hg_log hgrepo2 | grep -v phase > actual && test_cmp expected actual ' diff --git a/test/helper.t b/test/helper.t index 6b1562a..b54cd11 100755 --- a/test/helper.t +++ b/test/helper.t @@ -64,6 +64,7 @@ test_expect_success 'subcommand help' ' grep origin help ' +git config --global remote-hg.shared-marks false test_expect_success 'subcommand repo - no local proxy' ' test_when_finished "rm -rf gitrepo* hgrepo*" && @@ -82,6 +83,8 @@ test_expect_success 'subcommand repo - no local proxy' ' test_cmp expected actual ' +git config --global --unset remote-hg.shared-marks + GIT_REMOTE_HG_TEST_REMOTE=1 && export GIT_REMOTE_HG_TEST_REMOTE diff --git a/test/main-push.t b/test/main-push.t index 614816a..4c70c31 100755 --- a/test/main-push.t +++ b/test/main-push.t @@ -95,6 +95,9 @@ setup_check_hg_commits_repo () { ) } +# a shared bag would make all of the following pretty trivial +git config --global remote-hg.shared-marks false + git config --global remote-hg.check-hg-commits fail test_expect_success 'check-hg-commits with fail mode' ' test_when_finished "rm -rf gitrepo* hgrepo*" && @@ -146,4 +149,109 @@ test_expect_success 'check-hg-commits with push mode - with local proxy' ' check_hg_commits_push ' +setup_check_shared_marks_repo () { + ( + rm -rf hgrepo* && + hg init hgrepo && + cd hgrepo && + echo zero > content && + hg add content && + hg commit -m zero + ) && + + git clone "hg::hgrepo" gitrepo && + + ( + cd gitrepo && + git remote add second hg::../hgrepo && + git fetch second + ) +} + +check_marks () { + dir=$1 + + ls -al $dir && + if test "$2" = "y" + then + test -f $dir/marks-git && test -f $dir/marks-hg + else + test ! -f $dir/marks-git && test ! -f $dir/marks-hg + fi +} + +# cleanup setting +git config --global --unset remote-hg.shared-marks + +test_expect_success 'shared-marks unset' ' + test_when_finished "rm -rf gitrepo* hgrepo*" && + + setup_check_shared_marks_repo && + + ( + cd gitrepo && + check_marks .git/hg y && + check_marks .git/hg/origin n && + check_marks .git/hg/second n + ) +' + +test_expect_success 'shared-marks set to unset' ' + test_when_finished "rm -rf gitrepo* hgrepo*" && + + git config --global remote-hg.shared-marks true && + setup_check_shared_marks_repo && + + ( + cd gitrepo && + check_marks .git/hg y && + check_marks .git/hg/origin n && + check_marks .git/hg/second n + ) && + + git config --global remote-hg.shared-marks false && + ( + cd gitrepo && + git fetch origin && + check_marks .git/hg n && + check_marks .git/hg/origin y && + check_marks .git/hg/second y + ) +' + +test_expect_success 'shared-marks unset to set' ' + test_when_finished "rm -rf gitrepo* hgrepo*" && + + git config --global remote-hg.shared-marks false && + setup_check_shared_marks_repo && + + ( + cd gitrepo && + check_marks .git/hg n && + check_marks .git/hg/origin y && + check_marks .git/hg/second y + ) && + + git config --global --unset remote-hg.shared-marks && + ( + cd gitrepo && + git fetch origin && + check_marks .git/hg n && + check_marks .git/hg/origin y && + check_marks .git/hg/second y + ) && + + git config --global remote-hg.shared-marks true && + ( + cd gitrepo && + git fetch origin && + check_marks .git/hg y && + check_marks .git/hg/origin n && + check_marks .git/hg/second n + ) +' + +# cleanup setting +git config --global --unset remote-hg.shared-marks + test_done