Merge branch 'PR/293'

Closes #292

.gitattributes

@@ -0,0 +1,2 @@
+# Set the default behavior, in case people don't have core.autocrlf set.
+* text=auto

.github/contributing.md

@@ -0,0 +1,28 @@
+When submitting a patch make sure the commits in your pull request:
+
+* Have good commit messages
+
+  Please read Chris Beams' blog post [How to Write a Git Commit
+  Message](https://chris.beams.io/posts/git-commit/) on how to write a
+  good commit message. Although the article recommends at most 50
+  characters for the subject, up to 72 characters are frequently
+  accepted for fast-export.
+
+* Adhere to good [commit
+hygiene](http://www.ericbmerritt.com/2011/09/21/commit-hygiene-and-git.html)
+
+  When developing a pull request for hg-fast-export, base your work on
+  the current `master` branch and rebase your work if it no longer can
+  be merged into the current `master` without conflicts. Never merge
+  `master` into your development branch, rebase if your work needs
+  updates from `master`.
+
+  When a pull request is modified due to review feedback, please
+  incorporate the changes into the proper commit. A good reference on
+  how to modify history is in the [Pro Git book, Section
+  7.6](https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History).
+
+Please do not submit a pull request if you are not willing to spend
+the time required to address review comments or revise the patch until
+it follows the guidelines above. A _take it or leave it_ approach to
+contributing wastes both your and the maintainer's time.

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,71 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+name: "CodeQL"
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [master]
+  schedule:
+    - cron: '0 15 * * 4'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # Override automatic language detection by changing the below list
+        # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
+        language: ['python']
+        # Learn more...
+        # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        # We must fetch at least the immediate parents so that if this is
+        # a pull request then we can checkout the head.
+        fetch-depth: 2
+
+    # If this run was triggered by a pull request event, then checkout
+    # the head of the pull request instead of the merge commit.
+    - run: git checkout HEAD^2
+      if: ${{ github.event_name == 'pull_request' }}
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file. 
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.gitignore

@@ -1,2 +1,4 @@
+*.orig
 *.pyc
 .dotest
+.idea/

README-SUBMODULES.md

@@ -0,0 +1,75 @@
+# How to convert Mercurial Repositories with subrepos
+
+## Introduction
+
+hg-fast-export supports migrating mercurial subrepositories in the
+repository being converted into git submodules in the converted repository.
+
+Git submodules must be git repositories while mercurial's subrepositories can
+be git, mercurial or subversion repositories. hg-fast-export will handle any
+git subrepositories automatically, any other kinds must first be converted
+to git repositories. Currently hg-fast-export does not support the conversion
+of subversion subrepositories. The rest of this page covers the conversion of
+mercurial subrepositories which require some manual steps:
+
+The first step for mercurial subrepositories involves converting the
+subrepository into a git repository using hg-fast-export.  When all
+subrepositories have been converted, a mapping file that maps the mercurial
+subrepository path to a converted git submodule path must be created. The
+format for this file is:
+
+"<mercurial subrepo path>"="<git submodule path>"
+"<mercurial subrepo path2>"="<git submodule path2>"
+...
+
+The path of this mapping file is then provided with the --subrepo-map
+command line option.
+
+## Example
+
+Example mercurial repo folder structure (~/mercurial) containing two subrepos:
+    src/...
+    subrepos/subrepo1
+    subrepos/subrepo2
+
+### Setup
+Create an empty new folder where all the converted git modules will be imported:
+    mkdir ~/imported-gits
+    cd ~/imported-gits
+
+### Convert all submodules to git:
+    mkdir submodule1
+    cd submodule1
+    git init
+    hg-fast-export.sh -r ~/mercurial/subrepos/subrepo1
+    cd ..
+    mkdir submodule2
+    cd submodule2
+    git init
+    hg-fast-export.sh -r ~/mercurial/subrepos/subrepo2
+
+### Create mapping file
+    cd ~/imported-gits
+    cat > submodule-mappings << EOF
+    "subrepos/subrepo1"="../submodule1"
+    "subrepos/subrepo2"="../submodule2"
+    EOF
+
+### Convert main repository
+    cd ~/imported-gits
+    mkdir git-main-repo
+    cd git-main-repo
+    git init
+    hg-fast-export.sh -r ~/mercurial --subrepo-map=~/imported-gits/submodule-mappings
+
+### Result
+The resulting repository will now contain the submodules at the paths
+`subrepos/subrepo1` and `subrepos/subrepo2`. The created .gitmodules
+file will look like:
+
+    [submodule "subrepos/subrepo1"]
+          path = subrepos/subrepo1
+          url = ../submodule1
+    [submodule "subrepos/subrepo2"]
+          path = subrepos/subrepo2
+          url = ../submodule2

README.md

@@ -1,29 +1,42 @@
-hg-fast-export.(sh|py) - mercurial to git converter using git-fast-import
+hg-fast-export.sh - mercurial to git converter using git-fast-import
 =========================================================================
 
 Legal
 -----
 
-Most hg-* scripts are licensed under the [MIT license]
+Most hg-* scripts are licensed under the [MIT license] and were written
-(http://www.opensource.org/licenses/mit-license.php) and were written
 by Rocco Rutte <pdmef@gmx.net> with hints and help from the git list and
 \#mercurial on freenode. hg-reset.py is licensed under GPLv2 since it
 copies some code from the mercurial sources.
 
 The current maintainer is Frej Drejhammar <frej.drejhammar@gmail.com>.
 
+[MIT license]: http://www.opensource.org/licenses/mit-license.php
+
 Support
 -------
 
 If you have problems with hg-fast-export or have found a bug, please
-create an issue at the [github issue tracker]
+create an issue at the [github issue tracker]. Before creating a new
-(https://github.com/frej/fast-export/issues). Before creating a new
 issue, check that your problem has not already been addressed in an
 already closed issue. Do not contact the maintainer directly unless
 you want to report a security bug. That way the next person having the
 same problem can benefit from the time spent solving the problem the
 first time.
 
+[github issue tracker]: https://github.com/frej/fast-export/issues
+
+System Requirements
+-------------------
+
+This project depends on Python 2.7 or 3.5+, and the Mercurial >= 4.6
+package (>= 5.2, if Python 3.5+). If Python is not installed, install
+it before proceeding. The Mercurial package can be installed with `pip
+install mercurial`.
+
+On windows the bash that comes with "Git for Windows" is known to work
+well.
+
 Usage
 -----
 
@@ -33,7 +46,8 @@ Using hg-fast-export is quite simple for a mercurial repository <repo>:
 mkdir repo-git # or whatever
 cd repo-git
 git init
-hg-fast-export.sh -r <repo>
+hg-fast-export.sh -r <local-repo>
+git checkout HEAD
 ```
 
 Please note that hg-fast-export does not automatically check out the
@@ -65,15 +79,24 @@ As mercurial appears to be much less picky about the syntax of the
 author information than git, an author mapping file can be given to
 hg-fast-export to fix up malformed author strings. The file is
 specified using the -A option. The file should contain lines of the
-form `FromAuthor=ToAuthor`. The example authors.map below will
+form `"<key>"="<value>"`. Inside the key and value strings, all escape
-translate `User <garbage<user@example.com>` to `User <user@example.com>`.
+sequences understood by the python `unicode_escape` encoding are
+supported; strings are otherwise assumed to be UTF8-encoded.
+(Versions of fast-export prior to v171002 had a different syntax, the
+old syntax can be enabled by the flag `--mappings-are-raw`.)
+
+The example authors.map below will translate `User
+<garbage<tab><user@example.com>` to `User <user@example.com>`.
 
 ```
 -- Start of authors.map --
-User <garbage<user@example.com>=User <user@example.com>
+"User <garbage\t<user@example.com>"="User <user@example.com>"
 -- End of authors.map --
 ```
 
+If you have many Mercurial repositories, Chris J Billington's
+[hg-export-tool] allows you to batch convert them.
+
 Tag and Branch Naming
 ---------------------
 
@@ -82,12 +105,116 @@ name the -B and -T options allow a mapping file to be specified to
 rename branches and tags (respectively). The syntax of the mapping
 file is the same as for the author mapping.
 
+When the -B and -T flags are used, you will probably want to use the
+-n flag to disable the built-in (broken in many cases) sanitizing of
+branch/tag names. In the future -n will become the default, but in
+order to not break existing incremental conversions, the default
+remains with the old behavior.
+
+By default, the `default` mercurial branch is renamed to the `master` 
+branch on git. If your mercurial repo contains both `default` and 
+`master` branches, you'll need to override this behavior. Use
+`-M <newName>` to specify what name to give the `default` branch.
+
+Content filtering
+-----------------
+
+hg-fast-export supports filtering the content of exported files.
+The filter is supplied to the --filter-contents option. hg-fast-export
+runs the filter for each exported file, pipes its content to the filter's
+standard input, and uses the filter's standard output in place
+of the file's original content. The prototypical use of this feature
+is to convert line endings in text files from CRLF to git's preferred LF:
+
+```
+-- Start of crlf-filter.sh --
+#!/bin/sh
+# $1 = pathname of exported file relative to the root of the repo
+# $2 = Mercurial's hash of the file
+# $3 = "1" if Mercurial reports the file as binary, otherwise "0"
+
+if [ "$3" == "1" ]; then cat; else dos2unix -q; fi
+# -q option in call to dos2unix allows to avoid returning an
+# error code when handling non-ascii based text files (like UTF-16
+# encoded text files)
+-- End of crlf-filter.sh --
+```
+
+
+Plugins
+-----------------
+
+hg-fast-export supports plugins to manipulate the file data and commit
+metadata. The plugins are enabled with the --plugin option. The value
+of said option is a plugin name (by folder in the plugins directory),
+and optionally, and equals-sign followed by an initialization string.
+
+There is a readme accompanying each of the bundled plugins, with a
+description of the usage. To create a new plugin, one must simply
+add a new folder under the `plugins` directory, with the name of the
+new plugin. Inside, there must be an `__init__.py` file, which contains
+at a minimum:
+
+```
+def build_filter(args):
+    return Filter(args)
+
+class Filter:
+    def __init__(self, args):
+        pass
+        #Or don't pass, if you want to do some init code here
+```
+
+Beyond the boilerplate initialization, you can see the two different
+defined filter methods in the [dos2unix](./plugins/dos2unix) and
+[branch_name_in_commit](./plugins/branch_name_in_commit) plugins.
+
+```
+commit_data = {'branch': branch, 'parents': parents, 'author': author, 'desc': desc, 'revision': revision, 'hg_hash': hg_hash, 'committer': 'committer', 'extra': extra}
+
+def commit_message_filter(self,commit_data):
+```
+The `commit_message_filter` method is called for each commit, after parsing
+from hg, but before outputting to git. The dictionary `commit_data` contains the
+above attributes about the commit, and can be modified by any filter. The
+values in the dictionary after filters have been run are used to create the git
+commit.
+
+```
+file_data = {'filename':filename,'file_ctx':file_ctx,'d':d}
+
+def file_data_filter(self,file_data):
+```
+The `file_data_filter` method is called for each file within each commit.
+The dictionary `file_data` contains the above attributes about the file, and
+can be modified by any filter. `file_ctx` is the filecontext from the
+mercurial python library.  After all filters have been run, the values
+are used to add the file to the git commit.
+
+Submodules
+----------
+See README-SUBMODULES.md for how to convert subrepositories into git
+submodules.
+
 Notes/Limitations
 -----------------
 
 hg-fast-export supports multiple branches but only named branches with
 exactly one head each. Otherwise commits to the tip of these heads
-within the branch will get flattened into merge commits.
+within the branch will get flattened into merge commits. There are a
+few options to deal with this:
+1. Chris J Billington's [hg-export-tool] can help you to handle branches with
+   duplicate heads.
+2. Use the [head2branch plugin](./plugins/head2branch) to create a new named
+   branch from an unnamed head.
+3. You can ignore unnamed heads with the `-ignore-unnamed-heads` option, which
+   is appropriate in situations such as the extra heads being close commits
+   (abandoned, unmerged changes).
+
+hg-fast-export will ignore any files or directories tracked by mercurial
+called `.git`, and will print a warning if it encounters one. Git cannot
+track such files or directories. This is not to be confused with submodules,
+which are described in README-SUBMODULES.md.
 
 As each git-fast-import run creates a new pack file, it may be
 required to repack the repository quite often for incremental imports
@@ -101,8 +228,8 @@ possible to use hg-fast-export on remote repositories
 Design
 ------
 
-hg-fast-export.py was designed in a way that doesn't require a 2-pass
+hg-fast-export was designed in a way that doesn't require a 2-pass
-mechanism or any prior repository analysis: if just feeds what it
+mechanism or any prior repository analysis: it just feeds what it
 finds into git-fast-import. This also implies that it heavily relies
 on strictly linear ordering of changesets from hg, i.e. its
 append-only storage model so that changesets hg-fast-export already
@@ -111,6 +238,85 @@ saw never get modified.
 Submitting Patches
 ------------------
 
-Please use the issue-tracker at github
+Please create a pull request at
-https://github.com/frej/fast-export to report bugs and submit
+[Github](https://github.com/frej/fast-export/pulls) to submit patches.
-patches.
+
+When submitting a patch make sure the commits in your pull request:
+
+* Have good commit messages
+
+  Please read Chris Beams' blog post [How to Write a Git Commit
+  Message](https://chris.beams.io/posts/git-commit/) on how to write a
+  good commit message. Although the article recommends at most 50
+  characters for the subject, up to 72 characters are frequently
+  accepted for fast-export.
+
+* Adhere to good [commit
+hygiene](http://www.ericbmerritt.com/2011/09/21/commit-hygiene-and-git.html)
+
+  When developing a pull request for hg-fast-export, base your work on
+  the current `master` branch and rebase your work if it no longer can
+  be merged into the current `master` without conflicts. Never merge
+  `master` into your development branch, rebase if your work needs
+  updates from `master`.
+
+  When a pull request is modified due to review feedback, please
+  incorporate the changes into the proper commit. A good reference on
+  how to modify history is in the [Pro Git book, Section
+  7.6](https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History).
+
+Please do not submit a pull request if you are not willing to spend
+the time required to address review comments or revise the patch until
+it follows the guidelines above. A _take it or leave it_ approach to
+contributing wastes both your and the maintainer's time.
+
+Frequent Problems
+=================
+
+* git fast-import crashes with: `error: cannot lock ref 'refs/heads/...`
+
+  Branch names in git behave as file names (as they are just files and
+  sub-directories under `refs/heads/`, and a path cannot name both a
+  file and a directory, i.e. the branches `a` and `a/b` can never
+  exist at the same time in a git repo.
+
+  Use a mapping file to rename the troublesome branch names.
+
+* `Branch [<branch-name>] modified outside hg-fast-export` but I have
+  not touched the repo!
+
+  If you are running fast-export on a case-preserving but
+  case-insensitive file system (Windows and OSX), this will make git
+  treat `A` and `a` as the same branch. The solution is to use a
+  mapping file to rename branches which only differ in case.
+
+* My mapping file does not seem to work when I rename the branch `git
+  fast-import` crashes on!
+
+  fast-export (imperfectly) mangles branch names it thinks won't be
+  valid. The mechanism cannot be removed as it would break already
+  existing incremental imports that expects it. When fast export
+  mangles a name, it prints out a warning of the form `Warning:
+  sanitized branch [<unmangled>] to [<mangled>]`. If `git fast-import`
+  crashes on `<mangled>`, you need to put `<unmangled>` into the
+  mapping file.
+
+* fast-import mangles valid git branch names which I have remapped!
+
+  Use the `-n` flag to hg-fast-export.sh.
+
+* `git status` reports that all files are scheduled for deletion after
+  the initial conversion.
+
+  By design fast export does not touch your working directory, so to
+  git it looks like you have deleted all files, when in fact they have
+  never been checked out. Just do a checkout of the branch you want.
+
+* `Error: repository has at least one unnamed head: hg r<N>`
+
+  By design, hg-fast-export cannot deal with extra heads on a branch.
+  There are a few options depending on whether the extra heads are
+  in-use/open or normally closed. See [Notes/Limitations](#noteslimitations)
+  section for more details.
+
+[hg-export-tool]: https://github.com/chrisjbillington/hg-export-tool

hg-fast-export.py

@@ -1,17 +1,23 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 # Copyright (c) 2007, 2008 Rocco Rutte <pdmef@gmx.net> and others.
 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
 
 from mercurial import node
+from mercurial.scmutil import revsymbol
 from hg2git import setup_repo,fixup_user,get_branch,get_changeset
 from hg2git import load_cache,save_cache,get_git_sha1,set_default_branch,set_origin_name
 from optparse import OptionParser
 import re
 import sys
 import os
+from binascii import hexlify
+import pluginloader
+PY2 = sys.version_info.major == 2
+if PY2:
+  str = unicode
 
-if sys.platform == "win32":
+if PY2 and sys.platform == "win32":
   # On Windows, sys.stdout is initially opened in text mode, which means that
   # when a LF (\n) character is written to sys.stdout, it will be converted
   # into CRLF (\r\n).  That makes git blow up, so use this platform-specific
@@ -20,36 +26,47 @@ if sys.platform == "win32":
   msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
 
 # silly regex to catch Signed-off-by lines in log message
-sob_re=re.compile('^Signed-[Oo]ff-[Bb]y: (.+)$')
+sob_re=re.compile(b'^Signed-[Oo]ff-[Bb]y: (.+)$')
 # insert 'checkpoint' command after this many commits or none at all if 0
 cfg_checkpoint_count=0
 # write some progress message every this many file contents written
 cfg_export_boundary=1000
 
+subrepo_cache={}
+submodule_mappings=None
+
+# True if fast export should automatically try to sanitize
+# author/branch/tag names.
+auto_sanitize = None
+
+stdout_buffer = sys.stdout if PY2 else sys.stdout.buffer
+stderr_buffer = sys.stderr if PY2 else sys.stderr.buffer
+
 def gitmode(flags):
-  return 'l' in flags and '120000' or 'x' in flags and '100755' or '100644'
+  return b'l' in flags and b'120000' or b'x' in flags and b'100755' or b'100644'
 
-def wr_no_nl(msg=''):
+def wr_no_nl(msg=b''):
+  assert isinstance(msg, bytes)
   if msg:
-    sys.stdout.write(msg)
+    stdout_buffer.write(msg)
 
-def wr(msg=''):
+def wr(msg=b''):
   wr_no_nl(msg)
-  sys.stdout.write('\n')
+  stdout_buffer.write(b'\n')
   #map(lambda x: sys.stderr.write('\t[%s]\n' % x),msg.split('\n'))
 
 def checkpoint(count):
   count=count+1
   if cfg_checkpoint_count>0 and count%cfg_checkpoint_count==0:
-    sys.stderr.write("Checkpoint after %d commits\n" % count)
+    stderr_buffer.write(b"Checkpoint after %d commits\n" % count)
-    wr('checkpoint')
+    wr(b'checkpoint')
     wr()
   return count
 
 def revnum_to_revref(rev, old_marks):
   """Convert an hg revnum to a git-fast-import rev reference (an SHA1
   or a mark)"""
-  return old_marks.get(rev) or ':%d' % (rev+1)
+  return old_marks.get(rev) or b':%d' % (rev+1)
 
 def file_mismatch(f1,f2):
   """See if two revisions of a file are not equal."""
@@ -78,7 +95,7 @@ def get_filechanges(repo,revision,parents,mleft):
   l,c,r=[],[],[]
   for p in parents:
     if p<0: continue
-    mright=repo.changectx(p).manifest()
+    mright=revsymbol(repo,b"%d" %p).manifest()
     l,c,r=split_dict(mleft,mright,l,c,r)
   l.sort()
   c.sort()
@@ -101,7 +118,7 @@ def get_author(logmessage,committer,authors):
   "Signed-off-by: foo" and thus matching our detection regex. Prevent
   that."""
 
-  loglines=logmessage.split('\n')
+  loglines=logmessage.split(b'\n')
   i=len(loglines)
   # from tail walk to top skipping empty lines
   while i>=0:
@@ -122,28 +139,108 @@ def get_author(logmessage,committer,authors):
       return r
   return committer
 
-def export_file_contents(ctx,manifest,files,hgtags,encoding=''):
+def remove_gitmodules(ctx):
+  """Removes all submodules of ctx parents"""
+  # Removing all submoduies coming from all parents is safe, as the submodules
+  # of the current commit will be re-added below. A possible optimization would
+  # be to only remove the submodules of the first parent.
+  for parent_ctx in ctx.parents():
+    for submodule in parent_ctx.substate.keys():
+      wr(b'D %s' % submodule)
+  wr(b'D .gitmodules')
+
+def refresh_git_submodule(name,subrepo_info):
+  wr(b'M 160000 %s %s' % (subrepo_info[1],name))
+  stderr_buffer.write(
+    b"Adding/updating submodule %s, revision %s\n" % (name, subrepo_info[1])
+  )
+  return b'[submodule "%s"]\n\tpath = %s\n\turl = %s\n' % (name, name, subrepo_info[0])
+
+def refresh_hg_submodule(name,subrepo_info):
+  gitRepoLocation=submodule_mappings[name] + b"/.git"
+
+  # Populate the cache to map mercurial revision to git revision
+  if not name in subrepo_cache:
+    subrepo_cache[name]=(load_cache(gitRepoLocation+b"/hg2git-mapping"),
+                         load_cache(gitRepoLocation+b"/hg2git-marks",
+                                    lambda s: int(s)-1))
+
+  (mapping_cache,marks_cache)=subrepo_cache[name]
+  subrepo_hash=subrepo_info[1]
+  if subrepo_hash in mapping_cache:
+    revnum=mapping_cache[subrepo_hash]
+    gitSha=marks_cache[int(revnum)]
+    wr(b'M 160000 %s %s' % (gitSha,name))
+    stderr_buffer.write(
+      b"Adding/updating submodule %s, revision %s->%s\n"
+      % (name, subrepo_hash, gitSha)
+    )
+    return b'[submodule "%s"]\n\tpath = %s\n\turl = %s\n' % (name,name,
+      submodule_mappings[name])
+  else:
+    stderr_buffer.write(
+      b"Warning: Could not find hg revision %s for %s in git %s\n"
+      % (subrepo_hash, name, gitRepoLocation,)
+    )
+    return b''
+
+def refresh_gitmodules(ctx):
+  """Updates list of ctx submodules according to .hgsubstate file"""
+  remove_gitmodules(ctx)
+  gitmodules=b""
+  # Create the .gitmodules file and all submodules
+  for name,subrepo_info in ctx.substate.items():
+    if subrepo_info[2]==b'git':
+      gitmodules+=refresh_git_submodule(name,subrepo_info)
+    elif submodule_mappings and name in submodule_mappings:
+      gitmodules+=refresh_hg_submodule(name,subrepo_info)
+
+  if len(gitmodules):
+    wr(b'M 100644 inline .gitmodules')
+    wr(b'data %d' % (len(gitmodules)+1))
+    wr(gitmodules)
+
+def export_file_contents(ctx,manifest,files,hgtags,encoding='',plugins={}):
   count=0
   max=len(files)
+  is_submodules_refreshed=False
   for file in files:
+    if not is_submodules_refreshed and (file==b'.hgsub' or file==b'.hgsubstate'):
+      is_submodules_refreshed=True
+      refresh_gitmodules(ctx)
     # Skip .hgtags files. They only get us in trouble.
-    if not hgtags and file == ".hgtags":
+    if not hgtags and file == b".hgtags":
-      sys.stderr.write('Skip %s\n' % (file))
+      stderr_buffer.write(b'Skip %s\n' % file)
       continue
-    d=ctx.filectx(file).data()
     if encoding:
       filename=file.decode(encoding).encode('utf8')
     else:
       filename=file
-    wr('M %s inline %s' % (gitmode(manifest.flags(file)),
+    if b'.git' in filename.split(b'/'): # Even on Windows, the path separator is / here.
+      stderr_buffer.write(
+        b'Ignoring file %s which cannot be tracked by git\n' % filename
+      )
+      continue
+    file_ctx=ctx.filectx(file)
+    d=file_ctx.data()
+
+    if plugins and plugins['file_data_filters']:
+      file_data = {'filename':filename,'file_ctx':file_ctx,'data':d}
+      for filter in plugins['file_data_filters']:
+        filter(file_data)
+      d=file_data['data']
+      filename=file_data['filename']
+      file_ctx=file_data['file_ctx']
+
+    wr(b'M %s inline %s' % (gitmode(manifest.flags(file)),
                            strip_leading_slash(filename)))
-    wr('data %d' % len(d)) # had some trouble with size()
+    wr(b'data %d' % len(d)) # had some trouble with size()
     wr(d)
     count+=1
     if count%cfg_export_boundary==0:
-      sys.stderr.write('Exported %d/%d files\n' % (count,max))
+      stderr_buffer.write(b'Exported %d/%d files\n' % (count,max))
   if max>cfg_export_boundary:
-    sys.stderr.write('Exported %d/%d files\n' % (count,max))
+    stderr_buffer.write(b'Exported %d/%d files\n' % (count,max))
 
 def sanitize_name(name,what="branch", mapping={}):
   """Sanitize input roughly according to git-check-ref-format(1)"""
@@ -153,60 +250,86 @@ def sanitize_name(name,what="branch", mapping={}):
   # modifying names which previously were not touched it will break
   # preexisting setups which are doing incremental imports.
   #
-  # Use the -B and -T options to mangle branch and tag names
+  # Fast-export tries to not inflict arbitrary naming policy on the
-  # instead. If you have a source repository where this is too much
+  # user, instead it aims to provide mechanisms allowing the user to
-  # work to do manually, write a tool that does it for you.
+  # apply their own policy. Therefore do not add a transform which can
+  # already be implemented with the -B and -T options to mangle branch
+  # and tag names. If you have a source repository where this is too
+  # much work to do manually, write a tool that does it for you.
+  #
 
   def dot(name):
     if not name: return name
-    if name[0] == '.': return '_'+name[1:]
+    if name[0:1] == b'.': return b'_'+name[1:]
     return name
 
+  if not auto_sanitize:
+    return mapping.get(name,name)
   n=mapping.get(name,name)
-  p=re.compile('([[ ~^:?\\\\*]|\.\.)')
+  p=re.compile(b'([\\[ ~^:?\\\\*]|\.\.)')
-  n=p.sub('_', n)
+  n=p.sub(b'_', n)
-  if n[-1] in ('/', '.'): n=n[:-1]+'_'
+  if n[-1:] in (b'/', b'.'): n=n[:-1]+b'_'
-  n='/'.join(map(dot,n.split('/')))
+  n=b'/'.join([dot(s) for s in n.split(b'/')])
-  p=re.compile('_+')
+  p=re.compile(b'_+')
-  n=p.sub('_', n)
+  n=p.sub(b'_', n)
 
   if n!=name:
-    sys.stderr.write('Warning: sanitized %s [%s] to [%s]\n' % (what,name,n))
+    stderr_buffer.write(
+      b'Warning: sanitized %s [%s] to [%s]\n' % (what.encode(), name, n)
+    )
   return n
 
 def strip_leading_slash(filename):
-  if filename[0] == '/':
+  if filename[0:1] == b'/':
     return filename[1:]
   return filename
 
 def export_commit(ui,repo,revision,old_marks,max,count,authors,
-                  branchesmap,sob,brmap,hgtags,encoding='',fn_encoding=''):
+                  branchesmap,sob,brmap,hgtags,encoding='',fn_encoding='',
+                  plugins={}):
   def get_branchname(name):
-    if brmap.has_key(name):
+    if name in brmap:
       return brmap[name]
     n=sanitize_name(name, "branch", branchesmap)
     brmap[name]=n
     return n
 
-  (revnode,_,user,(time,timezone),files,desc,branch,_)=get_changeset(ui,repo,revision,authors,encoding)
+  (revnode,_,user,(time,timezone),files,desc,branch,extra)=get_changeset(ui,repo,revision,authors,encoding)
+  if repo[revnode].hidden():
+    return count
 
   branch=get_branchname(branch)
 
   parents = [p for p in repo.changelog.parentrevs(revision) if p >= 0]
+  author = get_author(desc,user,authors)
+  hg_hash=revsymbol(repo,b"%d" % revision).hex()
+
+  if plugins and plugins['commit_message_filters']:
+    commit_data = {'branch': branch, 'parents': parents,
+                   'author': author, 'desc': desc,
+                   'revision': revision, 'hg_hash': hg_hash,
+                   'committer': user, 'extra': extra}
+    for filter in plugins['commit_message_filters']:
+      filter(commit_data)
+    branch = commit_data['branch']
+    parents = commit_data['parents']
+    author = commit_data['author']
+    user = commit_data['committer']
+    desc = commit_data['desc']
 
   if len(parents)==0 and revision != 0:
-    wr('reset refs/heads/%s' % branch)
+    wr(b'reset refs/heads/%s' % branch)
 
-  wr('commit refs/heads/%s' % branch)
+  wr(b'commit refs/heads/%s' % branch)
-  wr('mark :%d' % (revision+1))
+  wr(b'mark :%d' % (revision+1))
   if sob:
-    wr('author %s %d %s' % (get_author(desc,user,authors),time,timezone))
+    wr(b'author %s %d %s' % (author,time,timezone))
-  wr('committer %s %d %s' % (user,time,timezone))
+  wr(b'committer %s %d %s' % (user,time,timezone))
-  wr('data %d' % (len(desc)+1)) # wtf?
+  wr(b'data %d' % (len(desc)+1)) # wtf?
   wr(desc)
   wr()
 
-  ctx=repo.changectx(str(revision))
+  ctx=revsymbol(repo, b"%d" % revision)
   man=ctx.manifest()
   added,changed,removed,type=[],[],[],''
 
@@ -216,104 +339,139 @@ def export_commit(ui,repo,revision,old_marks,max,count,authors,
     added.sort()
     type='full'
   else:
-    wr('from %s' % revnum_to_revref(parents[0], old_marks))
+    wr(b'from %s' % revnum_to_revref(parents[0], old_marks))
     if len(parents) == 1:
       # later non-merge revision: feed in changed manifest
       # if we have exactly one parent, just take the changes from the
       # manifest without expensively comparing checksums
-      f=repo.status(repo.lookup(parents[0]),revnode)[:3]
+      f=repo.status(parents[0],revnode)
-      added,changed,removed=f[1],f[0],f[2]
+      added,changed,removed=f.added,f.modified,f.removed
       type='simple delta'
     else: # a merge with two parents
-      wr('merge %s' % revnum_to_revref(parents[1], old_marks))
+      wr(b'merge %s' % revnum_to_revref(parents[1], old_marks))
       # later merge revision: feed in changed manifest
       # for many files comparing checksums is expensive so only do it for
       # merges where we really need it due to hg's revlog logic
       added,changed,removed=get_filechanges(repo,revision,parents,man)
       type='thorough delta'
 
-  sys.stderr.write('%s: Exporting %s revision %d/%d with %d/%d/%d added/changed/removed files\n' %
+  stderr_buffer.write(
-      (branch,type,revision+1,max,len(added),len(changed),len(removed)))
+    b'%s: Exporting %s revision %d/%d with %d/%d/%d added/changed/removed files\n'
+    % (branch, type.encode(), revision + 1, max, len(added), len(changed), len(removed))
+  )
 
-  if fn_encoding:
+  for filename in removed:
-    removed=[r.decode(fn_encoding).encode('utf8') for r in removed]
+    if fn_encoding:
+      filename=filename.decode(fn_encoding).encode('utf8')
+    filename=strip_leading_slash(filename)
+    if filename==b'.hgsub':
+      remove_gitmodules(ctx)
+    wr(b'D %s' % filename)
 
-  removed=[strip_leading_slash(x) for x in removed]
+  export_file_contents(ctx,man,added,hgtags,fn_encoding,plugins)
-
+  export_file_contents(ctx,man,changed,hgtags,fn_encoding,plugins)
-  map(lambda r: wr('D %s' % r),removed)
-  export_file_contents(ctx,man,added,hgtags,fn_encoding)
-  export_file_contents(ctx,man,changed,hgtags,fn_encoding)
   wr()
 
   return checkpoint(count)
 
 def export_note(ui,repo,revision,count,authors,encoding,is_first):
   (revnode,_,user,(time,timezone),_,_,_,_)=get_changeset(ui,repo,revision,authors,encoding)
+  if repo[revnode].hidden():
+    return count
 
   parents = [p for p in repo.changelog.parentrevs(revision) if p >= 0]
 
-  wr('commit refs/notes/hg')
+  wr(b'commit refs/notes/hg')
-  wr('committer %s %d %s' % (user,time,timezone))
+  wr(b'committer %s %d %s' % (user,time,timezone))
-  wr('data 0')
+  wr(b'data 0')
   if is_first:
-    wr('from refs/notes/hg^0')
+    wr(b'from refs/notes/hg^0')
-  wr('N inline :%d' % (revision+1))
+  wr(b'N inline :%d' % (revision+1))
-  hg_hash=repo.changectx(str(revision)).hex()
+  hg_hash=revsymbol(repo,b"%d" % revision).hex()
-  wr('data %d' % (len(hg_hash)))
+  wr(b'data %d' % (len(hg_hash)))
   wr_no_nl(hg_hash)
   wr()
   return checkpoint(count)
 
-  wr('data %d' % (len(desc)+1)) # wtf?
-  wr(desc)
-  wr()
-
 def export_tags(ui,repo,old_marks,mapping_cache,count,authors,tagsmap):
   l=repo.tagslist()
   for tag,node in l:
     # Remap the branch name
     tag=sanitize_name(tag,"tag",tagsmap)
     # ignore latest revision
-    if tag=='tip': continue
+    if tag==b'tip': continue
     # ignore tags to nodes that are missing (ie, 'in the future')
-    if node.encode('hex_codec') not in mapping_cache:
+    if hexlify(node) not in mapping_cache:
-      sys.stderr.write('Tag %s refers to unseen node %s\n' % (tag, node.encode('hex_codec')))
+      stderr_buffer.write(b'Tag %s refers to unseen node %s\n' % (tag, hexlify(node)))
       continue
 
-    rev=int(mapping_cache[node.encode('hex_codec')])
+    rev=int(mapping_cache[hexlify(node)])
 
     ref=revnum_to_revref(rev, old_marks)
     if ref==None:
-      sys.stderr.write('Failed to find reference for creating tag'
+      stderr_buffer.write(
-          ' %s at r%d\n' % (tag,rev))
+        b'Failed to find reference for creating tag %s at r%d\n' % (tag, rev)
+      )
       continue
-    sys.stderr.write('Exporting tag [%s] at [hg r%d] [git %s]\n' % (tag,rev,ref))
+    stderr_buffer.write(b'Exporting tag [%s] at [hg r%d] [git %s]\n' % (tag, rev, ref))
-    wr('reset refs/tags/%s' % tag)
+    wr(b'reset refs/tags/%s' % tag)
-    wr('from %s' % ref)
+    wr(b'from %s' % ref)
     wr()
     count=checkpoint(count)
   return count
 
-def load_mapping(name, filename):
+def load_mapping(name, filename, mapping_is_raw):
+  raw_regexp=re.compile(b'^([^=]+)[ ]*=[ ]*(.+)$')
+  string_regexp=b'"(((\\.)|(\\")|[^"])*)"'
+  quoted_regexp=re.compile(b'^'+string_regexp+b'[ ]*=[ ]*'+string_regexp+b'$')
+
+  def parse_raw_line(line):
+    m=raw_regexp.match(line)
+    if m==None:
+      return None
+    return (m.group(1).strip(), m.group(2).strip())
+
+  def process_unicode_escape_sequences(s):
+    # Replace unicode escape sequences in the otherwise UTF8-encoded bytestring s with
+    # the UTF8-encoded characters they represent. We need to do an additional
+    # .decode('utf8').encode('ascii', 'backslashreplace') to convert any non-ascii
+    # characters into their escape sequences so that the subsequent
+    # .decode('unicode-escape') succeeds:
+    return (
+      s.decode('utf8')
+      .encode('ascii', 'backslashreplace')
+      .decode('unicode-escape')
+      .encode('utf8')
+    )
+
+  def parse_quoted_line(line):
+    m=quoted_regexp.match(line)
+    if m==None:
+      return 
+    
+    return (process_unicode_escape_sequences(m.group(1)),
+            process_unicode_escape_sequences(m.group(5)))
+
   cache={}
   if not os.path.exists(filename):
     sys.stderr.write('Could not open mapping file [%s]\n' % (filename))
     return cache
-  f=open(filename,'r')
+  f=open(filename,'rb')
   l=0
   a=0
-  lre=re.compile('^([^=]+)[ ]*=[ ]*(.+)$')
   for line in f.readlines():
     l+=1
     line=line.strip()
-    if line=='' or line[0]=='#':
+    if l==1 and line[0:1]==b'#' and line==b'# quoted-escaped-strings':
       continue
-    m=lre.match(line)
+    elif line==b'' or line[0:1]==b'#':
+      continue
+    m=parse_raw_line(line) if mapping_is_raw else parse_quoted_line(line)
     if m==None:
       sys.stderr.write('Invalid file format in [%s], line %d\n' % (filename,l))
       continue
     # put key:value in cache, key without ^:
-    cache[m.group(1).strip()]=m.group(2).strip()
+    cache[m[0]]=m[1]
     a+=1
   f.close()
   sys.stderr.write('Loaded %d %s\n' % (a, name))
@@ -328,7 +486,7 @@ def branchtip(repo, heads):
       break
   return tip
 
-def verify_heads(ui,repo,cache,force,branchesmap):
+def verify_heads(ui,repo,cache,force,ignore_unnamed_heads,branchesmap):
   branches={}
   for bn, heads in repo.branchmap().iteritems():
     branches[bn] = branchtip(repo, heads)
@@ -341,26 +499,38 @@ def verify_heads(ui,repo,cache,force,branchesmap):
     sanitized_name=sanitize_name(b,"branch",branchesmap)
     sha1=get_git_sha1(sanitized_name)
     c=cache.get(sanitized_name)
-    if sha1!=c:
+    if not c and sha1:
-      sys.stderr.write('Error: Branch [%s] modified outside hg-fast-export:'
+      stderr_buffer.write(
-        '\n%s (repo) != %s (cache)\n' % (b,sha1,c))
+        b'Error: Branch [%s] already exists and was not created by hg-fast-export, '
+        b'export would overwrite unrelated branch\n' % b)
+      if not force: return False
+    elif sha1!=c:
+      stderr_buffer.write(
+        b'Error: Branch [%s] modified outside hg-fast-export:'
+        b'\n%s (repo) != %s (cache)\n' % (b, b'<None>' if sha1 is None else sha1, c)
+      )
       if not force: return False
 
   # verify that branch has exactly one head
   t={}
-  for h in repo.heads():
+  unnamed_heads=False
+  for h in repo.filtered(b'visible').heads():
     (_,_,_,_,_,_,branch,_)=get_changeset(ui,repo,h)
     if t.get(branch,False):
-      sys.stderr.write('Error: repository has at least one unnamed head: hg r%s\n' %
+      stderr_buffer.write(
-          repo.changelog.rev(h))
+        b'Error: repository has an unnamed head: hg r%d\n'
-      if not force: return False
+        % repo.changelog.rev(h)
+      )
+      unnamed_heads=True
+      if not force and not ignore_unnamed_heads: return False
     t[branch]=True
-
+  if unnamed_heads and not force and not ignore_unnamed_heads: return False
   return True
 
 def hg2git(repourl,m,marksfile,mappingfile,headsfile,tipfile,
            authors={},branchesmap={},tagsmap={},
-           sob=False,force=False,hgtags=False,notes=False,encoding='',fn_encoding=''):
+           sob=False,force=False,ignore_unnamed_heads=False,hgtags=False,notes=False,encoding='',fn_encoding='',
+           plugins={}):
   def check_cache(filename, contents):
     if len(contents) == 0:
       sys.stderr.write('Warning: %s does not contain any data, this will probably make an incremental import fail\n' % filename)
@@ -380,7 +550,7 @@ def hg2git(repourl,m,marksfile,mappingfile,headsfile,tipfile,
 
   ui,repo=setup_repo(repourl)
 
-  if not verify_heads(ui,repo,heads_cache,force,branchesmap):
+  if not verify_heads(ui,repo,heads_cache,force,ignore_unnamed_heads,branchesmap):
     return 1
 
   try:
@@ -388,27 +558,41 @@ def hg2git(repourl,m,marksfile,mappingfile,headsfile,tipfile,
   except AttributeError:
     tip=len(repo)
 
-  min=int(state_cache.get('tip',0))
+  min=int(state_cache.get(b'tip',0))
   max=_max
   if _max<0 or max>tip:
     max=tip
 
   for rev in range(0,max):
-  	(revnode,_,_,_,_,_,_,_)=get_changeset(ui,repo,rev,authors)
+    (revnode,_,_,_,_,_,_,_)=get_changeset(ui,repo,rev,authors)
-  	mapping_cache[revnode.encode('hex_codec')] = str(rev)
+    if repo[revnode].hidden():
+      continue
+    mapping_cache[hexlify(revnode)] = b"%d" % rev
 
+  if submodule_mappings:
+    # Make sure that all mercurial submodules are registered in the submodule-mappings file
+    for rev in range(0,max):
+      ctx=revsymbol(repo,b"%d" % rev)
+      if ctx.hidden():
+        continue
+      if ctx.substate:
+        for key in ctx.substate:
+          if ctx.substate[key][2]=='hg' and key not in submodule_mappings:
+            sys.stderr.write("Error: %s not found in submodule-mappings\n" % (key))
+            return 1
 
   c=0
   brmap={}
   for rev in range(min,max):
     c=export_commit(ui,repo,rev,old_marks,max,c,authors,branchesmap,
-                    sob,brmap,hgtags,encoding,fn_encoding)
+                    sob,brmap,hgtags,encoding,fn_encoding,
+                    plugins)
   if notes:
     for rev in range(min,max):
       c=export_note(ui,repo,rev,c,authors, encoding, rev == min and min != 0)
 
-  state_cache['tip']=max
+  state_cache[b'tip']=max
-  state_cache['repo']=repourl
+  state_cache[b'repo']=repourl
   save_cache(tipfile,state_cache)
   save_cache(mappingfile,mapping_cache)
 
@@ -426,6 +610,9 @@ if __name__=='__main__':
 
   parser=OptionParser()
 
+  parser.add_option("-n", "--no-auto-sanitize",action="store_false",
+      dest="auto_sanitize",default=True,
+      help="Do not perform built-in (broken in many cases) sanitizing of names")
   parser.add_option("-m","--max",type="int",dest="max",
       help="Maximum hg revision to import")
   parser.add_option("--mapping",dest="mappingfile",
@@ -449,7 +636,9 @@ if __name__=='__main__':
   parser.add_option("-T","--tags",dest="tagsfile",
       help="Read tags map from TAGSFILE")
   parser.add_option("-f","--force",action="store_true",dest="force",
-      default=False,help="Ignore validation errors by force")
+      default=False,help="Ignore validation errors by force, implies --ignore-unnamed-heads")
+  parser.add_option("--ignore-unnamed-heads",action="store_true",dest="ignore_unnamed_heads",
+      default=False,help="Ignore unnamed head errors")
   parser.add_option("-M","--default-branch",dest="default_branch",
       help="Set the default branch")
   parser.add_option("-o","--origin",dest="origin_name",
@@ -460,10 +649,21 @@ if __name__=='__main__':
       help="Assume commit and author strings retrieved from Mercurial are encoded in <encoding>")
   parser.add_option("--fe",dest="fn_encoding",
       help="Assume file names from Mercurial are encoded in <filename_encoding>")
+  parser.add_option("--mappings-are-raw",dest="raw_mappings", default=False,
+      help="Assume mappings are raw <key>=<value> lines")
+  parser.add_option("--filter-contents",dest="filter_contents",
+      help="Pipe contents of each exported file through FILTER_CONTENTS <file-path> <hg-hash> <is-binary>")
+  parser.add_option("--plugin-path", type="string", dest="pluginpath",
+      help="Additional search path for plugins ")
+  parser.add_option("--plugin", action="append", type="string", dest="plugins",
+      help="Add a plugin with the given init string <name=init>")
+  parser.add_option("--subrepo-map", type="string", dest="subrepo_map",
+      help="Provide a mapping file between the subrepository name and the submodule name")
 
   (options,args)=parser.parse_args()
 
   m=-1
+  auto_sanitize = options.auto_sanitize
   if options.max!=None: m=options.max
 
   if options.marksfile==None: bail(parser,'--marks')
@@ -472,17 +672,25 @@ if __name__=='__main__':
   if options.statusfile==None: bail(parser,'--status')
   if options.repourl==None: bail(parser,'--repo')
 
+  if options.subrepo_map:
+      if not os.path.exists(options.subrepo_map):
+        sys.stderr.write('Subrepo mapping file not found %s\n'
+                         % options.subrepo_map)
+        sys.exit(1)
+      submodule_mappings=load_mapping('subrepo mappings',
+                                      options.subrepo_map,False)
+
   a={}
   if options.authorfile!=None:
-    a=load_mapping('authors', options.authorfile)
+    a=load_mapping('authors', options.authorfile, options.raw_mappings)
 
   b={}
   if options.branchesfile!=None:
-    b=load_mapping('branches', options.branchesfile)
+    b=load_mapping('branches', options.branchesfile, options.raw_mappings)
 
   t={}
   if options.tagsfile!=None:
-    t=load_mapping('tags', options.tagsfile)
+    t=load_mapping('tags', options.tagsfile, options.raw_mappings)
 
   if options.default_branch!=None:
     set_default_branch(options.default_branch)
@@ -498,8 +706,36 @@ if __name__=='__main__':
   if options.fn_encoding!=None:
     fn_encoding=options.fn_encoding
 
+  plugins=[]
+  if options.plugins!=None:
+    plugins+=options.plugins
+
+  if options.filter_contents!=None:
+    plugins+=['shell_filter_file_contents='+options.filter_contents]
+
+  plugins_dict={}
+  plugins_dict['commit_message_filters']=[]
+  plugins_dict['file_data_filters']=[]
+
+  if plugins and options.pluginpath:
+    sys.stderr.write('Using additional plugin path: ' + options.pluginpath + '\n')
+
+  for plugin in plugins:
+    split = plugin.split('=')
+    name, opts = split[0], '='.join(split[1:])
+    i = pluginloader.get_plugin(name,options.pluginpath)
+    sys.stderr.write('Loaded plugin ' + i['name'] + ' from path: ' + i['path'] +' with opts: ' + opts + '\n')
+    plugin = pluginloader.load_plugin(i).build_filter(opts)
+    if hasattr(plugin,'file_data_filter') and callable(plugin.file_data_filter):
+      plugins_dict['file_data_filters'].append(plugin.file_data_filter)
+    if hasattr(plugin, 'commit_message_filter') and callable(plugin.commit_message_filter):
+      plugins_dict['commit_message_filters'].append(plugin.commit_message_filter)
+
   sys.exit(hg2git(options.repourl,m,options.marksfile,options.mappingfile,
                   options.headsfile, options.statusfile,
                   authors=a,branchesmap=b,tagsmap=t,
-                  sob=options.sob,force=options.force,hgtags=options.hgtags,
+                  sob=options.sob,force=options.force,
-                  notes=options.notes,encoding=encoding,fn_encoding=fn_encoding))
+                  ignore_unnamed_heads=options.ignore_unnamed_heads,
+                  hgtags=options.hgtags,
+                  notes=options.notes,encoding=encoding,fn_encoding=fn_encoding,
+                  plugins=plugins_dict))

hg-fast-export.sh

@@ -8,7 +8,7 @@ if command -v greadlink > /dev/null; then
   READLINK="greadlink" # Prefer greadlink over readlink
 fi
 
-if ! $READLINK -f "$(which "$0")" 2>&1 > /dev/null; then
+if ! $READLINK -f "$(which "$0")" > /dev/null 2>&1 ; then
     ROOT="$(dirname "$(which "$0")")"
     if [ ! -f "$ROOT/hg-fast-export.py" ] ; then
 	echo "hg-fast-exports requires a readlink implementation which knows" \
@@ -26,9 +26,26 @@ SFX_MARKS="marks"
 SFX_HEADS="heads"
 SFX_STATE="state"
 GFI_OPTS=""
-PYTHON=${PYTHON:-python}
 
-USAGE="[--quiet] [-r <repo>] [--force] [-m <max>] [-s] [--hgtags] [-A <file>] [-B <file>] [-T <file>] [-M <name>] [-o <name>] [--hg-hash] [-e <encoding>]"
+if [ -z "${PYTHON}" ]; then
+    # $PYTHON is not set, so we try to find a working python with mercurial:
+    for python_cmd in python2 python python3; do
+        if command -v $python_cmd > /dev/null; then
+            $python_cmd -c 'from mercurial.scmutil import revsymbol' 2> /dev/null
+            if [ $? -eq 0 ]; then
+                PYTHON=$python_cmd
+                break
+            fi
+        fi
+    done
+fi
+if [ -z "${PYTHON}" ]; then
+    echo "Could not find a python interpreter with the mercurial module >= 4.6 available. " \
+        "Please use the 'PYTHON' environment variable to specify the interpreter to use."
+    exit 1
+fi
+
+USAGE="[--quiet] [-r <repo>] [--force] [--ignore-unnamed-heads] [-m <max>] [-s] [--hgtags] [-A <file>] [-B <file>] [-T <file>] [-M <name>] [-o <name>] [--hg-hash] [-e <encoding>]"
 LONG_USAGE="Import hg repository <repo> up to either tip or <max>
 If <repo> is omitted, use last hg repository as obtained from state file,
 GIT_DIR/$PFX-$SFX_STATE by default.
@@ -48,6 +65,8 @@ Options:
 	-B <file> Read branch map from file
 	-T <file> Read tags map from file
 	-M <name> Set the default branch name (defaults to 'master')
+	-n        Do not perform built-in (broken in many cases) sanitizing
+                  of branch/tag names.
 	-o <name> Use <name> as branch namespace to track upstream (eg 'origin')
 	--hg-hash Annotate commits with the hg hash as git notes in the
                   hg namespace.
@@ -55,6 +74,11 @@ Options:
 	              Mercurial are encoded in <encoding>
 	--fe <filename_encoding> Assume filenames from Mercurial are encoded 
 	                         in <filename_encoding>
+	--mappings-are-raw Assume mappings are raw <key>=<value> lines
+	--filter-contents <cmd>  Pipe contents of each exported file through <cmd>
+	                         with <file-path> <hg-hash> <is-binary> as arguments
+	--plugin <plugin=init>  Add a plugin with the given init string (repeatable)
+	--plugin-path <plugin-path> Add an additional plugin lookup path
 "
 case "$1" in
     -h|--help)
@@ -152,7 +176,7 @@ $(
   exec 4>&3 3>&1 1>&4 4>&-
   {
     _e1=0
-    GIT_DIR="$GIT_DIR" $PYTHON "$ROOT/hg-fast-export.py" \
+    GIT_DIR="$GIT_DIR" "$PYTHON" "$ROOT/hg-fast-export.py" \
       --repo "$REPO" \
       --marks "$GIT_DIR/$PFX-$SFX_MARKS" \
       --mapping "$GIT_DIR/$PFX-$SFX_MAPPING" \

hg-reset.py

@@ -7,6 +7,7 @@ from mercurial import node
 from hg2git import setup_repo,load_cache,get_changeset,get_git_sha1
 from optparse import OptionParser
 import sys
+from binascii import hexlify
 
 def heads(ui,repo,start=None,stop=None,max=None):
   # this is copied from mercurial/revlog.py and differs only in
@@ -24,7 +25,7 @@ def heads(ui,repo,start=None,stop=None,max=None):
   heads = {startrev: 1}
 
   parentrevs = repo.changelog.parentrevs
-  for r in xrange(startrev + 1, max):
+  for r in range(startrev + 1, max):
     for p in parentrevs(r):
       if p in reachable:
         if r not in stoprevs:
@@ -33,7 +34,7 @@ def heads(ui,repo,start=None,stop=None,max=None):
       if p in heads and p not in stoprevs:
         del heads[p]
 
-  return [(repo.changelog.node(r),str(r)) for r in heads]
+  return [(repo.changelog.node(r), b"%d" % r) for r in heads]
 
 def get_branches(ui,repo,heads_cache,marks_cache,mapping_cache,max):
   h=heads(ui,repo,max=max)
@@ -44,11 +45,11 @@ def get_branches(ui,repo,heads_cache,marks_cache,mapping_cache,max):
     _,_,user,(_,_),_,desc,branch,_=get_changeset(ui,repo,rev)
     del stale[branch]
     git_sha1=get_git_sha1(branch)
-    cache_sha1=marks_cache.get(str(int(rev)+1))
+    cache_sha1=marks_cache.get(b"%d" % (int(rev)+1))
     if git_sha1!=None and git_sha1==cache_sha1:
-      unchanged.append([branch,cache_sha1,rev,desc.split('\n')[0],user])
+      unchanged.append([branch,cache_sha1,rev,desc.split(b'\n')[0],user])
     else:
-      changed.append([branch,cache_sha1,rev,desc.split('\n')[0],user])
+      changed.append([branch,cache_sha1,rev,desc.split(b'\n')[0],user])
   changed.sort()
   unchanged.sort()
   return stale,changed,unchanged
@@ -57,20 +58,20 @@ def get_tags(ui,repo,marks_cache,mapping_cache,max):
   l=repo.tagslist()
   good,bad=[],[]
   for tag,node in l:
-    if tag=='tip': continue
+    if tag==b'tip': continue
-    rev=int(mapping_cache[node.encode('hex_codec')])
+    rev=int(mapping_cache[hexlify(node)])
-    cache_sha1=marks_cache.get(str(int(rev)+1))
+    cache_sha1=marks_cache.get(b"%d" % (int(rev)+1))
     _,_,user,(_,_),_,desc,branch,_=get_changeset(ui,repo,rev)
     if int(rev)>int(max):
-      bad.append([tag,branch,cache_sha1,rev,desc.split('\n')[0],user])
+      bad.append([tag,branch,cache_sha1,rev,desc.split(b'\n')[0],user])
     else:
-      good.append([tag,branch,cache_sha1,rev,desc.split('\n')[0],user])
+      good.append([tag,branch,cache_sha1,rev,desc.split(b'\n')[0],user])
   good.sort()
   bad.sort()
   return good,bad
 
 def mangle_mark(mark):
-  return str(int(mark)-1)
+  return b"%d" % (int(mark)-1)
 
 if __name__=='__main__':
   def bail(parser,opt):
@@ -107,7 +108,7 @@ if __name__=='__main__':
   state_cache=load_cache(options.statusfile)
   mapping_cache = load_cache(options.mappingfile)
 
-  l=int(state_cache.get('tip',options.revision))
+  l=int(state_cache.get(b'tip',options.revision))
   if options.revision+1>l:
     sys.stderr.write('Revision is beyond last revision imported: %d>%d\n' % (options.revision,l))
     sys.exit(1)
@@ -117,19 +118,39 @@ if __name__=='__main__':
   stale,changed,unchanged=get_branches(ui,repo,heads_cache,marks_cache,mapping_cache,options.revision+1)
   good,bad=get_tags(ui,repo,marks_cache,mapping_cache,options.revision+1)
 
-  print "Possibly stale branches:"
+  print("Possibly stale branches:")
-  map(lambda b: sys.stdout.write('\t%s\n' % b),stale.keys())
+  for b in stale:
+    sys.stdout.write('\t%s\n' % b.decode('utf8'))
 
-  print "Possibly stale tags:"
+  print("Possibly stale tags:")
-  map(lambda b: sys.stdout.write('\t%s on %s (r%s)\n' % (b[0],b[1],b[3])),bad)
+  for b in bad:
+    sys.stdout.write(
+      '\t%s on %s (r%s)\n'
+      % (b[0].decode('utf8'), b[1].decode('utf8'), b[3].decode('utf8'))
+    )
 
-  print "Unchanged branches:"
+  print("Unchanged branches:")
-  map(lambda b: sys.stdout.write('\t%s (r%s)\n' % (b[0],b[2])),unchanged)
+  for b in unchanged:
+    sys.stdout.write('\t%s (r%s)\n' % (b[0].decode('utf8'),b[2].decode('utf8')))
 
-  print "Unchanged tags:"
+  print("Unchanged tags:")
-  map(lambda b: sys.stdout.write('\t%s on %s (r%s)\n' % (b[0],b[1],b[3])),good)
+  for b in good:
+    sys.stdout.write(
+      '\t%s on %s (r%s)\n'
+      % (b[0].decode('utf8'), b[1].decode('utf8'), b[3].decode('utf8'))
+    )
 
-  print "Reset branches in '%s' to:" % options.headsfile
+  print("Reset branches in '%s' to:" % options.headsfile)
-  map(lambda b: sys.stdout.write('\t:%s %s\n\t\t(r%s: %s: %s)\n' % (b[0],b[1],b[2],b[4],b[3])),changed)
+  for b in changed:
+    sys.stdout.write(
+      '\t:%s %s\n\t\t(r%s: %s: %s)\n'
+      % (
+        b[0].decode('utf8'),
+        b[1].decode('utf8'),
+        b[2].decode('utf8'),
+        b[4].decode('utf8'),
+        b[3].decode('utf8'),
+      )
+    )
 
-  print "Reset ':tip' in '%s' to '%d'" % (options.statusfile,options.revision)
+  print("Reset ':tip' in '%s' to '%d'" % (options.statusfile,options.revision))

hg-reset.sh

@@ -11,7 +11,24 @@ SFX_MAPPING="mapping"
 SFX_HEADS="heads"
 SFX_STATE="state"
 QUIET=""
-PYTHON=${PYTHON:-python}
+
+if [ -z "${PYTHON}" ]; then
+    # $PYTHON is not set, so we try to find a working python with mercurial:
+    for python_cmd in python2 python python3; do
+        if command -v $python_cmd > /dev/null; then
+            $python_cmd -c 'import mercurial' 2> /dev/null
+            if [ $? -eq 0 ]; then
+                PYTHON=$python_cmd
+                break
+            fi
+        fi
+    done
+fi
+if [ -z "${PYTHON}" ]; then
+    echo "Could not find a python interpreter with the mercurial module available. " \
+        "Please use the 'PYTHON'environment variable to specify the interpreter to use."
+    exit 1
+fi
 
 USAGE="[-r <repo>] -R <rev>"
 LONG_USAGE="Print SHA1s of latest changes per branch up to <rev> useful

hg2git.py

@@ -1,26 +1,36 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 # Copyright (c) 2007, 2008 Rocco Rutte <pdmef@gmx.net> and others.
 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
 
 from mercurial import hg,util,ui,templatefilters
+from mercurial import error as hgerror
+from mercurial.scmutil import revsymbol,binnode
+
 import re
 import os
 import sys
 import subprocess
 
+PY2 = sys.version_info.major < 3
+if PY2:
+  str = unicode
+  fsencode = lambda s: s.encode(sys.getfilesystemencoding())
+else:
+  from os import fsencode
+
 # default git branch name
-cfg_master='master'
+cfg_master=b'master'
 # default origin name
-origin_name=''
+origin_name=b''
 # silly regex to see if user field has email address
-user_re=re.compile('([^<]+) (<[^>]*>)$')
+user_re=re.compile(b'([^<]+) (<[^>]*>)$')
 # silly regex to clean out user names
-user_clean_re=re.compile('^["]([^"]+)["]$')
+user_clean_re=re.compile(b'^["]([^"]+)["]$')
 
 def set_default_branch(name):
   global cfg_master
-  cfg_master = name
+  cfg_master = name.encode('utf8') if not isinstance(name, bytes) else name
 
 def set_origin_name(name):
   global origin_name
@@ -31,24 +41,26 @@ def setup_repo(url):
     myui=ui.ui(interactive=False)
   except TypeError:
     myui=ui.ui()
-    myui.setconfig('ui', 'interactive', 'off')
+    myui.setconfig(b'ui', b'interactive', b'off')
-  return myui,hg.repository(myui,url)
+    # Avoids a warning when the repository has obsolete markers
+    myui.setconfig(b'experimental', b'evolution.createmarkers', True)
+  return myui,hg.repository(myui, fsencode(url)).unfiltered()
 
 def fixup_user(user,authors):
-  user=user.strip("\"")
+  user=user.strip(b"\"")
   if authors!=None:
     # if we have an authors table, try to get mapping
     # by defaulting to the current value of 'user'
     user=authors.get(user,user)
-  name,mail,m='','',user_re.match(user)
+  name,mail,m=b'',b'',user_re.match(user)
   if m==None:
     # if we don't have 'Name <mail>' syntax, extract name
     # and mail from hg helpers. this seems to work pretty well.
     # if email doesn't contain @, replace it with devnull@localhost
     name=templatefilters.person(user)
-    mail='<%s>' % util.email(user)
+    mail=b'<%s>' % templatefilters.email(user)
-    if '@' not in mail:
+    if b'@' not in mail:
-      mail = '<devnull@localhost>'
+      mail = b'<devnull@localhost>'
   else:
     # if we have 'Name <mail>' syntax, everything is fine :)
     name,mail=m.group(1),m.group(2)
@@ -57,25 +69,33 @@ def fixup_user(user,authors):
   m2=user_clean_re.match(name)
   if m2!=None:
     name=m2.group(1)
-  return '%s %s' % (name,mail)
+  return b'%s %s' % (name,mail)
 
 def get_branch(name):
   # 'HEAD' is the result of a bug in mutt's cvs->hg conversion,
   # other CVS imports may need it, too
-  if name=='HEAD' or name=='default' or name=='':
+  if name==b'HEAD' or name==b'default' or name==b'':
     name=cfg_master
   if origin_name:
-    return origin_name + '/' + name
+    return origin_name + b'/' + name
   return name
 
 def get_changeset(ui,repo,revision,authors={},encoding=''):
-  node=repo.lookup(revision)
+  # Starting with Mercurial 4.6 lookup no longer accepts raw hashes
+  # for lookups. Work around it by changing our behaviour depending on
+  # how it fails
+  try:
+    node=repo.lookup(revision)
+  except (TypeError, hgerror.ProgrammingError):
+    node=binnode(revsymbol(repo, b"%d" % revision)) # We were given a numeric rev
+  except hgerror.RepoLookupError:
+    node=revision # We got a raw hash
   (manifest,user,(time,timezone),files,desc,extra)=repo.changelog.read(node)
   if encoding:
     user=user.decode(encoding).encode('utf8')
     desc=desc.decode(encoding).encode('utf8')
-  tz="%+03d%02d" % (-timezone / 3600, ((-timezone % 3600) / 60))
+  tz=b"%+03d%02d" % (-timezone // 3600, ((-timezone % 3600) // 60))
-  branch=get_branch(extra.get('branch','master'))
+  branch=get_branch(extra.get(b'branch', b'master'))
   return (node,manifest,fixup_user(user,authors),(time,tz),files,desc,branch,extra)
 
 def mangle_key(key):
@@ -85,29 +105,35 @@ def load_cache(filename,get_key=mangle_key):
   cache={}
   if not os.path.exists(filename):
     return cache
-  f=open(filename,'r')
+  f=open(filename,'rb')
   l=0
   for line in f.readlines():
     l+=1
-    fields=line.split(' ')
+    fields=line.split(b' ')
-    if fields==None or not len(fields)==2 or fields[0][0]!=':':
+    if fields==None or not len(fields)==2 or fields[0][0:1]!=b':':
       sys.stderr.write('Invalid file format in [%s], line %d\n' % (filename,l))
       continue
     # put key:value in cache, key without ^:
-    cache[get_key(fields[0][1:])]=fields[1].split('\n')[0]
+    cache[get_key(fields[0][1:])]=fields[1].split(b'\n')[0]
   f.close()
   return cache
 
 def save_cache(filename,cache):
-  f=open(filename,'w+')
+  f=open(filename,'wb')
-  map(lambda x: f.write(':%s %s\n' % (str(x),str(cache.get(x)))),cache.keys())
+  for key, value in cache.items():
+    if not isinstance(key, bytes):
+      key = str(key).encode('utf8')
+    if not isinstance(value, bytes):
+      value = str(value).encode('utf8')
+    f.write(b':%s %s\n' % (key, value))
   f.close()
 
 def get_git_sha1(name,type='heads'):
   try:
     # use git-rev-parse to support packed refs
-    ref="refs/%s/%s" % (type,name)
+    ref="refs/%s/%s" % (type,name.decode('utf8'))
-    l=subprocess.check_output(["git", "rev-parse", "--verify", "--quiet", ref])
+    l=subprocess.check_output(["git", "rev-parse", "--verify",
+                               "--quiet", ref.encode('utf8')])
     if l == None or len(l) == 0:
       return None
     return l[0:40]

pluginloader/__init__.py

@@ -0,0 +1,19 @@
+import os
+import imp
+PluginFolder = os.path.join(os.path.dirname(os.path.realpath(__file__)),"..","plugins")
+MainModule = "__init__"
+
+def get_plugin(name, plugin_path):
+    search_dirs = [PluginFolder]
+    if plugin_path:
+        search_dirs = [plugin_path] + search_dirs
+    for dir in search_dirs:
+        location = os.path.join(dir, name)
+        if not os.path.isdir(location) or not MainModule + ".py" in os.listdir(location):
+            continue
+        info = imp.find_module(MainModule, [location])
+        return {"name": name, "info": info, "path": location}
+    raise Exception("Could not find plugin with name " + name)
+
+def load_plugin(plugin):
+    return imp.load_module(MainModule, *plugin["info"])

plugins/branch_name_in_commit/README.md

@@ -0,0 +1,20 @@
+## Branch Name in Commit Message
+
+Mercurial has a much stronger notion of branches than Git,
+and some parties may not wish to lose the branch information
+during the migration to Git. You can use this plugin to either
+prepend or append the branch name from the mercurial
+commit into the commit message in Git.
+
+Valid arguments are:
+
+- `start`: write the branch name at the start of the commit
+- `end`: write the branch name at the end of the commit
+- `sameline`: if `start` specified, put a colon and a space
+  after the branch name, such that the commit message reads
+  `branch_name: first line of commit message`. Otherwise, the
+  branch name is on the first line of the commit message by itself.
+- `skipmaster`: Don't write the branch name if the branch is `master`.
+
+To use the plugin, add
+`--plugin branch_name_in_commit=<comma_separated_list_of_args>`.

plugins/branch_name_in_commit/__init__.py

@@ -0,0 +1,25 @@
+def build_filter(args):
+    return Filter(args)
+
+class Filter:
+    def __init__(self, args):
+        args = {arg: True for arg in args.split(',')}
+        self.start = args.pop('start', False)
+        self.end = args.pop('end', False)
+        self.sameline = args.pop('sameline', False)
+        self.skip_master = args.pop('skipmaster', False)
+
+        if self.sameline and not self.start:
+            raise ValueError("sameline option only allowed if 'start' given")
+        if args:
+            raise ValueError("Unknown args: " + ','.join(args))
+
+    def commit_message_filter(self, commit_data):
+        if not (self.skip_master and commit_data['branch'] == b'master'):
+            if self.start:
+                sep = b': ' if self.sameline else b'\n'
+                commit_data['desc'] = commit_data['branch'] + sep + commit_data['desc']
+            if self.end:
+                commit_data['desc'] = (
+                    commit_data['desc'] + b'\n' + commit_data['branch']
+                )

plugins/dos2unix/README.md

@@ -0,0 +1,9 @@
+## Dos2unix filter
+
+This plugin converts CRLF line ending to LF in text files in the repo.
+It is recommended that you have a .gitattributes file that maintains
+the usage of LF endings going forward, for after you have converted your
+repository.
+
+To use the plugin, add
+`--plugin dos2unix`.

plugins/dos2unix/__init__.py

@@ -0,0 +1,11 @@
+def build_filter(args):
+    return Filter(args)
+
+class Filter():
+    def __init__(self, args):
+        pass
+
+    def file_data_filter(self,file_data):
+        file_ctx = file_data['file_ctx']
+        if not file_ctx.isbinary():
+            file_data['data'] = file_data['data'].replace(b'\r\n', b'\n')

plugins/drop/README.md

@@ -0,0 +1,12 @@
+## Drop commits from output
+
+To use the plugin, add the command line flag `--plugin drop=<spec>`.
+The flag can be given multiple times to drop more than one commit.
+
+The <spec> value can be either
+
+ - a comma-separated list of hg hashes in the full form (40
+   hexadecimal characters) to drop the corresponding changesets, or
+
+ - a regular expression pattern to drop all changesets with matching
+   descriptions.

plugins/drop/__init__.py

@@ -0,0 +1,61 @@
+from __future__ import print_function
+
+import sys, re
+
+
+def build_filter(args):
+    if re.match(r'([A-Fa-f0-9]{40}(,|$))+$', args):
+        return RevisionIdFilter(args.split(','))
+    else:
+        return DescriptionFilter(args)
+
+
+def log(fmt, *args):
+    print(fmt % args, file=sys.stderr)
+    sys.stderr.flush()
+
+
+class FilterBase(object):
+    def __init__(self):
+        self.remapped_parents = {}
+
+    def commit_message_filter(self, commit_data):
+        rev = commit_data['revision']
+
+        mapping = self.remapped_parents
+        parent_revs = [rp for p in commit_data['parents']
+                       for rp in mapping.get(p, [p])]
+
+        commit_data['parents'] = parent_revs
+
+        if self.should_drop_commit(commit_data):
+            log('Dropping revision %i.', rev)
+
+            self.remapped_parents[rev] = parent_revs
+
+            # Head commits cannot be dropped because they have no
+            # children, so detach them to a separate branch.
+            commit_data['branch'] = b'dropped-hg-head'
+            commit_data['parents'] = []
+
+    def should_drop_commit(self, commit_data):
+        return False
+
+
+class RevisionIdFilter(FilterBase):
+    def __init__(self, revision_hash_list):
+        super(RevisionIdFilter, self).__init__()
+        self.unwanted_hg_hashes = {h.encode('ascii', 'strict')
+                                   for h in revision_hash_list}
+
+    def should_drop_commit(self, commit_data):
+        return commit_data['hg_hash'] in self.unwanted_hg_hashes
+
+
+class DescriptionFilter(FilterBase):
+    def __init__(self, pattern):
+        super(DescriptionFilter, self).__init__()
+        self.pattern = re.compile(pattern.encode('ascii', 'strict'))
+
+    def should_drop_commit(self, commit_data):
+        return self.pattern.match(commit_data['desc'])

plugins/head2branch/README.md

@@ -0,0 +1,13 @@
+## Convert Head to Branch
+
+`fast-export` can only handle one head per branch. This plugin makes it possible
+to create a new branch from a head by specifying the new branch name and
+the first divergent commit for that head.
+
+Note: the hg hash must be in the full form, 40 hexadecimal characters.
+
+Note: you must run `fast-export` with `--ignore-unnamed-heads` option,
+otherwise, the conversion will fail.
+
+To use the plugin, add the command line flag `--plugin head2branch=name,<hg_hash>`.
+The flag can be given multiple times to name more than one head.

plugins/head2branch/__init__.py

@@ -0,0 +1,24 @@
+import sys
+
+def build_filter(args):
+    return Filter(args)
+
+class Filter:
+
+    def __init__(self, args):
+        args = args.split(',')
+        self.branch_name = args[0].encode('ascii', 'replace')
+        self.starting_commit_hash = args[1].encode('ascii', 'strict')
+        self.branch_parents = set()
+
+    def commit_message_filter(self, commit_data):
+        hg_hash = commit_data['hg_hash']
+        rev = commit_data['revision']
+        rev_parents = commit_data['parents']
+        if (hg_hash == self.starting_commit_hash
+            or any(rp in self.branch_parents for rp in rev_parents)
+            ):
+            self.branch_parents.add(rev)
+            commit_data['branch'] = self.branch_name
+            sys.stderr.write('\nchanging r%s to branch %r\n' % (rev, self.branch_name))
+            sys.stderr.flush()

plugins/issue_prefix/README.md

@@ -0,0 +1,19 @@
+## Issue Prefix
+
+When migrating to other source code hosting sites, there are cases where a
+project maintainer might want to reset their issue tracker and not have old
+issue numbers in commit messages referring to the wrong issue.  One way around
+this is to prefix issue numbers with some other string.
+
+If migrating to GitHub, this issue prefixing can be paired with GitHub's
+autolinking capabilitiy to link back to a different issue tracker:
+https://help.github.com/en/github/administering-a-repository/configuring-autolinks-to-reference-external-resources
+
+To use this plugin, add:
+`--plugin=issue_prefix=<some_prefix>`
+
+Example:
+`--plugin=issue_prefix=BB-`
+
+This will prefix issue numbers with the string `BB-`.  Example: `#123` will
+change to `#BB-123`.

plugins/issue_prefix/__init__.py

@@ -0,0 +1,17 @@
+# encoding=UTF-8
+"""__init__.py"""
+import re
+
+def build_filter(args):
+    return Filter(args)
+
+class Filter:
+    def __init__(self, args):
+        if not isinstance(args, bytes):
+            args = args.encode('utf8') 
+        self.prefix = args
+
+    def commit_message_filter(self, commit_data):
+        for match in re.findall(b'#[1-9][0-9]+', commit_data['desc']):
+            commit_data['desc'] = commit_data['desc'].replace(
+                match, b'#%s%s' % (self.prefix, match[1:]))

plugins/overwrite_null_messages/README.md

@@ -0,0 +1,23 @@
+## Overwrite Null Commit Messages
+
+There are cases (such as when creating a new, empty snippet on bitbucket
+before they deprecated mercurial repositories) where you could create a
+new repo with a single commit in it, but the message would be null.  Then,
+when attempting to convert this repository to a git repo and pushing to 
+a new host, the git push would fail with an error like this:
+
+    error: a NUL byte in commit log message not allowed
+
+To get around this, you may provide a string that will be used in place of
+a null byte in commit messages.
+
+To use the plugin, add
+
+    --plugin overwrite_null_messages=""
+
+This will use the default commit message `"<empty commit message>"`.
+
+Or to specify a different commit message, you may pass this in at the
+command line like so:
+
+    --plugin overwrite_null_messages="use this message instead"

plugins/overwrite_null_messages/__init__.py

@@ -0,0 +1,16 @@
+def build_filter(args):
+    return Filter(args)
+
+class Filter:
+    def __init__(self, args):
+        if args == '':
+            message = b'<empty commit message>'
+        else:
+            message = args.encode('utf8')
+        self.message = message
+
+    def commit_message_filter(self,commit_data):
+        # Only write the commit message if the recorded commit
+        # message is null.
+        if commit_data['desc'] == b'\x00':
+            commit_data['desc'] = self.message

plugins/shell_filter_file_contents/README.md

@@ -0,0 +1,30 @@
+## Shell Script File Filter
+
+This plugin uses shell scripts in order to perform filtering of files.
+If your preferred scripting is done via shell, this tool is for you.
+Be noted, though, that this method can cause an order of magnitude slow
+down. For small repositories, this wont be an issue.
+
+To use the plugin, add
+`--plugin shell_filter_file_contents=path/to/shell/script.sh`.
+The filter script is supplied to the plugin option after the plugin name,
+which is in turned passed to the plugin initialization. hg-fast-export
+runs the filter for each exported file, pipes its content to the filter's
+standard input, and uses the filter's standard output in place
+of the file's original content. An example use of this feature
+is to convert line endings in text files from CRLF to git's preferred LF,
+although this task is faster performed using the native plugin.
+
+The script is called with the following syntax:
+`FILTER_CONTENTS <file-path> <hg-hash> <is-binary>`
+
+```
+-- Start of crlf-filter.sh --
+#!/bin/sh
+# $1 = pathname of exported file relative to the root of the repo
+# $2 = Mercurial's hash of the file
+# $3 = "1" if Mercurial reports the file as binary, otherwise "0"
+
+if [ "$3" == "1" ]; then cat; else dos2unix; fi
+-- End of crlf-filter.sh --
+```

plugins/shell_filter_file_contents/__init__.py

@@ -0,0 +1,28 @@
+#Pipe contents of each exported file through FILTER_CONTENTS <file-path> <hg-hash> <is-binary>"
+import subprocess
+import shlex
+import sys
+from mercurial import node
+
+def build_filter(args):
+    return Filter(args)
+
+class Filter:
+    def __init__(self, args):
+        self.filter_contents = shlex.split(args)
+
+    def file_data_filter(self,file_data):
+        d = file_data['data']
+        file_ctx = file_data['file_ctx']
+        filename = file_data['filename']
+        filter_cmd = self.filter_contents + [filename, node.hex(file_ctx.filenode()), '1' if file_ctx.isbinary() else '0']
+        try:
+            filter_proc = subprocess.Popen(filter_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+            d, _ = filter_proc.communicate(d)
+        except:
+            sys.stderr.write('Running filter-contents %s:\n' % filter_cmd)
+            raise
+        filter_ret = filter_proc.poll()
+        if filter_ret:
+            raise subprocess.CalledProcessError(filter_ret, filter_cmd)
+        file_data['data'] = d

tests/test_drop_plugin.py

@@ -0,0 +1,223 @@
+import sys, os, subprocess
+from tempfile import TemporaryDirectory
+from unittest import TestCase
+from pathlib import Path
+
+
+class CommitDropTest(TestCase):
+    def test_drop_single_commit_by_hash(self):
+        hash1 = self.create_commit('commit 1')
+        self.create_commit('commit 2')
+
+        self.drop(hash1)
+
+        self.assertEqual(['commit 2'], self.git.log())
+
+    def test_drop_commits_by_desc(self):
+        self.create_commit('commit 1 is good')
+        self.create_commit('commit 2 is bad')
+        self.create_commit('commit 3 is good')
+        self.create_commit('commit 4 is bad')
+
+        self.drop('.*bad')
+
+        expected = ['commit 1 is good', 'commit 3 is good']
+        self.assertEqual(expected, self.git.log())
+
+    def test_drop_sequential_commits_in_single_plugin_instance(self):
+        self.create_commit('commit 1')
+        hash2 = self.create_commit('commit 2')
+        hash3 = self.create_commit('commit 3')
+        hash4 = self.create_commit('commit 4')
+        self.create_commit('commit 5')
+
+        self.drop(','.join((hash2, hash3, hash4)))
+
+        expected = ['commit 1', 'commit 5']
+        self.assertEqual(expected, self.git.log())
+
+    def test_drop_sequential_commits_in_multiple_plugin_instances(self):
+        self.create_commit('commit 1')
+        hash2 = self.create_commit('commit 2')
+        hash3 = self.create_commit('commit 3')
+        hash4 = self.create_commit('commit 4')
+        self.create_commit('commit 5')
+
+        self.drop(hash2, hash3, hash4)
+
+        expected = ['commit 1', 'commit 5']
+        self.assertEqual(expected, self.git.log())
+
+    def test_drop_nonsequential_commits(self):
+        self.create_commit('commit 1')
+        hash2 = self.create_commit('commit 2')
+        self.create_commit('commit 3')
+        hash4 = self.create_commit('commit 4')
+
+        self.drop(','.join((hash2, hash4)))
+
+        expected = ['commit 1', 'commit 3']
+        self.assertEqual(expected, self.git.log())
+
+    def test_drop_head(self):
+        self.create_commit('first')
+        self.create_commit('middle')
+        hash_last = self.create_commit('last')
+
+        self.drop(hash_last)
+
+        self.assertEqual(['first', 'middle'], self.git.log())
+
+    def test_drop_merge_commit(self):
+        initial_hash = self.create_commit('initial')
+        self.create_commit('branch A')
+        self.hg.checkout(initial_hash)
+        self.create_commit('branch B')
+        self.hg.merge()
+        merge_hash = self.create_commit('merge to drop')
+        self.create_commit('last')
+
+        self.drop(merge_hash)
+
+        expected_commits = ['initial', 'branch A', 'branch B', 'last']
+        self.assertEqual(expected_commits, self.git.log())
+        self.assertEqual(['branch B', 'branch A'], self.git_parents('last'))
+
+    def test_drop_different_commits_in_multiple_plugin_instances(self):
+        self.create_commit('good commit')
+        bad_hash = self.create_commit('bad commit')
+        self.create_commit('awful commit')
+        self.create_commit('another good commit')
+
+        self.drop('^awful.*', bad_hash)
+
+        expected = ['good commit', 'another good commit']
+        self.assertEqual(expected, self.git.log())
+
+    def test_drop_same_commit_in_multiple_plugin_instances(self):
+        self.create_commit('good commit')
+        bad_hash = self.create_commit('bad commit')
+        self.create_commit('another good commit')
+
+        self.drop('^bad.*', bad_hash)
+
+        expected = ['good commit', 'another good commit']
+        self.assertEqual(expected, self.git.log())
+
+    def setUp(self):
+        self.tempdir = TemporaryDirectory()
+
+        self.hg = HgDriver(Path(self.tempdir.name) / 'hgrepo')
+        self.hg.init()
+
+        self.git = GitDriver(Path(self.tempdir.name) / 'gitrepo')
+        self.git.init()
+
+        self.export = ExportDriver(self.hg.repodir, self.git.repodir)
+
+    def tearDown(self):
+        self.tempdir.cleanup()
+
+    def create_commit(self, message):
+        self.write_file_data('Data for %r.' % message)
+        return self.hg.commit(message)
+
+    def write_file_data(self, data, filename='test_file.txt'):
+        path = self.hg.repodir / filename
+        with path.open('w') as f:
+            print(data, file=f)
+
+    def drop(self, *spec):
+        self.export.run_with_drop(*spec)
+
+    def git_parents(self, message):
+        matches = self.git.grep_log(message)
+        if len(matches) != 1:
+            raise Exception('No unique commit with message %r.' % message)
+        subject, parents = self.git.details(matches[0])
+        return [self.git.details(p)[0] for p in parents]
+
+
+class ExportDriver:
+    def __init__(self, sourcedir, targetdir, *, quiet=True):
+        self.sourcedir = Path(sourcedir)
+        self.targetdir = Path(targetdir)
+        self.quiet = quiet
+        self.python_executable = str(
+            Path.cwd() / os.environ.get('PYTHON', sys.executable))
+        self.script = Path(__file__).parent / '../hg-fast-export.sh'
+
+    def run_with_drop(self, *plugin_args):
+        cmd = [self.script, '-r', str(self.sourcedir)]
+        for arg in plugin_args:
+            cmd.extend(['--plugin', 'drop=' + arg])
+        output = subprocess.DEVNULL if self.quiet else None
+        subprocess.run(cmd, check=True, cwd=str(self.targetdir),
+                       env={'PYTHON': self.python_executable},
+                       stdout=output, stderr=output)
+
+
+class HgDriver:
+    def __init__(self, repodir):
+        self.repodir = Path(repodir)
+
+    def init(self):
+        self.repodir.mkdir()
+        self.run_command('init')
+
+    def commit(self, message):
+        self.run_command('commit', '-A', '-m', message)
+        return self.run_command('id', '--id', '--debug').strip()
+
+    def log(self):
+        output = self.run_command('log', '-T', '{desc}\n')
+        commits = output.strip().splitlines()
+        commits.reverse()
+        return commits
+
+    def checkout(self, rev):
+        self.run_command('checkout', '-r', rev)
+
+    def merge(self):
+        self.run_command('merge', '--tool', ':local')
+
+    def run_command(self, *args):
+        p = subprocess.run(('hg', '-yq') + args,
+                           cwd=str(self.repodir),
+                           check=True,
+                           text=True,
+                           capture_output=True)
+        return p.stdout
+
+
+class GitDriver:
+    def __init__(self, repodir):
+        self.repodir = Path(repodir)
+
+    def init(self):
+        self.repodir.mkdir()
+        self.run_command('init')
+
+    def log(self):
+        output = self.run_command('log', '--format=%s', '--reverse')
+        return output.strip().splitlines()
+
+    def grep_log(self, pattern):
+        output = self.run_command('log', '--format=%H',
+                                  '-F', '--grep', pattern)
+        return output.strip().splitlines()
+
+    def details(self, commit_hash):
+        fmt = '%s%n%P'
+        output = self.run_command('show', '-s', '--format=' + fmt,
+                                  commit_hash)
+        subject, parents = output.splitlines()
+        return subject, parents.split()
+
+    def run_command(self, *args):
+        p = subprocess.run(('git', '--no-pager') + args,
+                           cwd=str(self.repodir),
+                           check=True,
+                           text=True,
+                           capture_output=True)
+        return p.stdout