Merge branch 'PR/305' into frej/felipec-pr-spree

Closes #305

.github/contributing.md

@@ -0,0 +1,28 @@
+When submitting a patch make sure the commits in your pull request:
+
+* Have good commit messages
+
+  Please read Chris Beams' blog post [How to Write a Git Commit
+  Message](https://chris.beams.io/posts/git-commit/) on how to write a
+  good commit message. Although the article recommends at most 50
+  characters for the subject, up to 72 characters are frequently
+  accepted for fast-export.
+
+* Adhere to good [commit
+hygiene](http://www.ericbmerritt.com/2011/09/21/commit-hygiene-and-git.html)
+
+  When developing a pull request for hg-fast-export, base your work on
+  the current `master` branch and rebase your work if it no longer can
+  be merged into the current `master` without conflicts. Never merge
+  `master` into your development branch, rebase if your work needs
+  updates from `master`.
+
+  When a pull request is modified due to review feedback, please
+  incorporate the changes into the proper commit. A good reference on
+  how to modify history is in the [Pro Git book, Section
+  7.6](https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History).
+
+Please do not submit a pull request if you are not willing to spend
+the time required to address review comments or revise the patch until
+it follows the guidelines above. A _take it or leave it_ approach to
+contributing wastes both your and the maintainer's time.

.github/workflows/ci.yml

@@ -0,0 +1,31 @@
+name: CI
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [master]
+
+jobs:
+  test:
+    name: Run test suite
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 1
+        submodules: 'recursive'
+
+    - name: Run tests
+      run: make -C t
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: python
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2

.github/workflows/codeql-analysis.yml

@@ -1,71 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-name: "CodeQL"
-
-on:
-  push:
-    branches: [master]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [master]
-  schedule:
-    - cron: '0 15 * * 4'
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-      matrix:
-        # Override automatic language detection by changing the below list
-        # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
-        language: ['python']
-        # Learn more...
-        # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-      with:
-        # We must fetch at least the immediate parents so that if this is
-        # a pull request then we can checkout the head.
-        fetch-depth: 2
-
-    # If this run was triggered by a pull request event, then checkout
-    # the head of the pull request instead of the merge commit.
-    - run: git checkout HEAD^2
-      if: ${{ github.event_name == 'pull_request' }}
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file. 
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    #- run: |
-    #   make bootstrap
-    #   make release
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "t/sharness"]
+	path = t/sharness
+	url = https://github.com/felipec/sharness.git

README-SUBMODULES.md

@@ -27,10 +27,10 @@ command line option.
 
 ## Example
 
-Example mercurial repo folder structure (~/mercurial):
+Example mercurial repo folder structure (~/mercurial) containing two subrepos:
     src/...
-    subrepo/subrepo1
+    subrepos/subrepo1
-    subrepo/subrepo2
+    subrepos/subrepo2
 
 ### Setup
 Create an empty new folder where all the converted git modules will be imported:
@@ -41,18 +41,18 @@ Create an empty new folder where all the converted git modules will be imported:
     mkdir submodule1
     cd submodule1
     git init
-    hg-fast-export.sh -r ~/mercurial/subrepo1
+    hg-fast-export.sh -r ~/mercurial/subrepos/subrepo1
     cd ..
     mkdir submodule2
     cd submodule2
     git init
-    hg-fast-export.sh -r ~/mercurial/subrepo2
+    hg-fast-export.sh -r ~/mercurial/subrepos/subrepo2
 
 ### Create mapping file
     cd ~/imported-gits
     cat > submodule-mappings << EOF
-    "subrepo/subrepo1"="../submodule1"
+    "subrepos/subrepo1"="../submodule1"
-    "subrepo/subrepo2"="../submodule2"
+    "subrepos/subrepo2"="../submodule2"
     EOF
 
 ### Convert main repository
@@ -60,16 +60,16 @@ Create an empty new folder where all the converted git modules will be imported:
     mkdir git-main-repo
     cd git-main-repo
     git init
-    hg-fast-export.sh -r ~/mercurial --subrepo-map=../submodule-mappings
+    hg-fast-export.sh -r ~/mercurial --subrepo-map=~/imported-gits/submodule-mappings
 
 ### Result
-The resulting repository will now contain the subrepo/subrepo1 and
+The resulting repository will now contain the submodules at the paths
-subrepo/subrepo1 submodules. The created .gitmodules file will look
+`subrepos/subrepo1` and `subrepos/subrepo2`. The created .gitmodules
-like:
+file will look like:
 
-    [submodule "subrepo/subrepo1"]
+    [submodule "subrepos/subrepo1"]
-          path = subrepo/subrepo1
+          path = subrepos/subrepo1
           url = ../submodule1
-    [submodule "subrepo/subrepo2"]
+    [submodule "subrepos/subrepo2"]
-          path = subrepo/subrepo2
+          path = subrepos/subrepo2
           url = ../submodule2

README.md

@@ -1,4 +1,4 @@
-hg-fast-export.(sh|py) - mercurial to git converter using git-fast-import
+hg-fast-export.sh - mercurial to git converter using git-fast-import
 =========================================================================
 
 Legal
@@ -43,11 +43,10 @@ Usage
 Using hg-fast-export is quite simple for a mercurial repository <repo>:
 
 ```
-mkdir repo-git # or whatever
+git init repo-git # or whatever
 cd repo-git
-git init
 hg-fast-export.sh -r <local-repo>
-git checkout HEAD
+git checkout
 ```
 
 Please note that hg-fast-export does not automatically check out the
@@ -133,7 +132,10 @@ is to convert line endings in text files from CRLF to git's preferred LF:
 # $2 = Mercurial's hash of the file
 # $3 = "1" if Mercurial reports the file as binary, otherwise "0"
 
-if [ "$3" == "1" ]; then cat; else dos2unix; fi
+if [ "$3" == "1" ]; then cat; else dos2unix -q; fi
+# -q option in call to dos2unix allows to avoid returning an
+# error code when handling non-ascii based text files (like UTF-16
+# encoded text files)
 -- End of crlf-filter.sh --
 ```
 
@@ -167,7 +169,7 @@ defined filter methods in the [dos2unix](./plugins/dos2unix) and
 [branch_name_in_commit](./plugins/branch_name_in_commit) plugins.
 
 ```
-commit_data = {'branch': branch, 'parents': parents, 'author': author, 'desc': desc, 'revision': revision, 'hg_hash': hg_hash, 'committer': 'committer'}
+commit_data = {'branch': branch, 'parents': parents, 'author': author, 'desc': desc, 'revision': revision, 'hg_hash': hg_hash, 'committer': 'committer', 'extra': extra}
 
 def commit_message_filter(self,commit_data):
 ```
@@ -198,11 +200,15 @@ Notes/Limitations
 
 hg-fast-export supports multiple branches but only named branches with
 exactly one head each. Otherwise commits to the tip of these heads
-within the branch will get flattened into merge commits. Chris J
+within the branch will get flattened into merge commits. There are a
-Billington's [hg-export-tool] can help you to handle branches with
+few options to deal with this:
-duplicate heads.
+1. Chris J Billington's [hg-export-tool] can help you to handle branches with
-Alternatively, you can use the [head2branch plugin](./plugins/head2branch)
+   duplicate heads.
-to create a new named branch from an unnamed head.
+2. Use the [head2branch plugin](./plugins/head2branch) to create a new named
+   branch from an unnamed head.
+3. You can ignore unnamed heads with the `--ignore-unnamed-heads` option, which
+   is appropriate in situations such as the extra heads being close commits
+   (abandoned, unmerged changes).
 
 hg-fast-export will ignore any files or directories tracked by mercurial
 called `.git`, and will print a warning if it encounters one. Git cannot
@@ -221,8 +227,8 @@ possible to use hg-fast-export on remote repositories
 Design
 ------
 
-hg-fast-export.py was designed in a way that doesn't require a 2-pass
+hg-fast-export was designed in a way that doesn't require a 2-pass
-mechanism or any prior repository analysis: if just feeds what it
+mechanism or any prior repository analysis: it just feeds what it
 finds into git-fast-import. This also implies that it heavily relies
 on strictly linear ordering of changesets from hg, i.e. its
 append-only storage model so that changesets hg-fast-export already
@@ -258,6 +264,10 @@ hygiene](http://www.ericbmerritt.com/2011/09/21/commit-hygiene-and-git.html)
   how to modify history is in the [Pro Git book, Section
   7.6](https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History).
 
+Please do not submit a pull request if you are not willing to spend
+the time required to address review comments or revise the patch until
+it follows the guidelines above. A _take it or leave it_ approach to
+contributing wastes both your and the maintainer's time.
 
 Frequent Problems
 =================
@@ -301,4 +311,11 @@ Frequent Problems
   git it looks like you have deleted all files, when in fact they have
   never been checked out. Just do a checkout of the branch you want.
 
+* `Error: repository has at least one unnamed head: hg r<N>`
+
+  By design, hg-fast-export cannot deal with extra heads on a branch.
+  There are a few options depending on whether the extra heads are
+  in-use/open or normally closed. See [Notes/Limitations](#noteslimitations)
+  section for more details.
+
 [hg-export-tool]: https://github.com/chrisjbillington/hg-export-tool

hg-fast-export.py

@@ -3,8 +3,6 @@
 # Copyright (c) 2007, 2008 Rocco Rutte <pdmef@gmx.net> and others.
 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
 
-from mercurial import node
-from mercurial.scmutil import revsymbol
 from hg2git import setup_repo,fixup_user,get_branch,get_changeset
 from hg2git import load_cache,save_cache,get_git_sha1,set_default_branch,set_origin_name
 from optparse import OptionParser
@@ -51,10 +49,13 @@ def wr_no_nl(msg=b''):
     stdout_buffer.write(msg)
 
 def wr(msg=b''):
-  wr_no_nl(msg)
+  wr_no_nl(msg + b'\n')
-  stdout_buffer.write(b'\n')
   #map(lambda x: sys.stderr.write('\t[%s]\n' % x),msg.split('\n'))
 
+def wr_data(data):
+  wr(b'data %d' % (len(data)))
+  wr(data)
+
 def checkpoint(count):
   count=count+1
   if cfg_checkpoint_count>0 and count%cfg_checkpoint_count==0:
@@ -68,39 +69,15 @@ def revnum_to_revref(rev, old_marks):
   or a mark)"""
   return old_marks.get(rev) or b':%d' % (rev+1)
 
-def file_mismatch(f1,f2):
+def get_filechanges(repo,revision,parents,files):
-  """See if two revisions of a file are not equal."""
-  return node.hex(f1)!=node.hex(f2)
-
-def split_dict(dleft,dright,l=[],c=[],r=[],match=file_mismatch):
-  """Loop over our repository and find all changed and missing files."""
-  for left in dleft.keys():
-    right=dright.get(left,None)
-    if right==None:
-      # we have the file but our parent hasn't: add to left set
-      l.append(left)
-    elif match(dleft[left],right) or gitmode(dleft.flags(left))!=gitmode(dright.flags(left)):
-      # we have it but checksums mismatch: add to center set
-      c.append(left)
-  for right in dright.keys():
-    left=dleft.get(right,None)
-    if left==None:
-      # if parent has file but we don't: add to right set
-      r.append(right)
-    # change is already handled when comparing child against parent
-  return l,c,r
-
-def get_filechanges(repo,revision,parents,mleft):
   """Given some repository and revision, find all changed/deleted files."""
-  l,c,r=[],[],[]
+  if not parents:
-  for p in parents:
+    # first revision: feed in full manifest
-    if p<0: continue
+    return files,[]
-    mright=revsymbol(repo,b"%d" %p).manifest()
+  else:
-    l,c,r=split_dict(mleft,mright,l,c,r)
+    # take the changes from the first parent
-  l.sort()
+    f=repo.status(parents[0],revision)
-  c.sort()
+    return f.modified+f.added,f.removed
-  r.sort()
-  return l,c,r
 
 def get_author(logmessage,committer,authors):
   """As git distincts between author and committer of a patch, try to
@@ -197,8 +174,7 @@ def refresh_gitmodules(ctx):
 
   if len(gitmodules):
     wr(b'M 100644 inline .gitmodules')
-    wr(b'data %d' % (len(gitmodules)+1))
+    wr_data(gitmodules)
-    wr(gitmodules)
 
 def export_file_contents(ctx,manifest,files,hgtags,encoding='',plugins={}):
   count=0
@@ -266,7 +242,7 @@ def sanitize_name(name,what="branch", mapping={}):
   if not auto_sanitize:
     return mapping.get(name,name)
   n=mapping.get(name,name)
-  p=re.compile(b'([[ ~^:?\\\\*]|\.\.)')
+  p=re.compile(b'([\\[ ~^:?\\\\*]|\.\.)')
   n=p.sub(b'_', n)
   if n[-1:] in (b'/', b'.'): n=n[:-1]+b'_'
   n=b'/'.join([dot(s) for s in n.split(b'/')])
@@ -294,28 +270,31 @@ def export_commit(ui,repo,revision,old_marks,max,count,authors,
     brmap[name]=n
     return n
 
-  (revnode,_,user,(time,timezone),files,desc,branch,_)=get_changeset(ui,repo,revision,authors,encoding)
+  ctx=repo[revision]
-  if repo[revnode].hidden():
+
+  if ctx.hidden():
     return count
 
+  (_,user,(time,timezone),files,desc,branch,extra)=get_changeset(ui,repo,revision,authors,encoding)
+
   branch=get_branchname(branch)
 
   parents = [p for p in repo.changelog.parentrevs(revision) if p >= 0]
   author = get_author(desc,user,authors)
-  hg_hash=revsymbol(repo,b"%d" % revision).hex()
+  hg_hash=ctx.hex()
 
   if plugins and plugins['commit_message_filters']:
     commit_data = {'branch': branch, 'parents': parents,
                    'author': author, 'desc': desc,
                    'revision': revision, 'hg_hash': hg_hash,
-                   'committer': user}
+                   'committer': user, 'extra': extra}
     for filter in plugins['commit_message_filters']:
       filter(commit_data)
     branch = commit_data['branch']
     parents = commit_data['parents']
     author = commit_data['author']
     user = commit_data['committer']
-    desc = commit_data['desc']
+    desc = commit_data['desc'] + b'\n'
 
   if len(parents)==0 and revision != 0:
     wr(b'reset refs/heads/%s' % branch)
@@ -325,39 +304,25 @@ def export_commit(ui,repo,revision,old_marks,max,count,authors,
   if sob:
     wr(b'author %s %d %s' % (author,time,timezone))
   wr(b'committer %s %d %s' % (user,time,timezone))
-  wr(b'data %d' % (len(desc)+1)) # wtf?
+  wr_data(desc)
-  wr(desc)
-  wr()
 
-  ctx=revsymbol(repo, b"%d" % revision)
   man=ctx.manifest()
-  added,changed,removed,type=[],[],[],''
 
-  if len(parents) == 0:
+  if not parents:
-    # first revision: feed in full manifest
-    added=man.keys()
-    added.sort()
     type='full'
   else:
     wr(b'from %s' % revnum_to_revref(parents[0], old_marks))
     if len(parents) == 1:
-      # later non-merge revision: feed in changed manifest
-      # if we have exactly one parent, just take the changes from the
-      # manifest without expensively comparing checksums
-      f=repo.status(parents[0],revnode)
-      added,changed,removed=f.added,f.modified,f.removed
       type='simple delta'
     else: # a merge with two parents
       wr(b'merge %s' % revnum_to_revref(parents[1], old_marks))
-      # later merge revision: feed in changed manifest
-      # for many files comparing checksums is expensive so only do it for
-      # merges where we really need it due to hg's revlog logic
-      added,changed,removed=get_filechanges(repo,revision,parents,man)
       type='thorough delta'
 
+  modified,removed=get_filechanges(repo,revision,parents,files)
+
   stderr_buffer.write(
-    b'%s: Exporting %s revision %d/%d with %d/%d/%d added/changed/removed files\n'
+    b'%s: Exporting %s revision %d/%d with %d/%d modified/removed files\n'
-    % (branch, type.encode(), revision + 1, max, len(added), len(changed), len(removed))
+    % (branch, type.encode(), revision + 1, max, len(modified), len(removed))
   )
 
   for filename in removed:
@@ -368,18 +333,18 @@ def export_commit(ui,repo,revision,old_marks,max,count,authors,
       remove_gitmodules(ctx)
     wr(b'D %s' % filename)
 
-  export_file_contents(ctx,man,added,hgtags,fn_encoding,plugins)
+  export_file_contents(ctx,man,modified,hgtags,fn_encoding,plugins)
-  export_file_contents(ctx,man,changed,hgtags,fn_encoding,plugins)
   wr()
 
   return checkpoint(count)
 
 def export_note(ui,repo,revision,count,authors,encoding,is_first):
-  (revnode,_,user,(time,timezone),_,_,_,_)=get_changeset(ui,repo,revision,authors,encoding)
+  ctx = repo[revision]
-  if repo[revnode].hidden():
+
+  if ctx.hidden():
     return count
 
-  parents = [p for p in repo.changelog.parentrevs(revision) if p >= 0]
+  (_,user,(time,timezone),_,_,_,_)=get_changeset(ui,repo,revision,authors,encoding)
 
   wr(b'commit refs/notes/hg')
   wr(b'committer %s %d %s' % (user,time,timezone))
@@ -387,9 +352,8 @@ def export_note(ui,repo,revision,count,authors,encoding,is_first):
   if is_first:
     wr(b'from refs/notes/hg^0')
   wr(b'N inline :%d' % (revision+1))
-  hg_hash=revsymbol(repo,b"%d" % revision).hex()
+  hg_hash=ctx.hex()
-  wr(b'data %d' % (len(hg_hash)))
+  wr_data(hg_hash)
-  wr_no_nl(hg_hash)
   wr()
   return checkpoint(count)
 
@@ -434,9 +398,15 @@ def load_mapping(name, filename, mapping_is_raw):
   def process_unicode_escape_sequences(s):
     # Replace unicode escape sequences in the otherwise UTF8-encoded bytestring s with
     # the UTF8-encoded characters they represent. We need to do an additional
-    # .decode('utf8').encode('unicode-escape') to convert any non-ascii characters into
+    # .decode('utf8').encode('ascii', 'backslashreplace') to convert any non-ascii
-    # their escape sequences so that the subsequent .decode('unicode-escape') succeeds:
+    # characters into their escape sequences so that the subsequent
-    return s.decode('utf8').encode('unicode-escape').decode('unicode-escape').encode('utf8')
+    # .decode('unicode-escape') succeeds:
+    return (
+      s.decode('utf8')
+      .encode('ascii', 'backslashreplace')
+      .decode('unicode-escape')
+      .encode('utf8')
+    )
 
   def parse_quoted_line(line):
     m=quoted_regexp.match(line)
@@ -493,7 +463,12 @@ def verify_heads(ui,repo,cache,force,ignore_unnamed_heads,branchesmap):
     sanitized_name=sanitize_name(b,"branch",branchesmap)
     sha1=get_git_sha1(sanitized_name)
     c=cache.get(sanitized_name)
-    if sha1!=c:
+    if not c and sha1:
+      stderr_buffer.write(
+        b'Error: Branch [%s] already exists and was not created by hg-fast-export, '
+        b'export would overwrite unrelated branch\n' % b)
+      if not force: return False
+    elif sha1!=c:
       stderr_buffer.write(
         b'Error: Branch [%s] modified outside hg-fast-export:'
         b'\n%s (repo) != %s (cache)\n' % (b, b'<None>' if sha1 is None else sha1, c)
@@ -504,7 +479,7 @@ def verify_heads(ui,repo,cache,force,ignore_unnamed_heads,branchesmap):
   t={}
   unnamed_heads=False
   for h in repo.filtered(b'visible').heads():
-    (_,_,_,_,_,_,branch,_)=get_changeset(ui,repo,h)
+    branch=get_branch(repo[h].branch())
     if t.get(branch,False):
       stderr_buffer.write(
         b'Error: repository has an unnamed head: hg r%d\n'
@@ -547,21 +522,21 @@ def hg2git(repourl,m,marksfile,mappingfile,headsfile,tipfile,
   except AttributeError:
     tip=len(repo)
 
-  min=int(state_cache.get('tip',0))
+  min=int(state_cache.get(b'tip',0))
   max=_max
   if _max<0 or max>tip:
     max=tip
 
   for rev in range(0,max):
-    (revnode,_,_,_,_,_,_,_)=get_changeset(ui,repo,rev,authors)
+    ctx=repo[rev]
-    if repo[revnode].hidden():
+    if ctx.hidden():
       continue
-    mapping_cache[hexlify(revnode)] = b"%d" % rev
+    mapping_cache[ctx.hex()] = b"%d" % rev
 
   if submodule_mappings:
     # Make sure that all mercurial submodules are registered in the submodule-mappings file
     for rev in range(0,max):
-      ctx=revsymbol(repo,b"%d" % rev)
+      ctx=repo[rev]
       if ctx.hidden():
         continue
       if ctx.substate:
@@ -580,8 +555,8 @@ def hg2git(repourl,m,marksfile,mappingfile,headsfile,tipfile,
     for rev in range(min,max):
       c=export_note(ui,repo,rev,c,authors, encoding, rev == min and min != 0)
 
-  state_cache['tip']=max
+  state_cache[b'tip']=max
-  state_cache['repo']=repourl
+  state_cache[b'repo']=repourl
   save_cache(tipfile,state_cache)
   save_cache(mappingfile,mapping_cache)
 

hg2git.py

@@ -5,7 +5,7 @@
 
 from mercurial import hg,util,ui,templatefilters
 from mercurial import error as hgerror
-from mercurial.scmutil import revsymbol,binnode
+from mercurial.scmutil import binnode
 
 import re
 import os
@@ -81,22 +81,13 @@ def get_branch(name):
   return name
 
 def get_changeset(ui,repo,revision,authors={},encoding=''):
-  # Starting with Mercurial 4.6 lookup no longer accepts raw hashes
+  (manifest,user,(time,timezone),files,desc,extra)=repo.changelog.read(revision)
-  # for lookups. Work around it by changing our behaviour depending on
-  # how it fails
-  try:
-    node=repo.lookup(revision)
-  except (TypeError, hgerror.ProgrammingError):
-    node=binnode(revsymbol(repo, b"%d" % revision)) # We were given a numeric rev
-  except hgerror.RepoLookupError:
-    node=revision # We got a raw hash
-  (manifest,user,(time,timezone),files,desc,extra)=repo.changelog.read(node)
   if encoding:
     user=user.decode(encoding).encode('utf8')
     desc=desc.decode(encoding).encode('utf8')
   tz=b"%+03d%02d" % (-timezone // 3600, ((-timezone % 3600) // 60))
-  branch=get_branch(extra.get(b'branch', b'master'))
+  branch=get_branch(extra.get(b'branch', b''))
-  return (node,manifest,fixup_user(user,authors),(time,tz),files,desc,branch,extra)
+  return (manifest,fixup_user(user,authors),(time,tz),files,desc,branch,extra)
 
 def mangle_key(key):
   return key

run-tests

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+READLINK="readlink"
+if command -v greadlink > /dev/null; then
+  READLINK="greadlink" # Prefer greadlink over readlink
+fi
+
+if ! $READLINK -f "$(which "$0")" > /dev/null 2>&1 ; then
+    ROOT="$(dirname "$(which "$0")")"
+    if [ ! -f "$ROOT/hg-fast-export.py" ] ; then
+	echo "test runner requires a readlink implementation which knows" \
+	     " how to canonicalize paths in order to be called via a symlink."
+	exit 1
+    fi
+else
+    ROOT="$(dirname "$($READLINK -f "$(which "$0")")")"
+fi
+
+
+export SHARNESS_TEST_SRCDIR="${SHARNESS_TEST_SRCDIR:-$ROOT/t/sharness}"
+
+TESTS=$(find $ROOT/t -maxdepth 1 -name \*.t -executable -type f)
+
+failed=0
+type parallel >& /dev/null
+if [ $? -eq 0 ]; then
+    echo "Using parallel to run tests"
+    function F() {
+	echo "Running test $1"
+	$1
+    }
+    export -f F
+    parallel F ::: $TESTS || failed=1
+else
+    for i in $TESTS ; do
+	echo "Running test $i"
+	$i || failed=1
+    done
+fi
+
+if [ "$failed" -eq "0" ]; then
+    echo "All tests passed";
+else
+    echo "There were failed tests";
+fi
+
+exit $failed
+
+

t/.gitignore

@@ -0,0 +1 @@
+/test-results/

t/Makefile

@@ -0,0 +1,12 @@
+T = $(wildcard *.t)
+
+test: $(T)
+	@$(MAKE) --silent clean
+
+$(T): clean
+	./$@ $(TEST_OPTS)
+
+clean:
+	@rm -fr test-results
+
+.PHONY: test $(T) clean

t/main.t

@@ -0,0 +1,94 @@
+#!/bin/bash
+#
+# Copyright (c) 2023 Felipe Contreras
+#
+
+test_description='Main tests'
+
+. "${SHARNESS_TEST_SRCDIR-$(dirname "$0")/sharness}"/sharness.sh || exit 1
+
+check() {
+	echo "$3" > expected &&
+	git -C "$1" show -q --format='%s' "$2" > actual &&
+	test_cmp expected actual
+}
+
+git_clone() {
+	(
+	git init -q "$2" &&
+	cd "$2" &&
+	hg-fast-export.sh --repo "../$1"
+	)
+}
+
+setup() {
+	cat > "$HOME"/.hgrc <<-EOF
+	[ui]
+	username = H G Wells <wells@example.com>
+	EOF
+}
+
+setup
+
+test_expect_success 'basic' '
+	test_when_finished "rm -rf hgrepo gitrepo" &&
+
+	(
+	hg init hgrepo &&
+	cd hgrepo &&
+	echo zero > content &&
+	hg add content &&
+	hg commit -m zero
+	) &&
+
+	git_clone hgrepo gitrepo &&
+	check gitrepo @ zero
+'
+
+test_expect_success 'merge' '
+	test_when_finished "rm -rf hgrepo gitrepo" &&
+
+	(
+	hg init hgrepo &&
+	cd hgrepo &&
+	echo a > content &&
+	echo a > file1 &&
+	hg add content file1 &&
+	hg commit -m "origin" &&
+
+	echo b > content &&
+	echo b > file2 &&
+	hg add file2 &&
+	hg rm file1 &&
+	hg commit -m "right" &&
+
+	hg update -r0 &&
+	echo c > content &&
+	hg commit -m "left" &&
+
+	HGMERGE=true hg merge -r1 &&
+	hg commit -m "merge"
+	) &&
+
+	git_clone hgrepo gitrepo &&
+
+	cat > expected <<-EOF &&
+	left
+	c
+	tree @:
+
+	content
+	file2
+	EOF
+
+	(
+	cd gitrepo
+	git show -q --format='%s' @^ &&
+	git show @:content &&
+	git show @:
+	) > actual &&
+
+	test_cmp expected actual
+'
+
+test_done

t/smoke-test.branchmap

@@ -0,0 +1,15 @@
+"feature"="renamed-feature"
+"a?"="valid-0"
+"a/"="valid-1"
+"a/b"="valid-2"
+"a/?"="valid-3"
+"?a"="valid-4"
+"a."="valid-5"
+"a.b"="valid-6"
+".a"="valid-7"
+"/"="valid-8"
+"___3"="___a"
+"__2"="__b"
+"_1"="_c"
+"åäö"="abc"
+"Feature- 12V Vac \"Venom\""="venom"

t/smoke-test.expected

@@ -0,0 +1,280 @@
+blob
+mark :1
+data 5
+r0-a
+
+blob
+mark :2
+data 5
+r0-b
+
+reset refs/heads/master
+commit refs/heads/master
+mark :3
+author Grevious Bodily Harmsworth <gbh@example.com> 1679014800 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679014800 +0000
+data 2
+r0M 100644 :1 a.txt
+M 100644 :2 b.txt
+
+blob
+mark :4
+data 5
+r1-c
+
+blob
+mark :5
+data 5
+r1-d
+
+commit refs/tags/2019_Spring_R2
+mark :6
+author Grevious Bodily Harmsworth <gbh@example.com> 1679018400 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679018400 +0000
+data 2
+r1from :3
+M 100644 :4 c.txt
+M 100644 :5 d.txt
+
+blob
+mark :7
+data 56
+e92e41dde44f9dbbac08bbb83351a65b6728f128 2019 Spring R2
+
+commit refs/heads/mainline
+mark :8
+author Grevious Bodily Harmsworth <gbh@example.com> 1679019000 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679019000 +0000
+data 51
+Added tag 2019 Spring R2 for changeset e92e41dde44ffrom :6
+M 100644 :7 .hgtags
+
+blob
+mark :9
+data 5
+r2-e
+
+blob
+mark :10
+data 5
+r2-f
+
+commit refs/heads/mainline
+mark :11
+author Grevious Bodily Harmsworth <gbh@example.com> 1679022000 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679022000 +0000
+data 2
+r2from :8
+M 100644 :9 e.txt
+M 100644 :10 f.txt
+
+commit refs/heads/mainline
+mark :12
+author badly-formed-user <devnull@localhost> 1679025600 +0000
+committer badly-formed-user <devnull@localhost> 1679025600 +0000
+data 2
+r3from :11
+M 100644 :9 g.txt
+M 100644 :10 h.txt
+
+blob
+mark :13
+data 10
+feature-a
+
+blob
+mark :14
+data 10
+feature-b
+
+commit refs/heads/renamed-feature
+mark :15
+author Grevious Bodily Harmsworth <gbh@example.com> 1679029200 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679029200 +0000
+data 7
+featurefrom :12
+M 100644 :13 feature-a.txt
+M 100644 :14 feature-b.txt
+
+blob
+mark :16
+data 3
+a?
+
+commit refs/heads/valid-0
+mark :17
+author Grevious Bodily Harmsworth <gbh@example.com> 1679032800 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679032800 +0000
+data 23
+Added file in branch a?from :15
+M 100644 :16 c1086ce03e4f52aadd1c93b1d097da510138522a
+
+blob
+mark :18
+data 3
+a/
+
+commit refs/heads/valid-1
+mark :19
+author Grevious Bodily Harmsworth <gbh@example.com> 1679036400 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679036400 +0000
+data 23
+Added file in branch a/from :17
+M 100644 :18 85ed6fbb96d655df9f194bc9107f2d86210b9263
+
+blob
+mark :20
+data 4
+a/b
+
+commit refs/heads/valid-2
+mark :21
+author Grevious Bodily Harmsworth <gbh@example.com> 1679040000 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679040000 +0000
+data 24
+Added file in branch a/bfrom :19
+M 100644 :20 aae42d317509399fdda80c4d8e46774d152dbd04
+
+blob
+mark :22
+data 4
+a/?
+
+commit refs/heads/valid-3
+mark :23
+author Grevious Bodily Harmsworth <gbh@example.com> 1679043600 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679043600 +0000
+data 24
+Added file in branch a/?from :21
+M 100644 :22 ba54a8de7fe91c5e6e0a2dd1b9b37de0976ff5a7
+
+blob
+mark :24
+data 3
+?a
+
+commit refs/heads/valid-4
+mark :25
+author Grevious Bodily Harmsworth <gbh@example.com> 1679047200 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679047200 +0000
+data 23
+Added file in branch ?afrom :23
+M 100644 :24 d4cde16119b586025976741e87775762a2598984
+
+blob
+mark :26
+data 3
+a.
+
+commit refs/heads/valid-5
+mark :27
+author Grevious Bodily Harmsworth <gbh@example.com> 1679050800 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679050800 +0000
+data 23
+Added file in branch a.from :25
+M 100644 :26 b4ce96ddcee0706a8c51130917f910b2b29faf77
+
+blob
+mark :28
+data 4
+a.b
+
+commit refs/heads/valid-6
+mark :29
+author Grevious Bodily Harmsworth <gbh@example.com> 1679054400 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679054400 +0000
+data 24
+Added file in branch a.bfrom :27
+M 100644 :28 97051191e1a92daa11165ef10770bf964268c58b
+
+blob
+mark :30
+data 3
+.a
+
+commit refs/heads/valid-7
+mark :31
+author Grevious Bodily Harmsworth <gbh@example.com> 1679058000 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679058000 +0000
+data 23
+Added file in branch .afrom :29
+M 100644 :30 a667f8feec02fdfa6649772f844a24cf1ad5ebec
+
+blob
+mark :32
+data 2
+/
+
+commit refs/heads/valid-8
+mark :33
+author Grevious Bodily Harmsworth <gbh@example.com> 1679061600 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679061600 +0000
+data 22
+Added file in branch /from :31
+M 100644 :32 8f27084b6294ddbe28dbcbf98f798730e8a79289
+
+blob
+mark :34
+data 5
+___3
+
+commit refs/heads/___a
+mark :35
+author Grevious Bodily Harmsworth <gbh@example.com> 1679065200 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679065200 +0000
+data 25
+Added file in branch ___3from :33
+M 100644 :34 9b171494eb6e5ce325934b1656e286ca0510a697
+
+blob
+mark :36
+data 4
+__2
+
+commit refs/heads/__b
+mark :37
+author Grevious Bodily Harmsworth <gbh@example.com> 1679068800 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679068800 +0000
+data 24
+Added file in branch __2from :35
+M 100644 :36 5dca703b71d2613c6bb3262b9b1741d6165e4a2f
+
+blob
+mark :38
+data 3
+_1
+
+commit refs/heads/_c
+mark :39
+author Grevious Bodily Harmsworth <gbh@example.com> 1679072400 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679072400 +0000
+data 23
+Added file in branch _1from :37
+M 100644 :38 2fee90e148a2afbd911b67ced9b6240151f904ec
+
+blob
+mark :40
+data 25
+Feature- 12V Vac "Venom"
+
+commit refs/heads/venom
+mark :41
+author Grevious Bodily Harmsworth <gbh@example.com> 1679076000 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679076000 +0000
+data 45
+Added file in branch Feature- 12V Vac "Venom"from :39
+M 100644 :40 b01def8779aed4be2f4b7325a89992a9aa566fec
+
+blob
+mark :42
+data 7
+åäö
+
+commit refs/heads/abc
+mark :43
+author Grevious Bodily Harmsworth <gbh@example.com> 1679079600 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679079600 +0000
+data 27
+Added file in branch åäöfrom :41
+M 100644 :42 a0d01fcbff5d86327d542687dcfd8b299d054147
+

t/smoke-test.t

@@ -0,0 +1,162 @@
+#!/bin/bash
+#
+# Copyright (c) 2023 Felipe Contreras
+# Copyright (c) 2023 Frej Drejhammar
+#
+# Smoke test used to sanity test changes to fast-export.
+#
+
+test_description='Smoke test'
+
+. "${SHARNESS_TEST_SRCDIR-$(dirname "$0")/sharness}"/sharness.sh || exit 1
+
+check() {
+	echo "$3" > expected &&
+	git -C "$1" show -q --format='%s' "$2" > actual &&
+	test_cmp expected actual
+}
+
+git_create() {
+	git init -q "$1"
+}
+
+git_convert() {
+	(
+	cd "$2" &&
+	hg-fast-export.sh --repo "../$1" \
+			  -s --hgtags -n \
+			  -B "$SHARNESS_TEST_DIRECTORY"/smoke-test.branchmap \
+			  -T "$SHARNESS_TEST_DIRECTORY"/smoke-test.tagsmap
+	)
+}
+
+setup() {
+	cat > "$HOME"/.hgrc <<-EOF
+	[ui]
+	username = Grevious Bodily Harmsworth <gbh@example.com>
+	EOF
+}
+
+commit0() {
+	(
+	cd hgrepo &&
+	echo "r0-a" > a.txt &&
+	echo "r0-b" > b.txt &&
+	hg add a.txt b.txt &&
+	hg commit -d "2023-03-17 01:00Z" -m "r0" &&
+	hg bookmark bm0
+	)
+}
+
+commit1() {
+	(
+	cd hgrepo &&
+	echo "r1-c" > c.txt &&
+	echo "r1-d" > d.txt &&
+	hg branch mainline &&
+	hg add c.txt d.txt &&
+	hg commit -d "2023-03-17 02:00Z" -m "r1" &&
+	hg tag -d "2023-03-17 02:10Z" "2019 Spring R2"
+	)
+}
+
+commit2() {
+	(
+	cd hgrepo &&
+	echo "r2-e" > e.txt &&
+	echo "r2-f" > f.txt &&
+	hg add e.txt f.txt &&
+	hg commit -d "2023-03-17 03:00Z" -m "r2" &&
+	hg bookmark bm1
+	)
+}
+
+commit3() {
+	(
+	cd hgrepo &&
+	echo "r2-e" > g.txt &&
+	echo "r2-f" > h.txt &&
+	hg add g.txt h.txt &&
+	hg commit -d "2023-03-17 04:00Z" -u "badly-formed-user" -m "r3"
+	)
+}
+
+commit_rest() {
+	(
+	cd hgrepo &&
+
+	hg branch feature &&
+	echo "feature-a" > feature-a.txt &&
+	echo "feature-b" > feature-b.txt &&
+	hg add feature-a.txt feature-b.txt &&
+	hg commit -d "2023-03-17 05:00Z" -m "feature" &&
+	hg bookmark bm2 &&
+
+	# Now create strangely named branches
+	make-branch "a?" 06 &&
+	make-branch "a/" 07 &&
+	make-branch "a/b" 08 &&
+	make-branch "a/?" 09 &&
+	make-branch "?a" 10 &&
+	make-branch "a." 11 &&
+	make-branch "a.b" 12 &&
+	make-branch ".a" 13 &&
+	make-branch "/" 14 &&
+	make-branch "___3" 15 &&
+	make-branch "__2" 16 &&
+	make-branch "_1" 17 &&
+	make-branch "Feature- 12V Vac \"Venom\"" 18 &&
+	make-branch "åäö" 19 &&
+
+	hg bookmark bm-for-the-rest
+	)
+}
+
+make-branch() {
+    hg branch "$1"
+    FILE=$(echo "$1" | sha1sum | cut -d " " -f 1)
+    echo "$1" > $FILE
+    hg add $FILE
+    hg commit -d "2023-03-17 $2:00Z" -m "Added file in branch $1"
+}
+
+setup
+
+test_expect_success 'all in one' '
+	test_when_finished "rm -rf hgrepo gitrepo" &&
+
+	(
+	hg init hgrepo &&
+	commit0 &&
+	commit1 &&
+	commit2 &&
+	commit3 &&
+	commit_rest
+	) &&
+	git_create gitrepo &&
+	git_convert hgrepo gitrepo &&
+	git -C gitrepo fast-export --all > actual &&
+
+ 	test_cmp "$SHARNESS_TEST_DIRECTORY"/smoke-test.expected actual
+'
+
+test_expect_success 'incremental' '
+	test_when_finished "rm -rf hgrepo gitrepo" &&
+
+	hg init hgrepo &&
+	commit0 &&
+	git_create gitrepo &&
+	git_convert hgrepo gitrepo &&
+	commit1 &&
+	git_convert hgrepo gitrepo &&
+	commit2 &&
+	commit3 &&
+	git_convert hgrepo gitrepo &&
+	commit_rest &&
+	git_convert hgrepo gitrepo &&
+	git -C gitrepo fast-export --all > actual &&
+
+ 	test_cmp "$SHARNESS_TEST_DIRECTORY"/smoke-test.expected actual
+'
+
+test_done

t/smoke-test.tagsmap

@@ -0,0 +1 @@
+"2019 Spring R2"="2019_Spring_R2"