Merge branch 'gh/337'

Import mercurial large files as ordinary files into git

The basic idea to this fix is based on
https://github.com/planestraveler/fast-export/tree/add-lfs-support-v2
from PR #65

Closes #141

.github/requirements-earliest.txt

@@ -0,0 +1 @@
+mercurial==5.2

.github/requirements-latest.txt

@@ -0,0 +1,2 @@
+mercurial
+

.github/workflows/ci.yml

@@ -0,0 +1,71 @@
+name: CI
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [master]
+
+jobs:
+  test-earliest:
+    name: Run test suite on the earliest supported Python version
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v4
+        name: Checkout repository
+        with:
+          fetch-depth: 1
+          submodules: 'recursive'
+      - uses: actions/setup-python@v5
+        id: earliest
+        with:
+          python-version: '3.7.x'
+          check-latest: true
+          cache: 'pip'
+          cache-dependency-path: '**/requirements-earliest.txt'
+
+      - name: Install dependencies
+        run: |
+            python -m pip install --upgrade pip
+            pip install -r .github/requirements-earliest.txt
+
+      - name: Report selected versions
+        run: |
+          echo Selected '${{ steps.earliest.outputs.python-version }}'
+          ./hg-fast-export.sh --debug
+
+      - name: Run tests on earliest supported Python version
+        run: make -C t
+
+  test-latest:
+    name: Run test suite on the latest supported python version
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        name: Checkout repository
+        with:
+          fetch-depth: 1
+          submodules: 'recursive'
+      - uses: actions/setup-python@v5
+        id: latest
+        with:
+          python-version: '3.x'
+          check-latest: true
+          cache: 'pip'
+          cache-dependency-path: '**/requirements-latest.txt'
+
+      - name: Install dependencies
+        run: |
+            python -m pip install --upgrade pip
+            pip install -r .github/requirements-latest.txt
+
+      - name: Report selected version
+        run: |
+          echo Selected '${{ steps.latest.outputs.python-version }}'
+          ./hg-fast-export.sh --debug
+
+      - name: Run tests on 3.x
+        run: make -C t

.github/workflows/codeql-analysis.yml

@@ -1,71 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-name: "CodeQL"
-
-on:
-  push:
-    branches: [master]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [master]
-  schedule:
-    - cron: '0 15 * * 4'
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-      matrix:
-        # Override automatic language detection by changing the below list
-        # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
-        language: ['python']
-        # Learn more...
-        # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-      with:
-        # We must fetch at least the immediate parents so that if this is
-        # a pull request then we can checkout the head.
-        fetch-depth: 2
-
-    # If this run was triggered by a pull request event, then checkout
-    # the head of the pull request instead of the merge commit.
-    - run: git checkout HEAD^2
-      if: ${{ github.event_name == 'pull_request' }}
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file. 
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    #- run: |
-    #   make bootstrap
-    #   make release
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "t/sharness"]
+	path = t/sharness
+	url = https://github.com/felipec/sharness.git

README-SUBMODULES.md

@@ -27,10 +27,10 @@ command line option.
 
 ## Example
 
-Example mercurial repo folder structure (~/mercurial):
+Example mercurial repo folder structure (~/mercurial) containing two subrepos:
     src/...
-    subrepo/subrepo1
-    subrepo/subrepo2
+    subrepos/subrepo1
+    subrepos/subrepo2
 
 ### Setup
 Create an empty new folder where all the converted git modules will be imported:
@@ -41,18 +41,18 @@ Create an empty new folder where all the converted git modules will be imported:
     mkdir submodule1
     cd submodule1
     git init
-    hg-fast-export.sh -r ~/mercurial/subrepo1
+    hg-fast-export.sh -r ~/mercurial/subrepos/subrepo1
     cd ..
     mkdir submodule2
     cd submodule2
     git init
-    hg-fast-export.sh -r ~/mercurial/subrepo2
+    hg-fast-export.sh -r ~/mercurial/subrepos/subrepo2
 
 ### Create mapping file
     cd ~/imported-gits
     cat > submodule-mappings << EOF
-    "subrepo/subrepo1"="../submodule1"
-    "subrepo/subrepo2"="../submodule2"
+    "subrepos/subrepo1"="../submodule1"
+    "subrepos/subrepo2"="../submodule2"
     EOF
 
 ### Convert main repository
@@ -60,16 +60,16 @@ Create an empty new folder where all the converted git modules will be imported:
     mkdir git-main-repo
     cd git-main-repo
     git init
-    hg-fast-export.sh -r ~/mercurial --subrepo-map=../submodule-mappings
+    hg-fast-export.sh -r ~/mercurial --subrepo-map=~/imported-gits/submodule-mappings
 
 ### Result
-The resulting repository will now contain the subrepo/subrepo1 and
-subrepo/subrepo1 submodules. The created .gitmodules file will look
-like:
+The resulting repository will now contain the submodules at the paths
+`subrepos/subrepo1` and `subrepos/subrepo2`. The created .gitmodules
+file will look like:
 
-    [submodule "subrepo/subrepo1"]
-          path = subrepo/subrepo1
+    [submodule "subrepos/subrepo1"]
+          path = subrepos/subrepo1
           url = ../submodule1
-    [submodule "subrepo/subrepo2"]
-          path = subrepo/subrepo2
+    [submodule "subrepos/subrepo2"]
+          path = subrepos/subrepo2
           url = ../submodule2

README.md

@@ -29,10 +29,9 @@ first time.
 System Requirements
 -------------------
 
-This project depends on Python 2.7 or 3.5+, and the Mercurial >= 4.6
-package (>= 5.2, if Python 3.5+). If Python is not installed, install
-it before proceeding. The Mercurial package can be installed with `pip
-install mercurial`.
+This project depends on Python (>=3.7) and the Mercurial package (>=
+5.2).  If Python is not installed, install it before proceeding. The
+Mercurial package can be installed with `pip install mercurial`.
 
 On windows the bash that comes with "Git for Windows" is known to work
 well.
@@ -43,11 +42,10 @@ Usage
 Using hg-fast-export is quite simple for a mercurial repository <repo>:
 
 ```
-mkdir repo-git # or whatever
+git init repo-git # or whatever
 cd repo-git
-git init
 hg-fast-export.sh -r <local-repo>
-git checkout HEAD
+git checkout
 ```
 
 Please note that hg-fast-export does not automatically check out the
@@ -111,8 +109,8 @@ branch/tag names. In the future -n will become the default, but in
 order to not break existing incremental conversions, the default
 remains with the old behavior.
 
-By default, the `default` mercurial branch is renamed to the `master` 
-branch on git. If your mercurial repo contains both `default` and 
+By default, the `default` mercurial branch is renamed to the `master`
+branch on git. If your mercurial repo contains both `default` and
 `master` branches, you'll need to override this behavior. Use
 `-M <newName>` to specify what name to give the `default` branch.
 
@@ -133,10 +131,22 @@ is to convert line endings in text files from CRLF to git's preferred LF:
 # $2 = Mercurial's hash of the file
 # $3 = "1" if Mercurial reports the file as binary, otherwise "0"
 
-if [ "$3" == "1" ]; then cat; else dos2unix; fi
+if [ "$3" == "1" ]; then cat; else dos2unix -q; fi
+# -q option in call to dos2unix allows to avoid returning an
+# error code when handling non-ascii based text files (like UTF-16
+# encoded text files)
 -- End of crlf-filter.sh --
 ```
 
+Mercurial Largefiles Extension
+------------------------------
+
+Mercurial largefiles are exported as ordinary files into git, i.e. not
+as git lfs files. In order to make the export work, make sure that 
+you have all largefiles of all mercurial commits available locally.
+This can be ensured by either cloning the mercurial repository with 
+the option --all-largefiles or by executing the command
+'hg lfpull --rev "all()"' inside the mercurial repository.
 
 Plugins
 -----------------
@@ -178,7 +188,7 @@ values in the dictionary after filters have been run are used to create the git
 commit.
 
 ```
-file_data = {'filename':filename,'file_ctx':file_ctx,'d':d}
+file_data = {'filename':filename,'file_ctx':file_ctx,'data':file_contents}
 
 def file_data_filter(self,file_data):
 ```
@@ -188,6 +198,11 @@ can be modified by any filter. `file_ctx` is the filecontext from the
 mercurial python library.  After all filters have been run, the values
 are used to add the file to the git commit.
 
+The `file_data_filter` method is also called when files are deleted,
+but in this case the `data` and `file_ctx` keys map to None. This is
+so that a filter which modifies file names can apply the same name
+transformations when files are deleted.
+
 Submodules
 ----------
 See README-SUBMODULES.md for how to convert subrepositories into git
@@ -204,7 +219,7 @@ few options to deal with this:
    duplicate heads.
 2. Use the [head2branch plugin](./plugins/head2branch) to create a new named
    branch from an unnamed head.
-3. You can ignore unnamed heads with the `-ignore-unnamed-heads` option, which
+3. You can ignore unnamed heads with the `--ignore-unnamed-heads` option, which
    is appropriate in situations such as the extra heads being close commits
    (abandoned, unmerged changes).
 

hg-fast-export.py

@@ -1,10 +1,9 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # Copyright (c) 2007, 2008 Rocco Rutte <pdmef@gmx.net> and others.
+# Copyright (c) 2025 Siemens
 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
 
-from mercurial import node
-from mercurial.scmutil import revsymbol
 from hg2git import setup_repo,fixup_user,get_branch,get_changeset
 from hg2git import load_cache,save_cache,get_git_sha1,set_default_branch,set_origin_name
 from optparse import OptionParser
@@ -13,17 +12,7 @@ import sys
 import os
 from binascii import hexlify
 import pluginloader
-PY2 = sys.version_info.major == 2
-if PY2:
-  str = unicode
-
-if PY2 and sys.platform == "win32":
-  # On Windows, sys.stdout is initially opened in text mode, which means that
-  # when a LF (\n) character is written to sys.stdout, it will be converted
-  # into CRLF (\r\n).  That makes git blow up, so use this platform-specific
-  # code to change the mode of sys.stdout to binary.
-  import msvcrt
-  msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
+from hgext.largefiles import lfutil
 
 # silly regex to catch Signed-off-by lines in log message
 sob_re=re.compile(b'^Signed-[Oo]ff-[Bb]y: (.+)$')
@@ -39,26 +28,26 @@ submodule_mappings=None
 # author/branch/tag names.
 auto_sanitize = None
 
-stdout_buffer = sys.stdout if PY2 else sys.stdout.buffer
-stderr_buffer = sys.stderr if PY2 else sys.stderr.buffer
-
 def gitmode(flags):
   return b'l' in flags and b'120000' or b'x' in flags and b'100755' or b'100644'
 
 def wr_no_nl(msg=b''):
   assert isinstance(msg, bytes)
   if msg:
-    stdout_buffer.write(msg)
+    sys.stdout.buffer.write(msg)
 
 def wr(msg=b''):
-  wr_no_nl(msg)
-  stdout_buffer.write(b'\n')
+  wr_no_nl(msg + b'\n')
   #map(lambda x: sys.stderr.write('\t[%s]\n' % x),msg.split('\n'))
 
+def wr_data(data):
+  wr(b'data %d' % (len(data)))
+  wr(data)
+
 def checkpoint(count):
   count=count+1
   if cfg_checkpoint_count>0 and count%cfg_checkpoint_count==0:
-    stderr_buffer.write(b"Checkpoint after %d commits\n" % count)
+    sys.stderr.buffer.write(b"Checkpoint after %d commits\n" % count)
     wr(b'checkpoint')
     wr()
   return count
@@ -68,39 +57,15 @@ def revnum_to_revref(rev, old_marks):
   or a mark)"""
   return old_marks.get(rev) or b':%d' % (rev+1)
 
-def file_mismatch(f1,f2):
-  """See if two revisions of a file are not equal."""
-  return node.hex(f1)!=node.hex(f2)
-
-def split_dict(dleft,dright,l=[],c=[],r=[],match=file_mismatch):
-  """Loop over our repository and find all changed and missing files."""
-  for left in dleft.keys():
-    right=dright.get(left,None)
-    if right==None:
-      # we have the file but our parent hasn't: add to left set
-      l.append(left)
-    elif match(dleft[left],right) or gitmode(dleft.flags(left))!=gitmode(dright.flags(left)):
-      # we have it but checksums mismatch: add to center set
-      c.append(left)
-  for right in dright.keys():
-    left=dleft.get(right,None)
-    if left==None:
-      # if parent has file but we don't: add to right set
-      r.append(right)
-    # change is already handled when comparing child against parent
-  return l,c,r
-
-def get_filechanges(repo,revision,parents,mleft):
+def get_filechanges(repo,revision,parents,files):
   """Given some repository and revision, find all changed/deleted files."""
-  l,c,r=[],[],[]
-  for p in parents:
-    if p<0: continue
-    mright=revsymbol(repo,b"%d" %p).manifest()
-    l,c,r=split_dict(mleft,mright,l,c,r)
-  l.sort()
-  c.sort()
-  r.sort()
-  return l,c,r
+  if not parents:
+    # first revision: feed in full manifest
+    return files,[]
+  else:
+    # take the changes from the first parent
+    f=repo.status(parents[0],revision)
+    return f.modified+f.added,f.removed
 
 def get_author(logmessage,committer,authors):
   """As git distincts between author and committer of a patch, try to
@@ -151,7 +116,7 @@ def remove_gitmodules(ctx):
 
 def refresh_git_submodule(name,subrepo_info):
   wr(b'M 160000 %s %s' % (subrepo_info[1],name))
-  stderr_buffer.write(
+  sys.stderr.buffer.write(
     b"Adding/updating submodule %s, revision %s\n" % (name, subrepo_info[1])
   )
   return b'[submodule "%s"]\n\tpath = %s\n\turl = %s\n' % (name, name, subrepo_info[0])
@@ -171,14 +136,14 @@ def refresh_hg_submodule(name,subrepo_info):
     revnum=mapping_cache[subrepo_hash]
     gitSha=marks_cache[int(revnum)]
     wr(b'M 160000 %s %s' % (gitSha,name))
-    stderr_buffer.write(
+    sys.stderr.buffer.write(
       b"Adding/updating submodule %s, revision %s->%s\n"
       % (name, subrepo_hash, gitSha)
     )
     return b'[submodule "%s"]\n\tpath = %s\n\turl = %s\n' % (name,name,
       submodule_mappings[name])
   else:
-    stderr_buffer.write(
+    sys.stderr.buffer.write(
       b"Warning: Could not find hg revision %s for %s in git %s\n"
       % (subrepo_hash, name, gitRepoLocation,)
     )
@@ -197,8 +162,33 @@ def refresh_gitmodules(ctx):
 
   if len(gitmodules):
     wr(b'M 100644 inline .gitmodules')
-    wr(b'data %d' % (len(gitmodules)+1))
-    wr(gitmodules)
+    wr_data(gitmodules)
+
+def is_largefile(filename):
+  return filename[:6] == b'.hglf/'
+
+def largefile_orig_name(filename):
+  return filename[6:]
+
+def largefile_data(ctx, file, filename):
+  lf_file_ctx=ctx.filectx(file)
+  lf_hash=lf_file_ctx.data().strip(b'\n')
+  sys.stderr.write("Detected large file hash %s\n" % lf_hash.decode())
+  #should detect where the large files are located
+  file_with_data = lfutil.findfile(ctx.repo(), lf_hash)
+  if file_with_data is None:
+    # Autodownloading from the mercurial repository would be an issue as there
+    # is a good chance that we may need to input some username and password.
+    # This will surely break fast-export as there will be some unexpected
+    # output.
+    sys.stderr.write("Large file wasn't found in local cache.\n")
+    sys.stderr.write("Please clone with --all-largefiles\n")
+    sys.stderr.write("or pull all large files with 'hg lfpull --rev "
+            "\"all()\"'\n")
+    # closing in the middle of import will revert everything to the last checkpoint
+    sys.exit(3)
+  with open(os.path.normpath(file_with_data), 'rb') as file_with_data_handle:
+    return file_with_data_handle.read()
 
 def export_file_contents(ctx,manifest,files,hgtags,encoding='',plugins={}):
   count=0
@@ -210,19 +200,23 @@ def export_file_contents(ctx,manifest,files,hgtags,encoding='',plugins={}):
       refresh_gitmodules(ctx)
     # Skip .hgtags files. They only get us in trouble.
     if not hgtags and file == b".hgtags":
-      stderr_buffer.write(b'Skip %s\n' % file)
+      sys.stderr.buffer.write(b'Skip %s\n' % file)
       continue
     if encoding:
       filename=file.decode(encoding).encode('utf8')
     else:
       filename=file
     if b'.git' in filename.split(b'/'): # Even on Windows, the path separator is / here.
-      stderr_buffer.write(
+      sys.stderr.buffer.write(
         b'Ignoring file %s which cannot be tracked by git\n' % filename
       )
       continue
-    file_ctx=ctx.filectx(file)
-    d=file_ctx.data()
+    if is_largefile(filename):
+      filename = largefile_orig_name(filename)
+      d = largefile_data(ctx, file, filename)
+    else:
+      file_ctx=ctx.filectx(file)
+      d=file_ctx.data()
 
     if plugins and plugins['file_data_filters']:
       file_data = {'filename':filename,'file_ctx':file_ctx,'data':d}
@@ -232,15 +226,16 @@ def export_file_contents(ctx,manifest,files,hgtags,encoding='',plugins={}):
       filename=file_data['filename']
       file_ctx=file_data['file_ctx']
 
-    wr(b'M %s inline %s' % (gitmode(manifest.flags(file)),
-                           strip_leading_slash(filename)))
-    wr(b'data %d' % len(d)) # had some trouble with size()
-    wr(d)
-    count+=1
-    if count%cfg_export_boundary==0:
-      stderr_buffer.write(b'Exported %d/%d files\n' % (count,max))
+    if d is not None:
+      wr(b'M %s inline %s' % (gitmode(manifest.flags(file)),
+                             strip_leading_slash(filename)))
+      wr(b'data %d' % len(d)) # had some trouble with size()
+      wr(d)
+      count+=1
+      if count%cfg_export_boundary==0:
+        sys.stderr.buffer.write(b'Exported %d/%d files\n' % (count,max))
   if max>cfg_export_boundary:
-    stderr_buffer.write(b'Exported %d/%d files\n' % (count,max))
+    sys.stderr.buffer.write(b'Exported %d/%d files\n' % (count,max))
 
 def sanitize_name(name,what="branch", mapping={}):
   """Sanitize input roughly according to git-check-ref-format(1)"""
@@ -266,7 +261,7 @@ def sanitize_name(name,what="branch", mapping={}):
   if not auto_sanitize:
     return mapping.get(name,name)
   n=mapping.get(name,name)
-  p=re.compile(b'([[ ~^:?\\\\*]|\.\.)')
+  p=re.compile(b'([\\[ ~^:?\\\\*]|\\.\\.)')
   n=p.sub(b'_', n)
   if n[-1:] in (b'/', b'.'): n=n[:-1]+b'_'
   n=b'/'.join([dot(s) for s in n.split(b'/')])
@@ -274,7 +269,7 @@ def sanitize_name(name,what="branch", mapping={}):
   n=p.sub(b'_', n)
 
   if n!=name:
-    stderr_buffer.write(
+    sys.stderr.buffer.write(
       b'Warning: sanitized %s [%s] to [%s]\n' % (what.encode(), name, n)
     )
   return n
@@ -294,15 +289,18 @@ def export_commit(ui,repo,revision,old_marks,max,count,authors,
     brmap[name]=n
     return n
 
-  (revnode,_,user,(time,timezone),files,desc,branch,extra)=get_changeset(ui,repo,revision,authors,encoding)
-  if repo[revnode].hidden():
+  ctx=repo[revision]
+
+  if ctx.hidden():
     return count
 
+  (_,user,(time,timezone),files,desc,branch,extra)=get_changeset(ui,repo,revision,authors,encoding)
+
   branch=get_branchname(branch)
 
   parents = [p for p in repo.changelog.parentrevs(revision) if p >= 0]
   author = get_author(desc,user,authors)
-  hg_hash=revsymbol(repo,b"%d" % revision).hex()
+  hg_hash=ctx.hex()
 
   if plugins and plugins['commit_message_filters']:
     commit_data = {'branch': branch, 'parents': parents,
@@ -325,61 +323,58 @@ def export_commit(ui,repo,revision,old_marks,max,count,authors,
   if sob:
     wr(b'author %s %d %s' % (author,time,timezone))
   wr(b'committer %s %d %s' % (user,time,timezone))
-  wr(b'data %d' % (len(desc)+1)) # wtf?
-  wr(desc)
-  wr()
+  wr_data(desc + b'\n')
 
-  ctx=revsymbol(repo, b"%d" % revision)
   man=ctx.manifest()
-  added,changed,removed,type=[],[],[],''
 
-  if len(parents) == 0:
-    # first revision: feed in full manifest
-    added=man.keys()
-    added.sort()
+  if not parents:
     type='full'
   else:
     wr(b'from %s' % revnum_to_revref(parents[0], old_marks))
     if len(parents) == 1:
-      # later non-merge revision: feed in changed manifest
-      # if we have exactly one parent, just take the changes from the
-      # manifest without expensively comparing checksums
-      f=repo.status(parents[0],revnode)
-      added,changed,removed=f.added,f.modified,f.removed
       type='simple delta'
     else: # a merge with two parents
       wr(b'merge %s' % revnum_to_revref(parents[1], old_marks))
-      # later merge revision: feed in changed manifest
-      # for many files comparing checksums is expensive so only do it for
-      # merges where we really need it due to hg's revlog logic
-      added,changed,removed=get_filechanges(repo,revision,parents,man)
       type='thorough delta'
 
-  stderr_buffer.write(
-    b'%s: Exporting %s revision %d/%d with %d/%d/%d added/changed/removed files\n'
-    % (branch, type.encode(), revision + 1, max, len(added), len(changed), len(removed))
+  modified,removed=get_filechanges(repo,revision,parents,files)
+
+  sys.stderr.buffer.write(
+    b'%s: Exporting %s revision %d/%d with %d/%d modified/removed files\n'
+    % (branch, type.encode(), revision + 1, max, len(modified), len(removed))
   )
 
-  for filename in removed:
+  for file in removed:
     if fn_encoding:
-      filename=filename.decode(fn_encoding).encode('utf8')
+      filename=file.decode(fn_encoding).encode('utf8')
+    else:
+      filename=file
+
+    if plugins and plugins['file_data_filters']:
+      file_data = {'filename':filename, 'file_ctx':None, 'data':None}
+      for filter in plugins['file_data_filters']:
+        filter(file_data)
+      filename=file_data['filename']
+
     filename=strip_leading_slash(filename)
     if filename==b'.hgsub':
       remove_gitmodules(ctx)
+    if is_largefile(filename):
+      filename=largefile_orig_name(filename)
     wr(b'D %s' % filename)
 
-  export_file_contents(ctx,man,added,hgtags,fn_encoding,plugins)
-  export_file_contents(ctx,man,changed,hgtags,fn_encoding,plugins)
+  export_file_contents(ctx,man,modified,hgtags,fn_encoding,plugins)
   wr()
 
   return checkpoint(count)
 
 def export_note(ui,repo,revision,count,authors,encoding,is_first):
-  (revnode,_,user,(time,timezone),_,_,_,_)=get_changeset(ui,repo,revision,authors,encoding)
-  if repo[revnode].hidden():
+  ctx = repo[revision]
+
+  if ctx.hidden():
     return count
 
-  parents = [p for p in repo.changelog.parentrevs(revision) if p >= 0]
+  (_,user,(time,timezone),_,_,_,_)=get_changeset(ui,repo,revision,authors,encoding)
 
   wr(b'commit refs/notes/hg')
   wr(b'committer %s %d %s' % (user,time,timezone))
@@ -387,9 +382,8 @@ def export_note(ui,repo,revision,count,authors,encoding,is_first):
   if is_first:
     wr(b'from refs/notes/hg^0')
   wr(b'N inline :%d' % (revision+1))
-  hg_hash=revsymbol(repo,b"%d" % revision).hex()
-  wr(b'data %d' % (len(hg_hash)))
-  wr_no_nl(hg_hash)
+  hg_hash=ctx.hex()
+  wr_data(hg_hash)
   wr()
   return checkpoint(count)
 
@@ -402,18 +396,18 @@ def export_tags(ui,repo,old_marks,mapping_cache,count,authors,tagsmap):
     if tag==b'tip': continue
     # ignore tags to nodes that are missing (ie, 'in the future')
     if hexlify(node) not in mapping_cache:
-      stderr_buffer.write(b'Tag %s refers to unseen node %s\n' % (tag, hexlify(node)))
+      sys.stderr.buffer.write(b'Tag %s refers to unseen node %s\n' % (tag, hexlify(node)))
       continue
 
     rev=int(mapping_cache[hexlify(node)])
 
     ref=revnum_to_revref(rev, old_marks)
     if ref==None:
-      stderr_buffer.write(
+      sys.stderr.buffer.write(
         b'Failed to find reference for creating tag %s at r%d\n' % (tag, rev)
       )
       continue
-    stderr_buffer.write(b'Exporting tag [%s] at [hg r%d] [git %s]\n' % (tag, rev, ref))
+    sys.stderr.buffer.write(b'Exporting tag [%s] at [hg r%d] [git %s]\n' % (tag, rev, ref))
     wr(b'reset refs/tags/%s' % tag)
     wr(b'from %s' % ref)
     wr()
@@ -434,15 +428,21 @@ def load_mapping(name, filename, mapping_is_raw):
   def process_unicode_escape_sequences(s):
     # Replace unicode escape sequences in the otherwise UTF8-encoded bytestring s with
     # the UTF8-encoded characters they represent. We need to do an additional
-    # .decode('utf8').encode('unicode-escape') to convert any non-ascii characters into
-    # their escape sequences so that the subsequent .decode('unicode-escape') succeeds:
-    return s.decode('utf8').encode('unicode-escape').decode('unicode-escape').encode('utf8')
+    # .decode('utf8').encode('ascii', 'backslashreplace') to convert any non-ascii
+    # characters into their escape sequences so that the subsequent
+    # .decode('unicode-escape') succeeds:
+    return (
+      s.decode('utf8')
+      .encode('ascii', 'backslashreplace')
+      .decode('unicode-escape')
+      .encode('utf8')
+    )
 
   def parse_quoted_line(line):
     m=quoted_regexp.match(line)
     if m==None:
-      return 
-    
+      return
+
     return (process_unicode_escape_sequences(m.group(1)),
             process_unicode_escape_sequences(m.group(5)))
 
@@ -494,12 +494,12 @@ def verify_heads(ui,repo,cache,force,ignore_unnamed_heads,branchesmap):
     sha1=get_git_sha1(sanitized_name)
     c=cache.get(sanitized_name)
     if not c and sha1:
-      stderr_buffer.write(
+      sys.stderr.buffer.write(
         b'Error: Branch [%s] already exists and was not created by hg-fast-export, '
         b'export would overwrite unrelated branch\n' % b)
       if not force: return False
     elif sha1!=c:
-      stderr_buffer.write(
+      sys.stderr.buffer.write(
         b'Error: Branch [%s] modified outside hg-fast-export:'
         b'\n%s (repo) != %s (cache)\n' % (b, b'<None>' if sha1 is None else sha1, c)
       )
@@ -509,9 +509,9 @@ def verify_heads(ui,repo,cache,force,ignore_unnamed_heads,branchesmap):
   t={}
   unnamed_heads=False
   for h in repo.filtered(b'visible').heads():
-    (_,_,_,_,_,_,branch,_)=get_changeset(ui,repo,h)
+    branch=get_branch(repo[h].branch())
     if t.get(branch,False):
-      stderr_buffer.write(
+      sys.stderr.buffer.write(
         b'Error: repository has an unnamed head: hg r%d\n'
         % repo.changelog.rev(h)
       )
@@ -558,15 +558,15 @@ def hg2git(repourl,m,marksfile,mappingfile,headsfile,tipfile,
     max=tip
 
   for rev in range(0,max):
-    (revnode,_,_,_,_,_,_,_)=get_changeset(ui,repo,rev,authors)
-    if repo[revnode].hidden():
+    ctx=repo[rev]
+    if ctx.hidden():
       continue
-    mapping_cache[hexlify(revnode)] = b"%d" % rev
+    mapping_cache[ctx.hex()] = b"%d" % rev
 
   if submodule_mappings:
     # Make sure that all mercurial submodules are registered in the submodule-mappings file
     for rev in range(0,max):
-      ctx=revsymbol(repo,b"%d" % rev)
+      ctx=repo[rev]
       if ctx.hidden():
         continue
       if ctx.substate:

hg-fast-export.sh

@@ -29,7 +29,7 @@ GFI_OPTS=""
 
 if [ -z "${PYTHON}" ]; then
     # $PYTHON is not set, so we try to find a working python with mercurial:
-    for python_cmd in python2 python python3; do
+    for python_cmd in python3 python; do
         if command -v $python_cmd > /dev/null; then
             $python_cmd -c 'from mercurial.scmutil import revsymbol' 2> /dev/null
             if [ $? -eq 0 ]; then
@@ -45,6 +45,14 @@ if [ -z "${PYTHON}" ]; then
     exit 1
 fi
 
+"${PYTHON}" -c 'import sys; exit(sys.version_info.major==3 and sys.version_info.minor >= 7)'
+
+if [ $? -eq 0 ]; then
+    echo "Could not find an interpreter for a supported Python version (>= 3.7)" \
+        "Please use the 'PYTHON' environment variable to specify the interpreter to use."
+    exit 1
+fi
+
 USAGE="[--quiet] [-r <repo>] [--force] [--ignore-unnamed-heads] [-m <max>] [-s] [--hgtags] [-A <file>] [-B <file>] [-T <file>] [-M <name>] [-o <name>] [--hg-hash] [-e <encoding>]"
 LONG_USAGE="Import hg repository <repo> up to either tip or <max>
 If <repo> is omitted, use last hg repository as obtained from state file,
@@ -86,6 +94,14 @@ case "$1" in
       echo ""
       echo "$LONG_USAGE"
       exit 0
+      ;;
+
+    --debug)
+      echo -n "Using Python: "
+      "${PYTHON}" --version
+      echo -n "Using Mercurial: "
+      hg --version
+      exit 0
 esac
 
 IS_BARE=$(git rev-parse --is-bare-repository) \

hg-reset.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright (c) 2007, 2008 Rocco Rutte <pdmef@gmx.net> and others.
 # License: GPLv2

hg2git.py

@@ -1,24 +1,17 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # Copyright (c) 2007, 2008 Rocco Rutte <pdmef@gmx.net> and others.
 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
 
 from mercurial import hg,util,ui,templatefilters
 from mercurial import error as hgerror
-from mercurial.scmutil import revsymbol,binnode
+from mercurial.scmutil import binnode
 
 import re
 import os
 import sys
 import subprocess
 
-PY2 = sys.version_info.major < 3
-if PY2:
-  str = unicode
-  fsencode = lambda s: s.encode(sys.getfilesystemencoding())
-else:
-  from os import fsencode
-
 # default git branch name
 cfg_master=b'master'
 # default origin name
@@ -44,7 +37,7 @@ def setup_repo(url):
     myui.setconfig(b'ui', b'interactive', b'off')
     # Avoids a warning when the repository has obsolete markers
     myui.setconfig(b'experimental', b'evolution.createmarkers', True)
-  return myui,hg.repository(myui, fsencode(url)).unfiltered()
+  return myui,hg.repository(myui, os.fsencode(url)).unfiltered()
 
 def fixup_user(user,authors):
   user=user.strip(b"\"")
@@ -81,22 +74,13 @@ def get_branch(name):
   return name
 
 def get_changeset(ui,repo,revision,authors={},encoding=''):
-  # Starting with Mercurial 4.6 lookup no longer accepts raw hashes
-  # for lookups. Work around it by changing our behaviour depending on
-  # how it fails
-  try:
-    node=repo.lookup(revision)
-  except (TypeError, hgerror.ProgrammingError):
-    node=binnode(revsymbol(repo, b"%d" % revision)) # We were given a numeric rev
-  except hgerror.RepoLookupError:
-    node=revision # We got a raw hash
-  (manifest,user,(time,timezone),files,desc,extra)=repo.changelog.read(node)
+  (manifest,user,(time,timezone),files,desc,extra)=repo.changelog.read(revision)
   if encoding:
     user=user.decode(encoding).encode('utf8')
     desc=desc.decode(encoding).encode('utf8')
   tz=b"%+03d%02d" % (-timezone // 3600, ((-timezone % 3600) // 60))
-  branch=get_branch(extra.get(b'branch', b'master'))
-  return (node,manifest,fixup_user(user,authors),(time,tz),files,desc,branch,extra)
+  branch=get_branch(extra.get(b'branch', b''))
+  return (manifest,fixup_user(user,authors),(time,tz),files,desc,branch,extra)
 
 def mangle_key(key):
   return key

pluginloader/__init__.py

@@ -1,19 +1,23 @@
 import os
-import imp
+import importlib.machinery
+import importlib.util
 PluginFolder = os.path.join(os.path.dirname(os.path.realpath(__file__)),"..","plugins")
 MainModule = "__init__"
 
 def get_plugin(name, plugin_path):
-    search_dirs = [PluginFolder]
+    search_dirs = [PluginFolder, '.']
     if plugin_path:
         search_dirs = [plugin_path] + search_dirs
     for dir in search_dirs:
         location = os.path.join(dir, name)
         if not os.path.isdir(location) or not MainModule + ".py" in os.listdir(location):
             continue
-        info = imp.find_module(MainModule, [location])
-        return {"name": name, "info": info, "path": location}
+        spec = importlib.machinery.PathFinder.find_spec(MainModule, [location])
+        return {"name": name, "spec": spec, "path": location}
     raise Exception("Could not find plugin with name " + name)
 
 def load_plugin(plugin):
-    return imp.load_module(MainModule, *plugin["info"])
+    spec = plugin["spec"]
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module

plugins/dos2unix/__init__.py

@@ -6,6 +6,8 @@ class Filter():
         pass
 
     def file_data_filter(self,file_data):
+        if file_data['file_ctx'] == None:
+            return
         file_ctx = file_data['file_ctx']
         if not file_ctx.isbinary():
             file_data['data'] = file_data['data'].replace(b'\r\n', b'\n')

plugins/shell_filter_file_contents/__init__.py

@@ -15,6 +15,8 @@ class Filter:
         d = file_data['data']
         file_ctx = file_data['file_ctx']
         filename = file_data['filename']
+        if file_ctx == None:
+            return
         filter_cmd = self.filter_contents + [filename, node.hex(file_ctx.filenode()), '1' if file_ctx.isbinary() else '0']
         try:
             filter_proc = subprocess.Popen(filter_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)

t/.gitignore

@@ -0,0 +1 @@
+/test-results/

t/Makefile

@@ -0,0 +1,12 @@
+T = $(wildcard *.t)
+
+test: $(T)
+	@$(MAKE) --silent clean
+
+$(T): clean
+	./$@ $(TEST_OPTS)
+
+clean:
+	@rm -fr test-results
+
+.PHONY: test $(T) clean

t/file_data_filter-removefiles.expected

@@ -0,0 +1,30 @@
+blob
+mark :1
+data 7
+good_a
+
+reset refs/heads/master
+commit refs/heads/master
+mark :2
+author Grevious Bodily Harmsworth <gbh@example.com> 1679014800 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679014800 +0000
+data 3
+r0
+M 100644 :1 good_a.txt
+
+commit refs/heads/master
+mark :3
+author Grevious Bodily Harmsworth <gbh@example.com> 1679018400 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679018400 +0000
+data 3
+r1
+from :2
+
+commit refs/heads/master
+mark :4
+author Grevious Bodily Harmsworth <gbh@example.com> 1679022000 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679022000 +0000
+data 3
+r2
+from :3
+

t/file_data_filter-removefiles.t

@@ -0,0 +1,91 @@
+#!/bin/bash
+#
+# Copyright (c) 2023 Felipe Contreras
+# Copyright (c) 2023 Frej Drejhammar
+# Copyright (c) 2024 Stephan Hohe
+#
+# Check that files that file_data_filter sets to None are removed from repository
+#
+
+test_description='Remove files from file_data_filter plugin test'
+
+. "${SHARNESS_TEST_SRCDIR-$(dirname "$0")/sharness}"/sharness.sh || exit 1
+
+check() {
+	echo "$3" > expected &&
+	git -C "$1" show -q --format='%s' "$2" > actual &&
+	test_cmp expected actual
+}
+
+git_create() {
+	git init -q "$1" &&
+	git -C "$1" config core.ignoreCase false
+}
+
+git_convert() {
+	(
+	cd "$2" &&
+	hg-fast-export.sh --repo "../$1" \
+			  -s --hgtags -n \
+			  --plugin ../../plugins/removefiles_test_plugin
+	)
+}
+
+setup() {
+	cat > "$HOME"/.hgrc <<-EOF
+	[ui]
+	username = Grevious Bodily Harmsworth <gbh@example.com>
+	EOF
+}
+
+commit0() {
+	(
+	# Test inital revision with suppressed file
+	cd hgrepo &&
+	echo "good_a" > good_a.txt &&
+	echo "bad_a" > bad_a.txt &&
+	hg add good_a.txt bad_a.txt &&
+	hg commit -d "2023-03-17 01:00Z" -m "r0"
+	)
+}
+
+commit1() {
+	(
+	# Test modifying suppressed file
+	# Test adding suppressed file
+	cd hgrepo &&
+	echo "bad_a_modif" > bad_a.txt &&
+	echo "bad_b" > bad_b.txt &&
+	hg add bad_b.txt &&
+	hg commit -d "2023-03-17 02:00Z" -m "r1"
+	)
+}
+
+commit2() {
+	(
+	# Test removing suppressed file
+	cd hgrepo &&
+	hg rm bad_a.txt &&
+	hg commit -d "2023-03-17 03:00Z" -m "r2"
+	)
+}
+
+setup
+
+test_expect_success 'all in one' '
+	test_when_finished "rm -rf hgrepo gitrepo" &&
+
+	(
+	hg init hgrepo &&
+	commit0 &&
+	commit1 &&
+	commit2
+	) &&
+	git_create gitrepo &&
+	git_convert hgrepo gitrepo &&
+	git -C gitrepo fast-export --all > actual &&
+
+ 	test_cmp "$SHARNESS_TEST_DIRECTORY"/file_data_filter-removefiles.expected actual
+'
+
+test_done

t/file_data_filter.expected

@@ -0,0 +1,29 @@
+blob
+mark :1
+data 7
+a_file
+
+blob
+mark :2
+data 17
+a_file_to_rename
+
+reset refs/heads/master
+commit refs/heads/master
+mark :3
+author Grevious Bodily Harmsworth <gbh@example.com> 1679014800 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679014800 +0000
+data 3
+r0
+M 100644 :1 a.txt
+M 100644 :2 c.txt
+
+commit refs/heads/master
+mark :4
+author Grevious Bodily Harmsworth <gbh@example.com> 1679018400 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679018400 +0000
+data 3
+r1
+from :3
+D c.txt
+

t/file_data_filter.t

@@ -0,0 +1,84 @@
+#!/bin/bash
+#
+# Copyright (c) 2023 Felipe Contreras
+# Copyright (c) 2023 Frej Drejhammar
+#
+# Check that the file_data_filter is called for removed files.
+#
+
+test_description='Smoke test'
+
+. "${SHARNESS_TEST_SRCDIR-$(dirname "$0")/sharness}"/sharness.sh || exit 1
+
+check() {
+	echo "$3" > expected &&
+	git -C "$1" show -q --format='%s' "$2" > actual &&
+	test_cmp expected actual
+}
+
+git_create() {
+	git init -q "$1" &&
+	git -C "$1" config core.ignoreCase false
+}
+
+git_convert() {
+	(
+	cd "$2" &&
+	hg-fast-export.sh --repo "../$1" \
+			  -s --hgtags -n \
+			  --plugin ../../plugins/rename_file_test_plugin \
+			  --plugin dos2unix \
+			  --plugin shell_filter_file_contents=../../plugins/id
+	)
+}
+
+setup() {
+	cat > "$HOME"/.hgrc <<-EOF
+	[ui]
+	username = Grevious Bodily Harmsworth <gbh@example.com>
+	EOF
+}
+
+commit0() {
+	(
+	cd hgrepo &&
+	echo "a_file" > a.txt &&
+	echo "a_file_to_rename" > b.txt &&
+	hg add a.txt b.txt &&
+	hg commit -d "2023-03-17 01:00Z" -m "r0"
+	)
+}
+
+commit1() {
+	(
+	cd hgrepo &&
+	hg remove b.txt &&
+	hg commit -d "2023-03-17 02:00Z" -m "r1"
+	)
+}
+make-branch() {
+    hg branch "$1"
+    FILE=$(echo "$1" | sha1sum | cut -d " " -f 1)
+    echo "$1" > $FILE
+    hg add $FILE
+    hg commit -d "2023-03-17 $2:00Z" -m "Added file in branch $1"
+}
+
+setup
+
+test_expect_success 'all in one' '
+	test_when_finished "rm -rf hgrepo gitrepo" &&
+
+	(
+	hg init hgrepo &&
+	commit0 &&
+	commit1
+	) &&
+	git_create gitrepo &&
+	git_convert hgrepo gitrepo &&
+	git -C gitrepo fast-export --all > actual &&
+
+ 	test_cmp "$SHARNESS_TEST_DIRECTORY"/file_data_filter.expected actual
+'
+
+test_done

t/main.t

@@ -0,0 +1,144 @@
+#!/bin/bash
+#
+# Copyright (c) 2023 Felipe Contreras
+#
+
+test_description='Main tests'
+
+. "${SHARNESS_TEST_SRCDIR-$(dirname "$0")/sharness}"/sharness.sh || exit 1
+
+check() {
+	echo "$3" > expected &&
+	git -C "$1" show -q --format='%s' "$2" > actual &&
+	test_cmp expected actual
+}
+
+git_clone() {
+	(
+	git init -q "$2" &&
+	cd "$2" &&
+	git config core.ignoreCase false &&
+	hg-fast-export.sh --repo "../$1"
+	)
+}
+
+setup() {
+	cat > "$HOME"/.hgrc <<-EOF
+	[ui]
+	username = H G Wells <wells@example.com>
+	EOF
+}
+
+setup
+
+test_expect_success 'basic' '
+	test_when_finished "rm -rf hgrepo gitrepo" &&
+
+	(
+	hg init hgrepo &&
+	cd hgrepo &&
+	echo zero > content &&
+	hg add content &&
+	hg commit -m zero
+	) &&
+
+	git_clone hgrepo gitrepo &&
+	check gitrepo @ zero
+'
+
+test_expect_success 'merge' '
+	test_when_finished "rm -rf hgrepo gitrepo" &&
+
+	(
+	hg init hgrepo &&
+	cd hgrepo &&
+	echo a > content &&
+	echo a > file1 &&
+	hg add content file1 &&
+	hg commit -m "origin" &&
+
+	echo b > content &&
+	echo b > file2 &&
+	hg add file2 &&
+	hg rm file1 &&
+	hg commit -m "right" &&
+
+	hg update -r0 &&
+	echo c > content &&
+	hg commit -m "left" &&
+
+	HGMERGE=true hg merge -r1 &&
+	hg commit -m "merge"
+	) &&
+
+	git_clone hgrepo gitrepo &&
+
+	cat > expected <<-EOF &&
+	left
+	c
+	tree @:
+
+	content
+	file2
+	EOF
+
+	(
+	cd gitrepo
+	git show -q --format='%s' @^ &&
+	git show @:content &&
+	git show @:
+	) > actual &&
+
+	test_cmp expected actual
+'
+
+test_expect_success 'hg large file' '
+	test_when_finished "rm -rf hgrepo gitrepo" &&
+
+	(
+	hg init hgrepo &&
+	cd hgrepo &&
+	echo "[extensions]" >> .hg/hgrc
+	echo "largefiles =" >> .hg/hgrc
+	echo a > content &&
+	echo a > file1 &&
+	hg add content &&
+	hg add --large file1 &&
+	hg commit -m "origin" &&
+
+	echo b > content &&
+	echo b > file2 &&
+	hg add --large file2 &&
+	hg rm file1 &&
+	hg commit -m "right" &&
+
+	hg update -r0 &&
+	echo c > content &&
+	hg commit -m "left" &&
+
+	HGMERGE=true hg merge -r1 &&
+	hg commit -m "merge"
+	) &&
+
+	git_clone hgrepo gitrepo &&
+
+	cat > expected <<-EOF &&
+	left
+	c
+	tree @:
+
+	content
+	file2
+	EOF
+
+	(
+	cd gitrepo
+	git show -q --format='%s' @^ &&
+	git show @:content &&
+	git show @:
+	) > actual &&
+
+	test_cmp expected actual
+'
+
+test_done

t/plugins/id

@@ -0,0 +1,2 @@
+#!/bin/bash
+cat

t/plugins/removefiles_test_plugin/__init__.py

@@ -0,0 +1,15 @@
+import subprocess
+import shlex
+import sys
+from mercurial import node
+
+def build_filter(args):
+    return Filter(args)
+
+class Filter:
+    def __init__(self, args):
+        self.filter_contents = shlex.split(args)
+
+    def file_data_filter(self,file_data):
+        if file_data['filename'].startswith(b'bad'):
+            file_data['data'] = None

t/plugins/rename_file_test_plugin/__init__.py

@@ -0,0 +1,15 @@
+import subprocess
+import shlex
+import sys
+from mercurial import node
+
+def build_filter(args):
+    return Filter(args)
+
+class Filter:
+    def __init__(self, args):
+        self.filter_contents = shlex.split(args)
+
+    def file_data_filter(self,file_data):
+        if file_data['filename'] == b'b.txt':
+            file_data['filename'] = b'c.txt'

t/smoke-test.branchmap

@@ -0,0 +1,15 @@
+"feature"="renamed-feature"
+"a?"="valid-0"
+"a/"="valid-1"
+"a/b"="valid-2"
+"a/?"="valid-3"
+"?a"="valid-4"
+"a."="valid-5"
+"a.b"="valid-6"
+".a"="valid-7"
+"/"="valid-8"
+"___3"="___a"
+"__2"="__b"
+"_1"="_c"
+"åäö"="abc"
+"Feature- 12V Vac \"Venom\""="venom"

t/smoke-test.expected

@@ -0,0 +1,300 @@
+blob
+mark :1
+data 5
+r0-a
+
+blob
+mark :2
+data 5
+r0-b
+
+reset refs/heads/master
+commit refs/heads/master
+mark :3
+author Grevious Bodily Harmsworth <gbh@example.com> 1679014800 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679014800 +0000
+data 3
+r0
+M 100644 :1 a.txt
+M 100644 :2 b.txt
+
+blob
+mark :4
+data 5
+r1-c
+
+blob
+mark :5
+data 5
+r1-d
+
+commit refs/tags/2019_Spring_R2
+mark :6
+author Grevious Bodily Harmsworth <gbh@example.com> 1679018400 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679018400 +0000
+data 3
+r1
+from :3
+M 100644 :4 c.txt
+M 100644 :5 d.txt
+
+blob
+mark :7
+data 56
+e92e41dde44f9dbbac08bbb83351a65b6728f128 2019 Spring R2
+
+commit refs/heads/mainline
+mark :8
+author Grevious Bodily Harmsworth <gbh@example.com> 1679019000 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679019000 +0000
+data 52
+Added tag 2019 Spring R2 for changeset e92e41dde44f
+from :6
+M 100644 :7 .hgtags
+
+blob
+mark :9
+data 5
+r2-e
+
+blob
+mark :10
+data 5
+r2-f
+
+commit refs/heads/mainline
+mark :11
+author Grevious Bodily Harmsworth <gbh@example.com> 1679022000 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679022000 +0000
+data 3
+r2
+from :8
+M 100644 :9 e.txt
+M 100644 :10 f.txt
+
+commit refs/heads/mainline
+mark :12
+author badly-formed-user <devnull@localhost> 1679025600 +0000
+committer badly-formed-user <devnull@localhost> 1679025600 +0000
+data 3
+r3
+from :11
+M 100644 :9 g.txt
+M 100644 :10 h.txt
+
+blob
+mark :13
+data 10
+feature-a
+
+blob
+mark :14
+data 10
+feature-b
+
+commit refs/heads/renamed-feature
+mark :15
+author Grevious Bodily Harmsworth <gbh@example.com> 1679029200 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679029200 +0000
+data 8
+feature
+from :12
+M 100644 :13 feature-a.txt
+M 100644 :14 feature-b.txt
+
+blob
+mark :16
+data 3
+a?
+
+commit refs/heads/valid-0
+mark :17
+author Grevious Bodily Harmsworth <gbh@example.com> 1679032800 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679032800 +0000
+data 24
+Added file in branch a?
+from :15
+M 100644 :16 c1086ce03e4f52aadd1c93b1d097da510138522a
+
+blob
+mark :18
+data 3
+a/
+
+commit refs/heads/valid-1
+mark :19
+author Grevious Bodily Harmsworth <gbh@example.com> 1679036400 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679036400 +0000
+data 24
+Added file in branch a/
+from :17
+M 100644 :18 85ed6fbb96d655df9f194bc9107f2d86210b9263
+
+blob
+mark :20
+data 4
+a/b
+
+commit refs/heads/valid-2
+mark :21
+author Grevious Bodily Harmsworth <gbh@example.com> 1679040000 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679040000 +0000
+data 25
+Added file in branch a/b
+from :19
+M 100644 :20 aae42d317509399fdda80c4d8e46774d152dbd04
+
+blob
+mark :22
+data 4
+a/?
+
+commit refs/heads/valid-3
+mark :23
+author Grevious Bodily Harmsworth <gbh@example.com> 1679043600 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679043600 +0000
+data 25
+Added file in branch a/?
+from :21
+M 100644 :22 ba54a8de7fe91c5e6e0a2dd1b9b37de0976ff5a7
+
+blob
+mark :24
+data 3
+?a
+
+commit refs/heads/valid-4
+mark :25
+author Grevious Bodily Harmsworth <gbh@example.com> 1679047200 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679047200 +0000
+data 24
+Added file in branch ?a
+from :23
+M 100644 :24 d4cde16119b586025976741e87775762a2598984
+
+blob
+mark :26
+data 3
+a.
+
+commit refs/heads/valid-5
+mark :27
+author Grevious Bodily Harmsworth <gbh@example.com> 1679050800 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679050800 +0000
+data 24
+Added file in branch a.
+from :25
+M 100644 :26 b4ce96ddcee0706a8c51130917f910b2b29faf77
+
+blob
+mark :28
+data 4
+a.b
+
+commit refs/heads/valid-6
+mark :29
+author Grevious Bodily Harmsworth <gbh@example.com> 1679054400 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679054400 +0000
+data 25
+Added file in branch a.b
+from :27
+M 100644 :28 97051191e1a92daa11165ef10770bf964268c58b
+
+blob
+mark :30
+data 3
+.a
+
+commit refs/heads/valid-7
+mark :31
+author Grevious Bodily Harmsworth <gbh@example.com> 1679058000 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679058000 +0000
+data 24
+Added file in branch .a
+from :29
+M 100644 :30 a667f8feec02fdfa6649772f844a24cf1ad5ebec
+
+blob
+mark :32
+data 2
+/
+
+commit refs/heads/valid-8
+mark :33
+author Grevious Bodily Harmsworth <gbh@example.com> 1679061600 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679061600 +0000
+data 23
+Added file in branch /
+from :31
+M 100644 :32 8f27084b6294ddbe28dbcbf98f798730e8a79289
+
+blob
+mark :34
+data 5
+___3
+
+commit refs/heads/___a
+mark :35
+author Grevious Bodily Harmsworth <gbh@example.com> 1679065200 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679065200 +0000
+data 26
+Added file in branch ___3
+from :33
+M 100644 :34 9b171494eb6e5ce325934b1656e286ca0510a697
+
+blob
+mark :36
+data 4
+__2
+
+commit refs/heads/__b
+mark :37
+author Grevious Bodily Harmsworth <gbh@example.com> 1679068800 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679068800 +0000
+data 25
+Added file in branch __2
+from :35
+M 100644 :36 5dca703b71d2613c6bb3262b9b1741d6165e4a2f
+
+blob
+mark :38
+data 3
+_1
+
+commit refs/heads/_c
+mark :39
+author Grevious Bodily Harmsworth <gbh@example.com> 1679072400 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679072400 +0000
+data 24
+Added file in branch _1
+from :37
+M 100644 :38 2fee90e148a2afbd911b67ced9b6240151f904ec
+
+blob
+mark :40
+data 25
+Feature- 12V Vac "Venom"
+
+commit refs/heads/venom
+mark :41
+author Grevious Bodily Harmsworth <gbh@example.com> 1679076000 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679076000 +0000
+data 46
+Added file in branch Feature- 12V Vac "Venom"
+from :39
+M 100644 :40 b01def8779aed4be2f4b7325a89992a9aa566fec
+
+blob
+mark :42
+data 7
+åäö
+
+commit refs/heads/abc
+mark :43
+author Grevious Bodily Harmsworth <gbh@example.com> 1679079600 +0000
+committer Grevious Bodily Harmsworth <gbh@example.com> 1679079600 +0000
+data 28
+Added file in branch åäö
+from :41
+M 100644 :42 a0d01fcbff5d86327d542687dcfd8b299d054147
+

t/smoke-test.t

@@ -0,0 +1,163 @@
+#!/bin/bash
+#
+# Copyright (c) 2023 Felipe Contreras
+# Copyright (c) 2023 Frej Drejhammar
+#
+# Smoke test used to sanity test changes to fast-export.
+#
+
+test_description='Smoke test'
+
+. "${SHARNESS_TEST_SRCDIR-$(dirname "$0")/sharness}"/sharness.sh || exit 1
+
+check() {
+	echo "$3" > expected &&
+	git -C "$1" show -q --format='%s' "$2" > actual &&
+	test_cmp expected actual
+}
+
+git_create() {
+	git init -q "$1" &&
+	git -C "$1" config core.ignoreCase false
+}
+
+git_convert() {
+	(
+	cd "$2" &&
+	hg-fast-export.sh --repo "../$1" \
+			  -s --hgtags -n \
+			  -B "$SHARNESS_TEST_DIRECTORY"/smoke-test.branchmap \
+			  -T "$SHARNESS_TEST_DIRECTORY"/smoke-test.tagsmap
+	)
+}
+
+setup() {
+	cat > "$HOME"/.hgrc <<-EOF
+	[ui]
+	username = Grevious Bodily Harmsworth <gbh@example.com>
+	EOF
+}
+
+commit0() {
+	(
+	cd hgrepo &&
+	echo "r0-a" > a.txt &&
+	echo "r0-b" > b.txt &&
+	hg add a.txt b.txt &&
+	hg commit -d "2023-03-17 01:00Z" -m "r0" &&
+	hg bookmark bm0
+	)
+}
+
+commit1() {
+	(
+	cd hgrepo &&
+	echo "r1-c" > c.txt &&
+	echo "r1-d" > d.txt &&
+	hg branch mainline &&
+	hg add c.txt d.txt &&
+	hg commit -d "2023-03-17 02:00Z" -m "r1" &&
+	hg tag -d "2023-03-17 02:10Z" "2019 Spring R2"
+	)
+}
+
+commit2() {
+	(
+	cd hgrepo &&
+	echo "r2-e" > e.txt &&
+	echo "r2-f" > f.txt &&
+	hg add e.txt f.txt &&
+	hg commit -d "2023-03-17 03:00Z" -m "r2" &&
+	hg bookmark bm1
+	)
+}
+
+commit3() {
+	(
+	cd hgrepo &&
+	echo "r2-e" > g.txt &&
+	echo "r2-f" > h.txt &&
+	hg add g.txt h.txt &&
+	hg commit -d "2023-03-17 04:00Z" -u "badly-formed-user" -m "r3"
+	)
+}
+
+commit_rest() {
+	(
+	cd hgrepo &&
+
+	hg branch feature &&
+	echo "feature-a" > feature-a.txt &&
+	echo "feature-b" > feature-b.txt &&
+	hg add feature-a.txt feature-b.txt &&
+	hg commit -d "2023-03-17 05:00Z" -m "feature" &&
+	hg bookmark bm2 &&
+
+	# Now create strangely named branches
+	make-branch "a?" 06 &&
+	make-branch "a/" 07 &&
+	make-branch "a/b" 08 &&
+	make-branch "a/?" 09 &&
+	make-branch "?a" 10 &&
+	make-branch "a." 11 &&
+	make-branch "a.b" 12 &&
+	make-branch ".a" 13 &&
+	make-branch "/" 14 &&
+	make-branch "___3" 15 &&
+	make-branch "__2" 16 &&
+	make-branch "_1" 17 &&
+	make-branch "Feature- 12V Vac \"Venom\"" 18 &&
+	make-branch "åäö" 19 &&
+
+	hg bookmark bm-for-the-rest
+	)
+}
+
+make-branch() {
+    hg branch "$1"
+    FILE=$(echo "$1" | sha1sum | cut -d " " -f 1)
+    echo "$1" > $FILE
+    hg add $FILE
+    hg commit -d "2023-03-17 $2:00Z" -m "Added file in branch $1"
+}
+
+setup
+
+test_expect_success 'all in one' '
+	test_when_finished "rm -rf hgrepo gitrepo" &&
+
+	(
+	hg init hgrepo &&
+	commit0 &&
+	commit1 &&
+	commit2 &&
+	commit3 &&
+	commit_rest
+	) &&
+	git_create gitrepo &&
+	git_convert hgrepo gitrepo &&
+	git -C gitrepo fast-export --all > actual &&
+
+ 	test_cmp "$SHARNESS_TEST_DIRECTORY"/smoke-test.expected actual
+'
+
+test_expect_success 'incremental' '
+	test_when_finished "rm -rf hgrepo gitrepo" &&
+
+	hg init hgrepo &&
+	commit0 &&
+	git_create gitrepo &&
+	git_convert hgrepo gitrepo &&
+	commit1 &&
+	git_convert hgrepo gitrepo &&
+	commit2 &&
+	commit3 &&
+	git_convert hgrepo gitrepo &&
+	commit_rest &&
+	git_convert hgrepo gitrepo &&
+	git -C gitrepo fast-export --all > actual &&
+
+ 	test_cmp "$SHARNESS_TEST_DIRECTORY"/smoke-test.expected actual
+'
+
+test_done

t/smoke-test.tagsmap

@@ -0,0 +1 @@
+"2019 Spring R2"="2019_Spring_R2"