From 17d31b80e0aacf485ed7f2a9a8e03392aa69f346 Mon Sep 17 00:00:00 2001 From: Sebastian Sdorra Date: Wed, 1 Jul 2020 15:01:14 +0200 Subject: [PATCH] implemented a new auto configuration mechanism for mercurial on unix This new mechanism is more robust and should find python and python3 installations --- .../scm/autoconfig/AutoConfigurator.java | 47 ++++ .../scm/autoconfig/UnixAutoConfigurator.java | 210 ++++++++++++++++++ .../sonia/scm/installer/MacOSHgInstaller.java | 4 +- .../sonia/scm/installer/UnixHgInstaller.java | 4 +- .../scm/repository/HgRepositoryHandler.java | 16 +- .../autoconfig/UnixAutoConfiguratorTest.java | 164 ++++++++++++++ 6 files changed, 437 insertions(+), 8 deletions(-) create mode 100644 scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/autoconfig/AutoConfigurator.java create mode 100644 scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/autoconfig/UnixAutoConfigurator.java create mode 100644 scm-plugins/scm-hg-plugin/src/test/java/sonia/scm/autoconfig/UnixAutoConfiguratorTest.java diff --git a/scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/autoconfig/AutoConfigurator.java b/scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/autoconfig/AutoConfigurator.java new file mode 100644 index 0000000000..ddaa332506 --- /dev/null +++ b/scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/autoconfig/AutoConfigurator.java @@ -0,0 +1,47 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.autoconfig; + +import sonia.scm.repository.HgConfig; +import sonia.scm.util.SystemUtil; + +import java.nio.file.Path; +import java.util.Optional; + +public interface AutoConfigurator { + + static Optional get() { + // at the moment we have only support for unix based systems. + if (SystemUtil.isUnix()) { + return Optional.of(new UnixAutoConfigurator(System.getenv())); + } + return Optional.empty(); + } + + HgConfig configure(); + + HgConfig configure(Path hg); + +} diff --git a/scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/autoconfig/UnixAutoConfigurator.java b/scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/autoconfig/UnixAutoConfigurator.java new file mode 100644 index 0000000000..01364e39aa --- /dev/null +++ b/scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/autoconfig/UnixAutoConfigurator.java @@ -0,0 +1,210 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.autoconfig; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.io.MoreFiles; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sonia.scm.repository.HgConfig; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +public class UnixAutoConfigurator implements AutoConfigurator { + + private static final Logger LOG = LoggerFactory.getLogger(UnixAutoConfigurator.class); + + private static final List ADDITIONAL_PATH = ImmutableList.of( + "/usr/bin", + "/usr/local/bin", + "/opt/local/bin" + ); + + private final Set fsPaths; + + private Executor executor = (Path binary, String... args) -> { + ProcessBuilder builder = new ProcessBuilder( + Lists.asList(binary.toString(), args).toArray(new String[0]) + ); + Process process = builder.start(); + int rc = process.waitFor(); + if (rc != 0) { + throw new IOException(binary.toString() + " failed with return code " + rc); + } + return process.getInputStream(); + }; + + UnixAutoConfigurator(Map env) { + this(env, ADDITIONAL_PATH); + } + + UnixAutoConfigurator(Map env, List additionalPaths) { + String path = env.getOrDefault("PATH", ""); + fsPaths = new LinkedHashSet<>(); + fsPaths.addAll(Splitter.on(File.pathSeparator).splitToList(path)); + fsPaths.addAll(additionalPaths); + } + + @VisibleForTesting + void setExecutor(Executor executor) { + this.executor = executor; + } + + @Override + public HgConfig configure() { + Optional hg = findInPath("hg"); + if (hg.isPresent()) { + return configure(hg.get()); + } + return new HgConfig(); + } + + private Optional findInPath(String binary) { + for (String directory : fsPaths) { + Path binaryPath = Paths.get(directory, binary); + if (Files.exists(binaryPath)) { + return Optional.of(binaryPath); + } + } + return Optional.empty(); + } + + + private Optional findModulePath(Path hg) { + if (!Files.isExecutable(hg)) { + LOG.warn("{} is not executable", hg); + return Optional.empty(); + } + try { + InputStream debuginstall = executor.execute(hg, "debuginstall"); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(debuginstall))) { + while (reader.ready()) { + String line = reader.readLine(); + if (line.contains("installed modules")) { + int start = line.indexOf("("); + int end = line.indexOf(")"); + Path modulePath = Paths.get(line.substring(start + 1, end)); + if (Files.exists(modulePath)) { + return Optional.of(modulePath); + } else { + LOG.warn("could not find module path at {}", modulePath); + } + } + } + } + } catch (IOException ex) { + LOG.warn("failed to parse debuginstall of {}", hg); + } catch (InterruptedException e) { + LOG.warn("interrupted during debuginstall parsing of {}", hg); + Thread.currentThread().interrupt(); + } + return Optional.empty(); + } + + @Override + public HgConfig configure(Path hg) { + HgConfig config = new HgConfig(); + try { + if (Files.exists(hg)) { + configureWithExistingHg(hg, config); + } else { + LOG.warn("{} does not exists", hg); + } + } catch (IOException e) { + LOG.warn("failed to read first line of {}", hg); + } + return config; + } + + private void configureWithExistingHg(Path hg, HgConfig config) throws IOException { + config.setHgBinary(hg.toAbsolutePath().toString()); + Optional pythonFromShebang = findPythonFromShebang(hg); + if (pythonFromShebang.isPresent()) { + config.setPythonBinary(pythonFromShebang.get().toAbsolutePath().toString()); + } else { + LOG.warn("could not find python from shebang, searching for python in path"); + Optional python = findInPath("python"); + if (!python.isPresent()) { + LOG.warn("could not find python in path, searching for python3 instead"); + python = findInPath("python3"); + } + if (python.isPresent()) { + config.setPythonBinary(python.get().toAbsolutePath().toString()); + } else { + LOG.warn("could not find python in path"); + } + } + + Optional modulePath = findModulePath(hg); + if (modulePath.isPresent()) { + config.setPythonPath(modulePath.get().toAbsolutePath().toString()); + } else { + LOG.warn("could not find module path"); + } + + } + + private Optional findPythonFromShebang(Path hg) throws IOException { + String shebang = MoreFiles.asCharSource(hg, StandardCharsets.UTF_8).readFirstLine(); + if (shebang != null && shebang.startsWith("#!")) { + String substring = shebang.substring(2); + String[] parts = substring.split("\\s+"); + if (parts.length > 1) { + return findInPath(parts[1]); + } else { + Path python = Paths.get(parts[0]); + if (Files.exists(python)) { + return Optional.of(python); + } else { + LOG.warn("python binary from shebang {} does not exists", python); + } + } + } else { + LOG.warn("first line does not look like a shebang: {}", shebang); + } + return Optional.empty(); + } + + @FunctionalInterface + interface Executor { + InputStream execute(Path binary, String... args) throws IOException, InterruptedException; + } +} diff --git a/scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/installer/MacOSHgInstaller.java b/scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/installer/MacOSHgInstaller.java index 2d33eb9f15..52a48d1cad 100644 --- a/scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/installer/MacOSHgInstaller.java +++ b/scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/installer/MacOSHgInstaller.java @@ -21,7 +21,7 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ - + package sonia.scm.installer; //~--- non-JDK imports -------------------------------------------------------- @@ -40,7 +40,9 @@ import java.io.IOException; /** * * @author Sebastian Sdorra + * @deprecated use {@link sonia.scm.autoconfig.AutoConfigurator} */ +@Deprecated public class MacOSHgInstaller extends UnixHgInstaller { diff --git a/scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/installer/UnixHgInstaller.java b/scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/installer/UnixHgInstaller.java index 470f53de5c..ba65b943e8 100644 --- a/scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/installer/UnixHgInstaller.java +++ b/scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/installer/UnixHgInstaller.java @@ -21,7 +21,7 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ - + package sonia.scm.installer; //~--- non-JDK imports -------------------------------------------------------- @@ -40,7 +40,9 @@ import java.util.List; /** * * @author Sebastian Sdorra + * @deprecated use {@link sonia.scm.autoconfig.AutoConfigurator} */ +@Deprecated public class UnixHgInstaller extends AbstractHgInstaller { diff --git a/scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/repository/HgRepositoryHandler.java b/scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/repository/HgRepositoryHandler.java index 6c33607750..a349a1f83c 100644 --- a/scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/repository/HgRepositoryHandler.java +++ b/scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/repository/HgRepositoryHandler.java @@ -33,6 +33,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import sonia.scm.ConfigurationException; import sonia.scm.SCMContextProvider; +import sonia.scm.autoconfig.AutoConfigurator; import sonia.scm.installer.HgInstaller; import sonia.scm.installer.HgInstallerFactory; import sonia.scm.io.ExtendedCommand; @@ -55,6 +56,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.text.MessageFormat; +import java.util.Optional; @Singleton @Extension @@ -91,14 +93,11 @@ public class HgRepositoryHandler this.hgContextProvider = hgContextProvider; this.workingCopyFactory = workingCopyFactory; - try - { + try { this.jaxbContext = JAXBContext.newInstance(BrowserResult.class, BlameResult.class, Changeset.class, ChangesetPagingResult.class, HgVersion.class); - } - catch (JAXBException ex) - { + } catch (JAXBException ex) { throw new ConfigurationException("could not create jaxbcontext", ex); } } @@ -139,7 +138,12 @@ public class HgRepositoryHandler super.loadConfig(); if (config == null) { - doAutoConfiguration(new HgConfig()); + HgConfig config = null; + Optional autoConfigurator = AutoConfigurator.get(); + if (autoConfigurator.isPresent()) { + config = autoConfigurator.get().configure(); + } + doAutoConfiguration(config != null ? config : new HgConfig()); } } diff --git a/scm-plugins/scm-hg-plugin/src/test/java/sonia/scm/autoconfig/UnixAutoConfiguratorTest.java b/scm-plugins/scm-hg-plugin/src/test/java/sonia/scm/autoconfig/UnixAutoConfiguratorTest.java new file mode 100644 index 0000000000..e0c09815f3 --- /dev/null +++ b/scm-plugins/scm-hg-plugin/src/test/java/sonia/scm/autoconfig/UnixAutoConfiguratorTest.java @@ -0,0 +1,164 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.autoconfig; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.assertj.core.util.Strings; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import sonia.scm.repository.HgConfig; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +class UnixAutoConfiguratorTest { + + @Test + void shouldConfigureWithShebangPath(@TempDir Path directory) throws IOException { + Path hg = directory.resolve("hg"); + Path python = directory.resolve("python"); + + Files.write(hg, ("#!" + python.toAbsolutePath().toString()).getBytes(StandardCharsets.UTF_8)); + Files.createFile(python); + + UnixAutoConfigurator configurator = create(directory); + HgConfig config = configurator.configure(); + + assertThat(config.getHgBinary()).isEqualTo(hg.toString()); + assertThat(config.getPythonBinary()).isEqualTo(python.toString()); + } + + private UnixAutoConfigurator create(@TempDir Path directory) { + return new UnixAutoConfigurator(createEnv(directory), Collections.emptyList()); + } + + private Map createEnv(Path... paths) { + return ImmutableMap.of("PATH", Joiner.on(File.pathSeparator).join(paths)); + } + + @Test + void shouldConfigureWithShebangEnv(@TempDir Path directory) throws IOException { + Path hg = directory.resolve("hg"); + Path python = directory.resolve("python3.8"); + + Files.write(hg, "#!/usr/bin/env python3.8".getBytes(StandardCharsets.UTF_8)); + Files.createFile(python); + + UnixAutoConfigurator configurator = create(directory); + HgConfig config = configurator.configure(); + + assertThat(config.getHgBinary()).isEqualTo(hg.toString()); + assertThat(config.getPythonBinary()).isEqualTo(python.toString()); + } + + @Test + void shouldConfigureWithoutShebang(@TempDir Path directory) throws IOException { + Path hg = directory.resolve("hg"); + Path python = directory.resolve("python"); + + Files.createFile(hg); + Files.createFile(python); + + UnixAutoConfigurator configurator = create(directory); + HgConfig config = configurator.configure(); + + assertThat(config.getHgBinary()).isEqualTo(hg.toString()); + assertThat(config.getPythonBinary()).isEqualTo(python.toString()); + } + + @Test + void shouldConfigureWithoutShebangButWithPython3(@TempDir Path directory) throws IOException { + Path hg = directory.resolve("hg"); + Path python = directory.resolve("python3"); + + Files.createFile(hg); + Files.createFile(python); + + UnixAutoConfigurator configurator = create(directory); + HgConfig config = configurator.configure(); + + assertThat(config.getHgBinary()).isEqualTo(hg.toString()); + assertThat(config.getPythonBinary()).isEqualTo(python.toString()); + } + + @Test + void shouldConfigureFindPythonInAdditionalPath(@TempDir Path directory) throws IOException { + Path def = directory.resolve("default"); + Files.createDirectory(def); + Path additional = directory.resolve("additional"); + Files.createDirectory(additional); + + Path hg = def.resolve("hg"); + Path python = additional.resolve("python"); + + Files.createFile(hg); + Files.createFile(python); + + UnixAutoConfigurator configurator = new UnixAutoConfigurator( + createEnv(def), ImmutableList.of(additional.toAbsolutePath().toString()) + ); + + HgConfig config = configurator.configure(); + assertThat(config.getHgBinary()).isEqualTo(hg.toString()); + assertThat(config.getPythonBinary()).isEqualTo(python.toString()); + } + + @Test + void shouldFindModulePathFromDebuginstallOutput(@TempDir Path directory) throws IOException { + Path hg = directory.resolve("hg"); + Files.createFile(hg); + hg.toFile().setExecutable(true); + + Path modules = directory.resolve("modules"); + Files.createDirectories(modules); + + UnixAutoConfigurator configurator = create(directory); + configurator.setExecutor((Path binary, String... args) -> { + String content = String.join("\n", + "checking Python executable (/python3.8)", + "checking Python lib (/python3.8)...", + "checking installed modules (" + modules.toString() + ")...", + "checking templates (/mercurial/templates)...", + "checking default template (/mercurial/templates/map-cmdline.default))" + ); + return new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + }); + HgConfig config = configurator.configure(); + + assertThat(config.getHgBinary()).isEqualTo(hg.toString()); + assertThat(config.getPythonPath()).isEqualTo(modules.toString()); + } + +}