improve plugin scan performance

scm-plugin-backend/src/main/java/sonia/scm/plugin/AbstractPluginBackend.java

@@ -0,0 +1,82 @@
+/**
+ * Copyright (c) 2010, Sebastian Sdorra
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ * 3. Neither the name of SCM-Manager; nor the names of its
+ *    contributors may be used to endorse or promote products derived from this
+ *    software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * http://bitbucket.org/sdorra/scm-manager
+ *
+ */
+
+
+
+package sonia.scm.plugin;
+
+//~--- non-JDK imports --------------------------------------------------------
+
+import sonia.scm.util.Util;
+
+//~--- JDK imports ------------------------------------------------------------
+
+import java.io.File;
+
+import java.util.Arrays;
+
+/**
+ *
+ * @author Sebastian Sdorra
+ */
+public abstract class AbstractPluginBackend implements PluginBackend
+{
+
+  /**
+   * Method description
+   *
+   *
+   * @param plugins
+   */
+  @Override
+  public void addPlugins(PluginInformation... plugins)
+  {
+    if (Util.isNotEmpty(plugins))
+    {
+      addPlugins(Arrays.asList(plugins));
+    }
+  }
+
+  /**
+   * Method description
+   *
+   *
+   * @param scannedFiles
+   */
+  @Override
+  public void addScannedFiles(File... scannedFiles)
+  {
+    if (Util.isNotEmpty(scannedFiles))
+    {
+      addScannedFiles(Arrays.asList(scannedFiles));
+    }
+  }
+}

scm-plugin-backend/src/main/java/sonia/scm/plugin/BackendConfiguration.java

@@ -70,6 +70,17 @@ public class BackendConfiguration
     return directories;
   }
 
+  /**
+   * Method description
+   *
+   *
+   * @return
+   */
+  public Set<String> getExcludes()
+  {
+    return excludes;
+  }
+
   /**
    * Method description
    *
@@ -116,6 +127,17 @@ public class BackendConfiguration
     this.directories = directories;
   }
 
+  /**
+   * Method description
+   *
+   *
+   * @param excludes
+   */
+  public void setExcludes(Set<String> excludes)
+  {
+    this.excludes = excludes;
+  }
+
   /**
    * Method description
    *
@@ -156,6 +178,11 @@ public class BackendConfiguration
   @XmlElementWrapper(name = "directories")
   private Set<File> directories;
 
+  /** Field description */
+  @XmlElement(name = "exclude")
+  @XmlElementWrapper(name = "excludes")
+  private Set<String> excludes;
+
   /** Field description */
   private boolean multithreaded = true;
 

scm-plugin-backend/src/main/java/sonia/scm/plugin/DefaultPluginBackend.java

@@ -42,13 +42,22 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import sonia.scm.ConfigurationException;
+import sonia.scm.util.IOUtil;
+import sonia.scm.util.Util;
 
 //~--- JDK imports ------------------------------------------------------------
 
+import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.PrintWriter;
 
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 import javax.xml.bind.JAXBContext;
 import javax.xml.bind.JAXBException;
@@ -57,19 +66,25 @@ import javax.xml.bind.JAXBException;
  *
  * @author Sebastian Sdorra
  */
-public class DefaultPluginBackend implements PluginBackend
+public class DefaultPluginBackend extends AbstractPluginBackend
 {
 
+  /** Field description */
+  public static final String FILE_SCANNED = "scanned-files";
+
   /** Field description */
   public static final String FILE_STORE = "store.xml";
 
   /** Field description */
-  private static final Object LOCK_OBJECT = new Object();
+  private static final Object LOCK_PLUGINS = new Object();
 
   /** the logger for DefaultPluginBackend */
   private static final Logger logger =
     LoggerFactory.getLogger(DefaultPluginBackend.class);
 
+  /** Field description */
+  private static final Object LOCK_SCANNEDFILE = new Object();
+
   //~--- constructors ---------------------------------------------------------
 
   /**
@@ -78,12 +93,15 @@ public class DefaultPluginBackend implements PluginBackend
    *
    *
    * @param baseDirectory
+   * @param configuration
    */
   @Inject
   public DefaultPluginBackend(
-          @Named(ScmBackendModule.DIRECTORY_PROPERTY) File baseDirectory)
+          @Named(ScmBackendModule.DIRECTORY_PROPERTY) File baseDirectory,
+          BackendConfiguration configuration)
   {
     this.baseDirectory = baseDirectory;
+    this.configuration = configuration;
     this.storeFile = new File(baseDirectory, FILE_STORE);
 
     try
@@ -100,10 +118,16 @@ public class DefaultPluginBackend implements PluginBackend
       {
         pluginStore = new PluginBackendStore();
       }
+
+      readScannedFiles();
     }
     catch (JAXBException ex)
     {
-      throw new ConfigurationException(ex);
+      throw new ConfigurationException("could not read store", ex);
+    }
+    catch (IOException ex)
+    {
+      throw new ConfigurationException("could not read scanned files", ex);
     }
   }
 
@@ -113,21 +137,61 @@ public class DefaultPluginBackend implements PluginBackend
    * Method description
    *
    *
-   * @param plugin
+   *
+   * @param plugins
    */
   @Override
-  public void addPlugin(PluginInformation plugin)
+  public void addPlugins(Collection<PluginInformation> plugins)
   {
-    if (!pluginStore.contains(plugin))
+    synchronized (LOCK_PLUGINS)
     {
-      synchronized (LOCK_OBJECT)
+      boolean changed = false;
+
+      for (PluginInformation plugin : plugins)
+      {
+        if (!pluginStore.contains(plugin))
+        {
+          pluginStore.add(plugin);
+          changed = true;
+        }
+      }
+
+      if (changed)
       {
-        pluginStore.add(plugin);
         store();
       }
     }
   }
 
+  /**
+   * Method description
+   *
+   *
+   * @param scannedFiles
+   */
+  @Override
+  public void addScannedFiles(Collection<File> scannedFiles)
+  {
+    synchronized (LOCK_SCANNEDFILE)
+    {
+      boolean changed = false;
+
+      for (File file : scannedFiles)
+      {
+        if (!this.scannedFiles.contains(file))
+        {
+          changed = true;
+          this.scannedFiles.add(file);
+        }
+      }
+
+      if (changed)
+      {
+        writeScannedFiles();
+      }
+    }
+  }
+
   //~--- get methods ----------------------------------------------------------
 
   /**
@@ -142,6 +206,18 @@ public class DefaultPluginBackend implements PluginBackend
     return baseDirectory;
   }
 
+  /**
+   * Method description
+   *
+   *
+   * @return
+   */
+  @Override
+  public Set<String> getExcludes()
+  {
+    return configuration.getExcludes();
+  }
+
   /**
    * Method description
    *
@@ -179,14 +255,74 @@ public class DefaultPluginBackend implements PluginBackend
     return filteredPlugins;
   }
 
+  /**
+   * Method description
+   *
+   *
+   * @return
+   */
+  @Override
+  public Set<File> getScannedFiles()
+  {
+    return scannedFiles;
+  }
+
   //~--- methods --------------------------------------------------------------
 
+  /**
+   * Method description
+   *
+   *
+   *
+   * @throws IOException
+   */
+  private void readScannedFiles() throws IOException
+  {
+    File file = new File(baseDirectory, FILE_SCANNED);
+
+    if (file.exists())
+    {
+      BufferedReader reader = null;
+
+      try
+      {
+        reader = new BufferedReader(new FileReader(file));
+
+        String line = reader.readLine();
+
+        while (line != null)
+        {
+          if (Util.isNotEmpty(line))
+          {
+            File jar = new File(line);
+
+            if (jar.exists())
+            {
+              scannedFiles.add(jar);
+            }
+          }
+
+          line = reader.readLine();
+        }
+      }
+      finally
+      {
+        IOUtil.close(reader);
+      }
+    }
+  }
+
   /**
    * Method description
    *
    */
   private void store()
   {
+    if (logger.isInfoEnabled())
+    {
+      logger.info("store plugins");
+    }
+
     try
     {
       storeContext.createMarshaller().marshal(pluginStore, storeFile);
@@ -197,14 +333,53 @@ public class DefaultPluginBackend implements PluginBackend
     }
   }
 
+  /**
+   * Method description
+   *
+   */
+  private void writeScannedFiles()
+  {
+    if (logger.isInfoEnabled())
+    {
+      logger.info("store scanned files");
+    }
+
+    File file = new File(baseDirectory, FILE_SCANNED);
+    PrintWriter writer = null;
+
+    try
+    {
+      writer = new PrintWriter(file);
+
+      for (File f : scannedFiles)
+      {
+        writer.println(f.getAbsolutePath());
+      }
+    }
+    catch (IOException ex)
+    {
+      logger.error("could not write scanned files", ex);
+    }
+    finally
+    {
+      IOUtil.close(writer);
+    }
+  }
+
   //~--- fields ---------------------------------------------------------------
 
   /** Field description */
   private File baseDirectory;
 
+  /** Field description */
+  private BackendConfiguration configuration;
+
   /** Field description */
   private PluginBackendStore pluginStore;
 
+  /** Field description */
+  private Set<File> scannedFiles = new HashSet<File>();
+
   /** Field description */
   private JAXBContext storeContext;
 

scm-plugin-backend/src/main/java/sonia/scm/plugin/PluginBackend.java

@@ -37,7 +37,9 @@ package sonia.scm.plugin;
 
 import java.io.File;
 
+import java.util.Collection;
 import java.util.List;
+import java.util.Set;
 
 /**
  *
@@ -50,9 +52,35 @@ public interface PluginBackend
    * Method description
    *
    *
-   * @param plugin
+   *
+   * @param plugins
    */
-  public void addPlugin(PluginInformation plugin);
+  public void addPlugins(PluginInformation... plugins);
+
+  /**
+   * Method description
+   *
+   *
+   * @param plugins
+   */
+  public void addPlugins(Collection<PluginInformation> plugins);
+
+  /**
+   * Method description
+   *
+   *
+   * @param scannedFiles
+   */
+  public void addScannedFiles(Collection<File> scannedFiles);
+
+  /**
+   * Method description
+   *
+   *
+   *
+   * @param scannedFiles
+   */
+  public void addScannedFiles(File... scannedFiles);
 
   //~--- get methods ----------------------------------------------------------
 
@@ -64,6 +92,14 @@ public interface PluginBackend
    */
   public File getBaseDirectory();
 
+  /**
+   * Method description
+   *
+   *
+   * @return
+   */
+  public Set<String> getExcludes();
+
   /**
    * Method description
    *
@@ -81,4 +117,12 @@ public interface PluginBackend
    * @return
    */
   public List<PluginInformation> getPlugins(PluginFilter filter);
+
+  /**
+   * Method description
+   *
+   *
+   * @return
+   */
+  public Set<File> getScannedFiles();
 }

scm-plugin-backend/src/main/java/sonia/scm/plugin/rest/DefaultPluginFilter.java

@@ -83,9 +83,12 @@ public class DefaultPluginFilter implements PluginFilter
   public boolean accept(PluginInformation plugin)
   {
     PluginCondition condition = plugin.getCondition();
+
     return ((condition != null) && condition.isSupported(version, os, arch))
-           || (condition == null) &&
-           (snapshot || !plugin.getVersion().toUpperCase().contains(VERSION_SNAPSHOT));
+           || (condition == null)
+              && (snapshot
+                  ||!plugin.getVersion().toUpperCase().contains(
+                      VERSION_SNAPSHOT));
   }
 
   //~--- fields ---------------------------------------------------------------

scm-plugin-backend/src/main/java/sonia/scm/plugin/rest/PluginResource.java

@@ -98,17 +98,19 @@ public class PluginResource
    *
    *
    * @param version
+   * @param os
+   * @param arch
    * @param snapshot
    *
    * @return
    */
   @GET
   @Produces(MediaType.APPLICATION_XML)
-  public Response getPlugins(
-          @PathParam("version") String version,
-          @QueryParam("os") String os,
-          @QueryParam("arch") String arch,
-          @DefaultValue("false") @QueryParam("snapshot") boolean snapshot)
+  public Response getPlugins(@PathParam("version") String version,
+                             @QueryParam("os") String os,
+                             @QueryParam("arch") String arch,
+                             @DefaultValue("false")
+  @QueryParam("snapshot") boolean snapshot)
   {
     if (logger.isDebugEnabled())
     {
@@ -117,8 +119,7 @@ public class PluginResource
     }
 
     List<PluginInformation> plugins =
-      backend.getPlugins(new DefaultPluginFilter(version, os, arch,
-        snapshot));
+      backend.getPlugins(new DefaultPluginFilter(version, os, arch, snapshot));
     PluginCenter pc = new PluginCenter();
 
     pc.setPlugins(getNewestPlugins(plugins));

scm-plugin-backend/src/main/java/sonia/scm/plugin/scanner/DefaultFileFilter.java

@@ -0,0 +1,129 @@
+/**
+ * Copyright (c) 2010, Sebastian Sdorra
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ * 3. Neither the name of SCM-Manager; nor the names of its
+ *    contributors may be used to endorse or promote products derived from this
+ *    software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * http://bitbucket.org/sdorra/scm-manager
+ *
+ */
+
+
+
+package sonia.scm.plugin.scanner;
+
+//~--- JDK imports ------------------------------------------------------------
+
+import java.io.File;
+import java.io.FileFilter;
+
+import java.util.Collection;
+import java.util.Collections;
+
+/**
+ *
+ * @author Sebastian Sdorra
+ */
+public class DefaultFileFilter implements FileFilter
+{
+
+  /** Field description */
+  public static final String PLUGIN_EXTENSION = ".jar";
+
+  //~--- constructors ---------------------------------------------------------
+
+  /**
+   * Constructs ...
+   *
+   */
+  public DefaultFileFilter()
+  {
+    this(null);
+  }
+
+  /**
+   * Constructs ...
+   *
+   *
+   * @param excludes
+   */
+  public DefaultFileFilter(Collection<String> excludes)
+  {
+    if (excludes == null)
+    {
+      excludes = Collections.emptySet();
+    }
+    else
+    {
+      this.excludes = excludes;
+    }
+  }
+
+  //~--- methods --------------------------------------------------------------
+
+  /**
+   * Method description
+   *
+   *
+   * @param file
+   *
+   * @return
+   */
+  @Override
+  public boolean accept(File file)
+  {
+    boolean accepted = false;
+
+    if (file.isDirectory())
+    {
+      accepted = true;
+    }
+    else
+    {
+      String name = file.getName();
+
+      if (name.endsWith(PLUGIN_EXTENSION))
+      {
+        accepted = true;
+
+        for (String exclude : excludes)
+        {
+          if (name.matches(exclude))
+          {
+            accepted = false;
+
+            break;
+          }
+        }
+      }
+    }
+
+    return accepted;
+  }
+
+  //~--- fields ---------------------------------------------------------------
+
+  /** Field description */
+  private Collection<String> excludes;
+}

scm-plugin-backend/src/main/java/sonia/scm/plugin/scanner/DefaultPluginScanner.java

@@ -42,13 +42,19 @@ import sonia.scm.plugin.Plugin;
 import sonia.scm.plugin.PluginBackend;
 import sonia.scm.plugin.PluginCondition;
 import sonia.scm.plugin.PluginException;
+import sonia.scm.plugin.PluginInformation;
 import sonia.scm.util.IOUtil;
+import sonia.scm.util.Util;
 
 //~--- JDK imports ------------------------------------------------------------
 
 import java.io.File;
+import java.io.FileFilter;
 import java.io.FileInputStream;
+import java.io.IOException;
 
+import java.util.HashSet;
+import java.util.Set;
 import java.util.jar.JarEntry;
 import java.util.jar.JarInputStream;
 
@@ -66,7 +72,6 @@ public class DefaultPluginScanner implements PluginScanner
   public static final String PLUGIN_DESCRIPTOR = "META-INF/scm/plugin.xml";
 
   /** Field description */
-  public static final String PLUGIN_EXTENSION = ".jar";
 
   /** the logger for DefaultPluginScanner */
   private static final Logger logger =
@@ -102,23 +107,30 @@ public class DefaultPluginScanner implements PluginScanner
   @Override
   public void scannDirectory(PluginBackend backend, File directory)
   {
-    if (logger.isDebugEnabled())
+    if (logger.isInfoEnabled())
     {
-      logger.debug("scann directory {}", directory.getPath());
+      logger.info("start scann of basedirectory {}", directory.getPath());
     }
 
-    File[] files = directory.listFiles();
+    Set<File> scannedFiles = new HashSet<File>();
+    Set<PluginInformation> plugins = new HashSet<PluginInformation>();
+    FileFilter filter = new DefaultFileFilter(backend.getExcludes());
 
-    for (File file : files)
+    scannDirectory(backend, scannedFiles, plugins, directory, filter);
+
+    if (Util.isNotEmpty(plugins))
     {
-      if (file.isDirectory())
-      {
-        scannDirectory(backend, file);
-      }
-      else if (file.getName().endsWith(PLUGIN_EXTENSION))
-      {
-        scannFile(backend, file);
-      }
+      backend.addPlugins(plugins);
+    }
+
+    if (Util.isNotEmpty(scannedFiles))
+    {
+      backend.addScannedFiles(scannedFiles);
+    }
+
+    if (logger.isInfoEnabled())
+    {
+      logger.info("finish scann of basedirectory {}", directory.getPath());
     }
   }
 
@@ -127,9 +139,57 @@ public class DefaultPluginScanner implements PluginScanner
    *
    *
    * @param backend
-   * @param file
+   * @param scannedFiles
+   * @param plugins
+   * @param directory
+   * @param filter
    */
-  private void scannFile(PluginBackend backend, File file)
+  private void scannDirectory(PluginBackend backend, Set<File> scannedFiles,
+                              Set<PluginInformation> plugins, File directory,
+                              FileFilter filter)
+  {
+    if (logger.isDebugEnabled())
+    {
+      logger.debug("scann directory {}", directory.getPath());
+    }
+
+    File[] files = directory.listFiles(filter);
+
+    for (File file : files)
+    {
+      if (file.isDirectory())
+      {
+        scannDirectory(backend, scannedFiles, plugins, file, filter);
+      }
+      else if (!backend.getScannedFiles().contains(file))
+      {
+        try
+        {
+          scannFile(plugins, file);
+          scannedFiles.add(file);
+        }
+        catch (Exception ex)
+        {
+          logger.error(
+              "could not read plugin descriptor ".concat(file.getPath()), ex);
+        }
+      }
+    }
+  }
+
+  /**
+   * Method description
+   *
+   *
+   *
+   * @param plugins
+   * @param file
+   *
+   * @throws IOException
+   * @throws JAXBException
+   */
+  private void scannFile(Set<PluginInformation> plugins, File file)
+          throws IOException, JAXBException
   {
     if (logger.isDebugEnabled())
     {
@@ -171,7 +231,7 @@ public class DefaultPluginScanner implements PluginScanner
               logger.info("add plugin {} to backend", file.getPath());
             }
 
-            backend.addPlugin(plugin.getInformation());
+            plugins.add(plugin.getInformation());
           }
           else if (logger.isWarnEnabled())
           {
@@ -187,11 +247,6 @@ public class DefaultPluginScanner implements PluginScanner
         }
       }
     }
-    catch (Exception ex)
-    {
-      logger.error("could not read plugin descriptor ".concat(file.getPath()),
-                   ex);
-    }
     finally
     {
       IOUtil.close(inputStream);

scm-plugin-backend/src/main/java/sonia/scm/plugin/scanner/PluginScanner.java

@@ -33,10 +33,13 @@
 
 package sonia.scm.plugin.scanner;
 
+//~--- non-JDK imports --------------------------------------------------------
+
+import sonia.scm.plugin.PluginBackend;
+
 //~--- JDK imports ------------------------------------------------------------
 
 import java.io.File;
-import sonia.scm.plugin.PluginBackend;
 
 /**
  *

scm-plugin-backend/src/main/java/sonia/scm/plugin/scanner/PluginScannerTimerTask.java

@@ -93,10 +93,11 @@ public class PluginScannerTimerTask extends TimerTask
 
     for (File directory : configuration.getDirectories())
     {
-      if ( logger.isDebugEnabled() )
+      if (logger.isDebugEnabled())
       {
         logger.info("scann directory {}", directory.getPath());
       }
+
       PluginScanner scanner = scannerFactory.createScanner();
 
       if (configuration.isMultithreaded())