(refs #1259) Add SQL import capability and remove XML export / import

src/main/scala/gitbucket/core/controller/FileUploadController.scala

@@ -79,15 +79,10 @@ class FileUploadController extends ScalatraServlet with FileUploadSupport with R
   }
 
   post("/import") {
+    import JDBCUtil._
     session.get(Keys.Session.LoginAccount).collect { case loginAccount: Account if loginAccount.isAdmin =>
       execute({ (file, fileId) =>
-        if(file.getName.endsWith(".xml")){
+        request2Session(request).conn.importAsSQL(file.getInputStream)
-          import JDBCUtil._
-          val conn = request2Session(request).conn
-          conn.importAsXML(file.getInputStream)
-        } else {
-          throw new RuntimeException("Import is available for only the XML file.")
-        }
       }, _ => true)
     }
     redirect("/admin/data")

src/main/scala/gitbucket/core/controller/SystemSettingsController.scala

@@ -303,12 +303,7 @@ trait SystemSettingsControllerBase extends AccountManagementControllerBase {
 
   post("/admin/export")(adminOnly {
     import gitbucket.core.util.JDBCUtil._
-    val session = request2Session(request)
+    val file = request2Session(request).conn.exportAsSQL(request.getParameterValues("tableNames").toSeq)
-    val file = if(params("type") == "sql"){
-      session.conn.exportAsSQL(request.getParameterValues("tableNames").toSeq)
-    } else {
-      session.conn.exportAsXML(request.getParameterValues("tableNames").toSeq)
-    }
 
     contentType = "application/octet-stream"
     response.setHeader("Content-Disposition", "attachment; filename=" + file.getName)

src/main/scala/gitbucket/core/util/JDBCUtil.scala

@@ -3,11 +3,8 @@ package gitbucket.core.util
 import java.io._
 import java.sql._
 import java.text.SimpleDateFormat
-import javax.xml.stream.{XMLStreamConstants, XMLInputFactory, XMLOutputFactory}
 import ControlUtil._
-import scala.StringBuilder
 import scala.annotation.tailrec
-import scala.collection.mutable
 import scala.collection.mutable.ListBuffer
 
 /**
@@ -64,65 +61,34 @@ object JDBCUtil {
       }
     }
 
-    def importAsXML(in: InputStream): Unit = {
+    def importAsSQL(in: InputStream): Unit = {
       conn.setAutoCommit(false)
       try {
-        val factory = XMLInputFactory.newInstance()
+        using(in){ in =>
-        using(factory.createXMLStreamReader(in, "UTF-8")){ reader =>
+          var out = new ByteArrayOutputStream()
-          // stateful objects
-          var elementName   = ""
-          var insertTable   = ""
-          var insertColumns = Map.empty[String, (String, String)]
 
-          while(reader.hasNext){
+          var length = 0
-            reader.next()
+          val bytes = new scala.Array[Byte](1024 * 8)
+          var stringLiteral = false
 
-            reader.getEventType match {
+          var count = 0
-              case XMLStreamConstants.START_ELEMENT =>
-                elementName = reader.getName.getLocalPart
-                if(elementName == "insert"){
-                  insertTable = reader.getAttributeValue(null, "table")
-                } else if(elementName == "delete"){
-                  val tableName = reader.getAttributeValue(null, "table")
-                  conn.update(s"DELETE FROM ${tableName}")
-                } else if(elementName == "column"){
-                  val columnName  = reader.getAttributeValue(null, "name")
-                  val columnType  = reader.getAttributeValue(null, "type")
-                  val columnValue = reader.getElementText
-                  insertColumns = insertColumns + (columnName -> (columnType, columnValue))
-                }
-              case XMLStreamConstants.END_ELEMENT =>
-                // Execute insert statement
-                reader.getName.getLocalPart match {
-                  case "insert" => {
-                    val sb = new StringBuilder()
-                    sb.append(s"INSERT INTO ${insertTable} (")
-                    sb.append(insertColumns.map { case (columnName, _) => columnName }.mkString(", "))
-                    sb.append(") VALUES (")
-                    sb.append(insertColumns.map { case (_, (columnType, columnValue)) =>
-                      if(columnType == null || columnValue == null){
-                        "NULL"
-                      } else if(columnType == "string"){
-                        "'" + columnValue.replace("'", "''") + "'"
-                      } else if(columnType == "timestamp"){
-                        "'" + columnValue + "'"
-                      } else {
-                        columnValue.toString
-                      }
-                    }.mkString(", "))
-                    sb.append(")")
 
-                    conn.update(sb.toString)
+          while({ length = in.read(bytes); length != -1 }){
-
+            for(i <- 0 to length - 1){
-                    insertColumns = Map.empty[String, (String, String)] // Clear column information
+              val c = bytes(i)
-                  }
+              if(c == '\''){
-                  case _ => // Nothing to do
+                stringLiteral = !stringLiteral
-                }
+              }
-              case _ => // Nothing to do
+              if(c == ';' && !stringLiteral){
+                val sql = new String(out.toByteArray, "UTF-8")
+                conn.update(sql)
+                out = new ByteArrayOutputStream()
+              } else {
+                out.write(c)
+              }
             }
           }
         }
-
         conn.commit()
 
       } catch {
@@ -133,68 +99,6 @@ object JDBCUtil {
       }
     }
 
-    def exportAsXML(targetTables: Seq[String]): File = {
-      val dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss")
-      val file = File.createTempFile("gitbucket-export-", ".xml")
-
-      val factory = XMLOutputFactory.newInstance()
-      using(factory.createXMLStreamWriter(new FileOutputStream(file), "UTF-8")){ writer =>
-        val dbMeta = conn.getMetaData
-        val allTablesInDatabase = allTablesOrderByDependencies(dbMeta)
-
-        writer.writeStartDocument("UTF-8", "1.0")
-        writer.writeStartElement("tables")
-
-        println(allTablesInDatabase.mkString(", "))
-
-        allTablesInDatabase.reverse.foreach { tableName =>
-          if (targetTables.contains(tableName)) {
-            writer.writeStartElement("delete")
-            writer.writeAttribute("table", tableName)
-            writer.writeEndElement()
-          }
-        }
-
-        allTablesInDatabase.foreach { tableName =>
-          if (targetTables.contains(tableName)) {
-            select(s"SELECT * FROM ${tableName}") { rs =>
-              writer.writeStartElement("insert")
-              writer.writeAttribute("table", tableName)
-              val rsMeta = rs.getMetaData
-              (1 to rsMeta.getColumnCount).foreach { i =>
-                val columnName = rsMeta.getColumnName(i)
-                val (columnType, columnValue) = if(rs.getObject(columnName) == null){
-                  (null, null)
-                } else {
-                  rsMeta.getColumnType(i) match {
-                    case Types.BOOLEAN | Types.BIT => ("boolean", rs.getBoolean(columnName))
-                    case Types.VARCHAR | Types.CLOB | Types.CHAR | Types.LONGVARCHAR => ("string", rs.getString(columnName))
-                    case Types.INTEGER => ("int", rs.getInt(columnName))
-                    case Types.TIMESTAMP => ("timestamp", dateFormat.format(rs.getTimestamp(columnName)))
-                  }
-                }
-                writer.writeStartElement("column")
-                writer.writeAttribute("name", columnName)
-                if(columnType != null){
-                  writer.writeAttribute("type", columnType)
-                }
-                if(columnValue != null){
-                  writer.writeCharacters(columnValue.toString)
-                }
-                writer.writeEndElement()
-              }
-              writer.writeEndElement()
-            }
-          }
-        }
-
-        writer.writeEndElement()
-        writer.writeEndDocument()
-      }
-
-      file
-    }
-
     def exportAsSQL(targetTables: Seq[String]): File = {
       val dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss")
       val file = File.createTempFile("gitbucket-export-", ".sql")

src/main/twirl/gitbucket/core/admin/data.scala.html

@@ -13,22 +13,12 @@
             </div>
           }
           <input type="submit" class="btn btn-success pull-right" value="Export">
-          <div class="radio pull-right" style="margin-right: 10px;">
-            <label>
-              <input type="radio" name="type" value="sql">SQL
-            </label>
-          </div>
-          <div class="radio pull-right" style="margin-right: 10px;">
-            <label>
-              <input type="radio" name="type" value="xml" checked>XML
-            </label>
-          </div>
         </form>
       </div>
     </div>
 
     <div class="panel panel-default">
-      <div class="panel-heading strong">Import (only XML)</div>
+      <div class="panel-heading strong">Import</div>
       <div class="panel-body">
         <form class="form form-horizontal" action="@context.path/upload/import" method="POST" enctype="multipart/form-data" id="import-form">
           <input type="file" name="file" id="file">
@@ -42,10 +32,10 @@
 $(function(){
   $('#import-form').submit(function(){
     if($('#file').val() == ''){
-      alert('Choose an import XML file.');
+      alert('Choose an import SQL file.');
       return false;
-    } else if(!$('#file').val().endsWith(".xml")){
+    } else if(!$('#file').val().endsWith(".sql")){
-      alert('Import is available for only the XML file.');
+      alert('Import is available for only the SQL file.');
       return false;
     }
     return confirm('All existing data is deleted before importing.\nAre you sure?');