From 5036b1a1aa000a96f16a21acebb080c905bf5b0e Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Sun, 10 Jan 2016 01:12:56 +0900 Subject: [PATCH 01/47] Replace migration system with Solidbase --- project/build.scala | 1 + src/main/resources/update/1_0.sql | 135 --------- src/main/resources/update/1_1.sql | 8 - src/main/resources/update/1_12.sql | 11 - src/main/resources/update/1_13.sql | 1 - src/main/resources/update/1_2.sql | 24 -- src/main/resources/update/1_3.sql | 8 - src/main/resources/update/1_4.sql | 24 -- src/main/resources/update/1_5.sql | 21 -- src/main/resources/update/1_6.sql | 8 - src/main/resources/update/1_7.sql | 5 - src/main/resources/update/1_8.sql | 1 - src/main/resources/update/2_3.sql | 6 - src/main/resources/update/2_7.sql | 18 -- src/main/resources/update/2_8.sql | 1 - src/main/resources/update/3_1.sql | 42 --- src/main/resources/update/3_9.sql | 55 ---- .../resources/update/gitbucket-core_4.0.sql | 18 ++ .../resources/update/gitbucket-core_4.0.xml | 266 ++++++++++++++++++ .../gitbucket/core/GitBucketCoreModule.scala | 11 + .../scala/gitbucket/core/plugin/Plugin.scala | 2 +- .../core/plugin/PluginRegistory.scala | 26 +- .../gitbucket/core/servlet/AutoUpdate.scala | 183 ------------ .../core/servlet/InitializeListener.scala | 15 +- .../scala/gitbucket/core/util/Version.scala | 67 ----- src/main/twirl/gitbucket/core/main.scala.html | 5 +- .../core/service/ServiceSpecBase.scala | 1 - 27 files changed, 311 insertions(+), 652 deletions(-) delete mode 100644 src/main/resources/update/1_0.sql delete mode 100644 src/main/resources/update/1_1.sql delete mode 100644 src/main/resources/update/1_12.sql delete mode 100644 src/main/resources/update/1_13.sql delete mode 100644 src/main/resources/update/1_2.sql delete mode 100644 src/main/resources/update/1_3.sql delete mode 100644 src/main/resources/update/1_4.sql delete mode 100644 src/main/resources/update/1_5.sql delete mode 100644 src/main/resources/update/1_6.sql delete mode 100644 src/main/resources/update/1_7.sql delete mode 100644 src/main/resources/update/1_8.sql delete mode 100644 src/main/resources/update/2_3.sql delete mode 100644 src/main/resources/update/2_7.sql delete mode 100644 src/main/resources/update/2_8.sql delete mode 100644 src/main/resources/update/3_1.sql delete mode 100644 src/main/resources/update/3_9.sql create mode 100644 src/main/resources/update/gitbucket-core_4.0.sql create mode 100644 src/main/resources/update/gitbucket-core_4.0.xml create mode 100644 src/main/scala/gitbucket/core/GitBucketCoreModule.scala delete mode 100644 src/main/scala/gitbucket/core/servlet/AutoUpdate.scala delete mode 100644 src/main/scala/gitbucket/core/util/Version.scala diff --git a/project/build.scala b/project/build.scala index ff54248fb..84822c985 100644 --- a/project/build.scala +++ b/project/build.scala @@ -52,6 +52,7 @@ object MyBuild extends Build { "io.github.gitbucket" %% "scalatra-forms" % "1.0.0", "commons-io" % "commons-io" % "2.4", "io.github.gitbucket" % "markedj" % "1.0.6-SNAPSHOT", + "io.github.gitbucket" % "solidbase" % "1.0.0-SNAPSHOT", "org.apache.commons" % "commons-compress" % "1.10", "org.apache.commons" % "commons-email" % "1.4", "org.apache.httpcomponents" % "httpclient" % "4.5.1", diff --git a/src/main/resources/update/1_0.sql b/src/main/resources/update/1_0.sql deleted file mode 100644 index 7d64af606..000000000 --- a/src/main/resources/update/1_0.sql +++ /dev/null @@ -1,135 +0,0 @@ -CREATE TABLE ACCOUNT( - USER_NAME VARCHAR(100) NOT NULL, - MAIL_ADDRESS VARCHAR(100) NOT NULL, - PASSWORD VARCHAR(40) NOT NULL, - ADMINISTRATOR BOOLEAN NOT NULL, - URL VARCHAR(200), - REGISTERED_DATE TIMESTAMP NOT NULL, - UPDATED_DATE TIMESTAMP NOT NULL, - LAST_LOGIN_DATE TIMESTAMP -); - -CREATE TABLE REPOSITORY( - USER_NAME VARCHAR(100) NOT NULL, - REPOSITORY_NAME VARCHAR(100) NOT NULL, - PRIVATE BOOLEAN NOT NULL, - DESCRIPTION TEXT, - DEFAULT_BRANCH VARCHAR(100), - REGISTERED_DATE TIMESTAMP NOT NULL, - UPDATED_DATE TIMESTAMP NOT NULL, - LAST_ACTIVITY_DATE TIMESTAMP NOT NULL -); - -CREATE TABLE COLLABORATOR( - USER_NAME VARCHAR(100) NOT NULL, - REPOSITORY_NAME VARCHAR(100) NOT NULL, - COLLABORATOR_NAME VARCHAR(100) NOT NULL -); - -CREATE TABLE ISSUE( - USER_NAME VARCHAR(100) NOT NULL, - REPOSITORY_NAME VARCHAR(100) NOT NULL, - ISSUE_ID INT NOT NULL, - OPENED_USER_NAME VARCHAR(100) NOT NULL, - MILESTONE_ID INT, - ASSIGNED_USER_NAME VARCHAR(100), - TITLE TEXT NOT NULL, - CONTENT TEXT, - CLOSED BOOLEAN NOT NULL, - REGISTERED_DATE TIMESTAMP NOT NULL, - UPDATED_DATE TIMESTAMP NOT NULL -); - -CREATE TABLE ISSUE_ID( - USER_NAME VARCHAR(100) NOT NULL, - REPOSITORY_NAME VARCHAR(100) NOT NULL, - ISSUE_ID INT NOT NULL -); - -CREATE TABLE ISSUE_COMMENT( - USER_NAME VARCHAR(100) NOT NULL, - REPOSITORY_NAME VARCHAR(100) NOT NULL, - ISSUE_ID INT NOT NULL, - COMMENT_ID INT AUTO_INCREMENT, - ACTION VARCHAR(10), - COMMENTED_USER_NAME VARCHAR(100) NOT NULL, - CONTENT TEXT NOT NULL, - REGISTERED_DATE TIMESTAMP NOT NULL, - UPDATED_DATE TIMESTAMP NOT NULL -); - -CREATE TABLE LABEL( - USER_NAME VARCHAR(100) NOT NULL, - REPOSITORY_NAME VARCHAR(100) NOT NULL, - LABEL_ID INT AUTO_INCREMENT, - LABEL_NAME VARCHAR(100) NOT NULL, - COLOR CHAR(6) NOT NULL -); - -CREATE TABLE ISSUE_LABEL( - USER_NAME VARCHAR(100) NOT NULL, - REPOSITORY_NAME VARCHAR(100) NOT NULL, - ISSUE_ID INT NOT NULL, - LABEL_ID INT NOT NULL -); - -CREATE TABLE MILESTONE( - USER_NAME VARCHAR(100) NOT NULL, - REPOSITORY_NAME VARCHAR(100) NOT NULL, - MILESTONE_ID INT AUTO_INCREMENT, - TITLE VARCHAR(100) NOT NULL, - DESCRIPTION TEXT, - DUE_DATE TIMESTAMP, - CLOSED_DATE TIMESTAMP -); - -ALTER TABLE ACCOUNT ADD CONSTRAINT IDX_ACCOUNT_PK PRIMARY KEY (USER_NAME); -ALTER TABLE ACCOUNT ADD CONSTRAINT IDX_ACCOUNT_1 UNIQUE (MAIL_ADDRESS); - -ALTER TABLE REPOSITORY ADD CONSTRAINT IDX_REPOSITORY_PK PRIMARY KEY (USER_NAME, REPOSITORY_NAME); -ALTER TABLE REPOSITORY ADD CONSTRAINT IDX_REPOSITORY_FK0 FOREIGN KEY (USER_NAME) REFERENCES ACCOUNT (USER_NAME); - -ALTER TABLE COLLABORATOR ADD CONSTRAINT IDX_COLLABORATOR_PK PRIMARY KEY (USER_NAME, REPOSITORY_NAME); -ALTER TABLE COLLABORATOR ADD CONSTRAINT IDX_COLLABORATOR_FK0 FOREIGN KEY (USER_NAME, REPOSITORY_NAME) REFERENCES REPOSITORY (USER_NAME, REPOSITORY_NAME); -ALTER TABLE COLLABORATOR ADD CONSTRAINT IDX_COLLABORATOR_FK1 FOREIGN KEY (COLLABORATOR_NAME) REFERENCES ACCOUNT (USER_NAME); - -ALTER TABLE ISSUE ADD CONSTRAINT IDX_ISSUE_PK PRIMARY KEY (ISSUE_ID, USER_NAME, REPOSITORY_NAME); -ALTER TABLE ISSUE ADD CONSTRAINT IDX_ISSUE_FK0 FOREIGN KEY (USER_NAME, REPOSITORY_NAME) REFERENCES REPOSITORY (USER_NAME, REPOSITORY_NAME); -ALTER TABLE ISSUE ADD CONSTRAINT IDX_ISSUE_FK1 FOREIGN KEY (OPENED_USER_NAME) REFERENCES ACCOUNT (USER_NAME); -ALTER TABLE ISSUE ADD CONSTRAINT IDX_ISSUE_FK2 FOREIGN KEY (MILESTONE_ID) REFERENCES MILESTONE (MILESTONE_ID); - -ALTER TABLE ISSUE_ID ADD CONSTRAINT IDX_ISSUE_ID_PK PRIMARY KEY (USER_NAME, REPOSITORY_NAME); -ALTER TABLE ISSUE_ID ADD CONSTRAINT IDX_ISSUE_ID_FK1 FOREIGN KEY (USER_NAME, REPOSITORY_NAME) REFERENCES REPOSITORY (USER_NAME, REPOSITORY_NAME); - -ALTER TABLE ISSUE_COMMENT ADD CONSTRAINT IDX_ISSUE_COMMENT_PK PRIMARY KEY (COMMENT_ID); -ALTER TABLE ISSUE_COMMENT ADD CONSTRAINT IDX_ISSUE_COMMENT_1 UNIQUE (USER_NAME, REPOSITORY_NAME, ISSUE_ID, COMMENT_ID); -ALTER TABLE ISSUE_COMMENT ADD CONSTRAINT IDX_ISSUE_COMMENT_FK0 FOREIGN KEY (USER_NAME, REPOSITORY_NAME, ISSUE_ID) REFERENCES ISSUE (USER_NAME, REPOSITORY_NAME, ISSUE_ID); - -ALTER TABLE LABEL ADD CONSTRAINT IDX_LABEL_PK PRIMARY KEY (USER_NAME, REPOSITORY_NAME, LABEL_ID); -ALTER TABLE LABEL ADD CONSTRAINT IDX_LABEL_FK0 FOREIGN KEY (USER_NAME, REPOSITORY_NAME) REFERENCES REPOSITORY (USER_NAME, REPOSITORY_NAME); - -ALTER TABLE ISSUE_LABEL ADD CONSTRAINT IDX_ISSUE_LABEL_PK PRIMARY KEY (USER_NAME, REPOSITORY_NAME, ISSUE_ID, LABEL_ID); -ALTER TABLE ISSUE_LABEL ADD CONSTRAINT IDX_ISSUE_LABEL_FK0 FOREIGN KEY (USER_NAME, REPOSITORY_NAME, ISSUE_ID) REFERENCES ISSUE (USER_NAME, REPOSITORY_NAME, ISSUE_ID); - -ALTER TABLE MILESTONE ADD CONSTRAINT IDX_MILESTONE_PK PRIMARY KEY (USER_NAME, REPOSITORY_NAME, MILESTONE_ID); -ALTER TABLE MILESTONE ADD CONSTRAINT IDX_MILESTONE_FK0 FOREIGN KEY (USER_NAME, REPOSITORY_NAME) REFERENCES REPOSITORY (USER_NAME, REPOSITORY_NAME); - -INSERT INTO ACCOUNT ( - USER_NAME, - MAIL_ADDRESS, - PASSWORD, - ADMINISTRATOR, - URL, - REGISTERED_DATE, - UPDATED_DATE, - LAST_LOGIN_DATE -) VALUES ( - 'root', - 'root@localhost', - 'dc76e9f0c0006e8f919e0c515c66dbba3982f785', - true, - 'https://github.com/gitbucket/gitbucket', - SYSDATE, - SYSDATE, - NULL -); diff --git a/src/main/resources/update/1_1.sql b/src/main/resources/update/1_1.sql deleted file mode 100644 index 9cfd50a4c..000000000 --- a/src/main/resources/update/1_1.sql +++ /dev/null @@ -1,8 +0,0 @@ --- Fix COLLABORATOR constraints -ALTER TABLE COLLABORATOR DROP CONSTRAINT IDX_COLLABORATOR_FK1 IF EXISTS; -ALTER TABLE COLLABORATOR DROP CONSTRAINT IDX_COLLABORATOR_FK0 IF EXISTS; -ALTER TABLE COLLABORATOR DROP CONSTRAINT IDX_COLLABORATOR_PK IF EXISTS; - -ALTER TABLE COLLABORATOR ADD CONSTRAINT IDX_COLLABORATOR_PK PRIMARY KEY (USER_NAME, REPOSITORY_NAME, COLLABORATOR_NAME); -ALTER TABLE COLLABORATOR ADD CONSTRAINT IDX_COLLABORATOR_FK0 FOREIGN KEY (USER_NAME, REPOSITORY_NAME) REFERENCES REPOSITORY (USER_NAME, REPOSITORY_NAME); -ALTER TABLE COLLABORATOR ADD CONSTRAINT IDX_COLLABORATOR_FK1 FOREIGN KEY (COLLABORATOR_NAME) REFERENCES ACCOUNT (USER_NAME); diff --git a/src/main/resources/update/1_12.sql b/src/main/resources/update/1_12.sql deleted file mode 100644 index f8658a24c..000000000 --- a/src/main/resources/update/1_12.sql +++ /dev/null @@ -1,11 +0,0 @@ -ALTER TABLE GROUP_MEMBER ADD COLUMN MANAGER BOOLEAN DEFAULT FALSE; - -CREATE TABLE SSH_KEY ( - USER_NAME VARCHAR(100) NOT NULL, - SSH_KEY_ID INT AUTO_INCREMENT, - TITLE VARCHAR(100) NOT NULL, - PUBLIC_KEY TEXT NOT NULL -); - -ALTER TABLE SSH_KEY ADD CONSTRAINT IDX_SSH_KEY_PK PRIMARY KEY (USER_NAME, SSH_KEY_ID); -ALTER TABLE SSH_KEY ADD CONSTRAINT IDX_SSH_KEY_FK0 FOREIGN KEY (USER_NAME) REFERENCES ACCOUNT (USER_NAME); diff --git a/src/main/resources/update/1_13.sql b/src/main/resources/update/1_13.sql deleted file mode 100644 index ed26f65ac..000000000 --- a/src/main/resources/update/1_13.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE COMMIT_LOG; \ No newline at end of file diff --git a/src/main/resources/update/1_2.sql b/src/main/resources/update/1_2.sql deleted file mode 100644 index d2765bc22..000000000 --- a/src/main/resources/update/1_2.sql +++ /dev/null @@ -1,24 +0,0 @@ -CREATE TABLE ACTIVITY( - ACTIVITY_ID INT AUTO_INCREMENT, - USER_NAME VARCHAR(100) NOT NULL, - REPOSITORY_NAME VARCHAR(100) NOT NULL, - ACTIVITY_USER_NAME VARCHAR(100) NOT NULL, - ACTIVITY_TYPE VARCHAR(100) NOT NULL, - MESSAGE TEXT NOT NULL, - ADDITIONAL_INFO TEXT, - ACTIVITY_DATE TIMESTAMP NOT NULL -); - -CREATE TABLE COMMIT_LOG ( - USER_NAME VARCHAR(100) NOT NULL, - REPOSITORY_NAME VARCHAR(100) NOT NULL, - COMMIT_ID VARCHAR(40) NOT NULL -); - -ALTER TABLE ACTIVITY ADD CONSTRAINT IDX_ACTIVITY_PK PRIMARY KEY (ACTIVITY_ID); -ALTER TABLE ACTIVITY ADD CONSTRAINT IDX_ACTIVITY_FK0 FOREIGN KEY (USER_NAME, REPOSITORY_NAME) REFERENCES REPOSITORY (USER_NAME, REPOSITORY_NAME); -ALTER TABLE ACTIVITY ADD CONSTRAINT IDX_ACTIVITY_FK1 FOREIGN KEY (ACTIVITY_USER_NAME) REFERENCES ACCOUNT (USER_NAME); - -ALTER TABLE COMMIT_LOG ADD CONSTRAINT IDX_COMMIT_LOG_PK PRIMARY KEY (USER_NAME, REPOSITORY_NAME, COMMIT_ID); -ALTER TABLE COMMIT_LOG ADD CONSTRAINT IDX_COMMIT_LOG_FK0 FOREIGN KEY (USER_NAME, REPOSITORY_NAME) REFERENCES REPOSITORY (USER_NAME, REPOSITORY_NAME); - diff --git a/src/main/resources/update/1_3.sql b/src/main/resources/update/1_3.sql deleted file mode 100644 index 59ab00941..000000000 --- a/src/main/resources/update/1_3.sql +++ /dev/null @@ -1,8 +0,0 @@ -ALTER TABLE ACCOUNT ADD COLUMN IMAGE VARCHAR(100); - -UPDATE ISSUE_COMMENT SET ACTION = 'comment' WHERE ACTION IS NULL; - -ALTER TABLE ISSUE_COMMENT ALTER COLUMN ACTION VARCHAR(20) NOT NULL; - -UPDATE ISSUE_COMMENT SET ACTION = 'close_comment' WHERE ACTION = 'close'; -UPDATE ISSUE_COMMENT SET ACTION = 'reopen_comment' WHERE ACTION = 'reopen'; diff --git a/src/main/resources/update/1_4.sql b/src/main/resources/update/1_4.sql deleted file mode 100644 index 2d3c49264..000000000 --- a/src/main/resources/update/1_4.sql +++ /dev/null @@ -1,24 +0,0 @@ -CREATE TABLE GROUP_MEMBER( - GROUP_NAME VARCHAR(100) NOT NULL, - USER_NAME VARCHAR(100) NOT NULL -); - -ALTER TABLE GROUP_MEMBER ADD CONSTRAINT IDX_GROUP_MEMBER_PK PRIMARY KEY (GROUP_NAME, USER_NAME); -ALTER TABLE GROUP_MEMBER ADD CONSTRAINT IDX_GROUP_MEMBER_FK0 FOREIGN KEY (GROUP_NAME) REFERENCES ACCOUNT (USER_NAME); -ALTER TABLE GROUP_MEMBER ADD CONSTRAINT IDX_GROUP_MEMBER_FK1 FOREIGN KEY (USER_NAME) REFERENCES ACCOUNT (USER_NAME); - -ALTER TABLE ACCOUNT ADD COLUMN GROUP_ACCOUNT BOOLEAN NOT NULL DEFAULT FALSE; - -CREATE OR REPLACE VIEW ISSUE_OUTLINE_VIEW AS - SELECT - A.USER_NAME, - A.REPOSITORY_NAME, - A.ISSUE_ID, - NVL(B.COMMENT_COUNT, 0) AS COMMENT_COUNT - FROM ISSUE A - LEFT OUTER JOIN ( - SELECT USER_NAME, REPOSITORY_NAME, ISSUE_ID, COUNT(COMMENT_ID) AS COMMENT_COUNT FROM ISSUE_COMMENT - WHERE ACTION IN ('comment', 'close_comment', 'reopen_comment') - GROUP BY USER_NAME, REPOSITORY_NAME, ISSUE_ID - ) B - ON (A.USER_NAME = B.USER_NAME AND A.REPOSITORY_NAME = B.REPOSITORY_NAME AND A.ISSUE_ID = B.ISSUE_ID); diff --git a/src/main/resources/update/1_5.sql b/src/main/resources/update/1_5.sql deleted file mode 100644 index 03fc1bf90..000000000 --- a/src/main/resources/update/1_5.sql +++ /dev/null @@ -1,21 +0,0 @@ -ALTER TABLE REPOSITORY ADD COLUMN ORIGIN_USER_NAME VARCHAR(100); -ALTER TABLE REPOSITORY ADD COLUMN ORIGIN_REPOSITORY_NAME VARCHAR(100); -ALTER TABLE REPOSITORY ADD COLUMN PARENT_USER_NAME VARCHAR(100); -ALTER TABLE REPOSITORY ADD COLUMN PARENT_REPOSITORY_NAME VARCHAR(100); - -CREATE TABLE PULL_REQUEST( - USER_NAME VARCHAR(100) NOT NULL, - REPOSITORY_NAME VARCHAR(100) NOT NULL, - ISSUE_ID INT NOT NULL, - BRANCH VARCHAR(100) NOT NULL, - REQUEST_USER_NAME VARCHAR(100) NOT NULL, - REQUEST_REPOSITORY_NAME VARCHAR(100) NOT NULL, - REQUEST_BRANCH VARCHAR(100) NOT NULL, - COMMIT_ID_FROM VARCHAR(40) NOT NULL, - COMMIT_ID_TO VARCHAR(40) NOT NULL -); - -ALTER TABLE PULL_REQUEST ADD CONSTRAINT IDX_PULL_REQUEST_PK PRIMARY KEY (USER_NAME, REPOSITORY_NAME, ISSUE_ID); -ALTER TABLE PULL_REQUEST ADD CONSTRAINT IDX_PULL_REQUEST_FK0 FOREIGN KEY (USER_NAME, REPOSITORY_NAME, ISSUE_ID) REFERENCES ISSUE (USER_NAME, REPOSITORY_NAME, ISSUE_ID); - -ALTER TABLE ISSUE ADD COLUMN PULL_REQUEST BOOLEAN NOT NULL DEFAULT FALSE; diff --git a/src/main/resources/update/1_6.sql b/src/main/resources/update/1_6.sql deleted file mode 100644 index 43eb92d85..000000000 --- a/src/main/resources/update/1_6.sql +++ /dev/null @@ -1,8 +0,0 @@ -CREATE TABLE WEB_HOOK ( - USER_NAME VARCHAR(100) NOT NULL, - REPOSITORY_NAME VARCHAR(100) NOT NULL, - URL VARCHAR(200) NOT NULL -); - -ALTER TABLE WEB_HOOK ADD CONSTRAINT IDX_WEB_HOOK_PK PRIMARY KEY (USER_NAME, REPOSITORY_NAME, URL); -ALTER TABLE WEB_HOOK ADD CONSTRAINT IDX_WEB_HOOK_FK0 FOREIGN KEY (USER_NAME, REPOSITORY_NAME) REFERENCES REPOSITORY (USER_NAME, REPOSITORY_NAME); diff --git a/src/main/resources/update/1_7.sql b/src/main/resources/update/1_7.sql deleted file mode 100644 index 9005ff9c2..000000000 --- a/src/main/resources/update/1_7.sql +++ /dev/null @@ -1,5 +0,0 @@ -ALTER TABLE ACCOUNT ADD COLUMN FULL_NAME VARCHAR(100); - -UPDATE ACCOUNT SET FULL_NAME = USER_NAME WHERE FULL_NAME IS NULL; - -ALTER TABLE ACCOUNT ALTER COLUMN FULL_NAME SET NOT NULL; diff --git a/src/main/resources/update/1_8.sql b/src/main/resources/update/1_8.sql deleted file mode 100644 index 8b50ddf5a..000000000 --- a/src/main/resources/update/1_8.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE ACCOUNT ADD COLUMN REMOVED BOOLEAN DEFAULT FALSE; \ No newline at end of file diff --git a/src/main/resources/update/2_3.sql b/src/main/resources/update/2_3.sql deleted file mode 100644 index 76200926b..000000000 --- a/src/main/resources/update/2_3.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE TABLE PLUGIN ( - PLUGIN_ID VARCHAR(100) NOT NULL, - VERSION VARCHAR(100) NOT NULL -); - -ALTER TABLE PLUGIN ADD CONSTRAINT IDX_PLUGIN_PK PRIMARY KEY (PLUGIN_ID); diff --git a/src/main/resources/update/2_7.sql b/src/main/resources/update/2_7.sql deleted file mode 100644 index 6fa0684d4..000000000 --- a/src/main/resources/update/2_7.sql +++ /dev/null @@ -1,18 +0,0 @@ -CREATE TABLE COMMIT_COMMENT ( - USER_NAME VARCHAR(100) NOT NULL, - REPOSITORY_NAME VARCHAR(100) NOT NULL, - COMMIT_ID VARCHAR(100) NOT NULL, - COMMENT_ID INT AUTO_INCREMENT, - COMMENTED_USER_NAME VARCHAR(100) NOT NULL, - CONTENT TEXT NOT NULL, - FILE_NAME NVARCHAR(100), - OLD_LINE_NUMBER INT, - NEW_LINE_NUMBER INT, - REGISTERED_DATE TIMESTAMP NOT NULL, - UPDATED_DATE TIMESTAMP NOT NULL, - PULL_REQUEST BOOLEAN NOT NULL -); - -ALTER TABLE COMMIT_COMMENT ADD CONSTRAINT IDX_COMMIT_COMMENT_PK PRIMARY KEY (COMMENT_ID); -ALTER TABLE COMMIT_COMMENT ADD CONSTRAINT IDX_COMMIT_COMMENT_FK0 FOREIGN KEY (USER_NAME, REPOSITORY_NAME) REFERENCES REPOSITORY (USER_NAME, REPOSITORY_NAME); -ALTER TABLE COMMIT_COMMENT ADD CONSTRAINT IDX_COMMIT_COMMENT_1 UNIQUE (USER_NAME, REPOSITORY_NAME, COMMIT_ID, COMMENT_ID); diff --git a/src/main/resources/update/2_8.sql b/src/main/resources/update/2_8.sql deleted file mode 100644 index 38c95d371..000000000 --- a/src/main/resources/update/2_8.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE COMMIT_COMMENT ALTER COLUMN FILE_NAME NVARCHAR(260); diff --git a/src/main/resources/update/3_1.sql b/src/main/resources/update/3_1.sql deleted file mode 100644 index 3ddc48a45..000000000 --- a/src/main/resources/update/3_1.sql +++ /dev/null @@ -1,42 +0,0 @@ -DROP TABLE IF EXISTS ACCESS_TOKEN; - -CREATE TABLE ACCESS_TOKEN ( - ACCESS_TOKEN_ID INT NOT NULL AUTO_INCREMENT, - TOKEN_HASH VARCHAR(40) NOT NULL, - USER_NAME VARCHAR(100) NOT NULL, - NOTE TEXT NOT NULL -); - -ALTER TABLE ACCESS_TOKEN ADD CONSTRAINT IDX_ACCESS_TOKEN_PK PRIMARY KEY (ACCESS_TOKEN_ID); -ALTER TABLE ACCESS_TOKEN ADD CONSTRAINT IDX_ACCESS_TOKEN_FK0 FOREIGN KEY (USER_NAME) REFERENCES ACCOUNT (USER_NAME) - ON DELETE CASCADE ON UPDATE CASCADE; -ALTER TABLE ACCESS_TOKEN ADD CONSTRAINT IDX_ACCESS_TOKEN_TOKEN_HASH UNIQUE(TOKEN_HASH); - - -DROP TABLE IF EXISTS COMMIT_STATUS; -CREATE TABLE COMMIT_STATUS( - COMMIT_STATUS_ID INT AUTO_INCREMENT, - USER_NAME VARCHAR(100) NOT NULL, - REPOSITORY_NAME VARCHAR(100) NOT NULL, - COMMIT_ID VARCHAR(40) NOT NULL, - CONTEXT VARCHAR(255) NOT NULL, -- context is too long (maximum is 255 characters) - STATE VARCHAR(10) NOT NULL, -- pending, success, error, or failure - TARGET_URL VARCHAR(200), - DESCRIPTION TEXT, - CREATOR VARCHAR(100) NOT NULL, - REGISTERED_DATE TIMESTAMP NOT NULL, -- CREATED_AT - UPDATED_DATE TIMESTAMP NOT NULL -- UPDATED_AT -); -ALTER TABLE COMMIT_STATUS ADD CONSTRAINT IDX_COMMIT_STATUS_PK PRIMARY KEY (COMMIT_STATUS_ID); -ALTER TABLE COMMIT_STATUS ADD CONSTRAINT IDX_COMMIT_STATUS_1 - UNIQUE (USER_NAME, REPOSITORY_NAME, COMMIT_ID, CONTEXT); -ALTER TABLE COMMIT_STATUS ADD CONSTRAINT IDX_COMMIT_STATUS_FK1 - FOREIGN KEY (USER_NAME, REPOSITORY_NAME) - REFERENCES REPOSITORY (USER_NAME, REPOSITORY_NAME) - ON DELETE CASCADE ON UPDATE CASCADE; -ALTER TABLE COMMIT_STATUS ADD CONSTRAINT IDX_COMMIT_STATUS_FK2 - FOREIGN KEY (USER_NAME) REFERENCES ACCOUNT (USER_NAME) - ON DELETE CASCADE ON UPDATE CASCADE; -ALTER TABLE COMMIT_STATUS ADD CONSTRAINT IDX_COMMIT_STATUS_FK3 - FOREIGN KEY (CREATOR) REFERENCES ACCOUNT (USER_NAME) - ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/src/main/resources/update/3_9.sql b/src/main/resources/update/3_9.sql deleted file mode 100644 index b2f5c56c8..000000000 --- a/src/main/resources/update/3_9.sql +++ /dev/null @@ -1,55 +0,0 @@ -DROP TABLE IF EXISTS WEB_HOOK_EVENT; - -CREATE TABLE WEB_HOOK_EVENT( - USER_NAME VARCHAR(100) NOT NULL, - REPOSITORY_NAME VARCHAR(100) NOT NULL, - URL VARCHAR(200) NOT NULL, - EVENT VARCHAR(30) NOT NULL -); - -ALTER TABLE WEB_HOOK_EVENT ADD CONSTRAINT IDX_WEB_HOOK_EVENT_PK PRIMARY KEY (USER_NAME, REPOSITORY_NAME, URL, EVENT); -ALTER TABLE WEB_HOOK_EVENT ADD CONSTRAINT IDX_WEB_HOOK_EVENT_FK0 FOREIGN KEY (USER_NAME, REPOSITORY_NAME, URL) REFERENCES WEB_HOOK (USER_NAME, REPOSITORY_NAME, URL) - ON DELETE CASCADE ON UPDATE CASCADE; - -CREATE TEMPORARY TABLE TMP_EVENTS (EVENT VARCHAR(30)); - -INSERT INTO TMP_EVENTS VALUES ('push'),('issue_comment'),('issues'),('pull_request'); - -INSERT INTO WEB_HOOK_EVENT (USER_NAME, REPOSITORY_NAME, URL, EVENT) - SELECT USER_NAME, REPOSITORY_NAME, URL, EVENT - FROM WEB_HOOK, TMP_EVENTS; - -DROP TABLE TMP_EVENTS; - -ALTER TABLE COMMIT_COMMENT ADD COLUMN ISSUE_ID INT; - -CREATE OR REPLACE VIEW ISSUE_OUTLINE_VIEW AS - SELECT - A.USER_NAME, - A.REPOSITORY_NAME, - A.ISSUE_ID, - NVL(B.COMMENT_COUNT, 0) + NVL(C.COMMENT_COUNT, 0) AS COMMENT_COUNT - FROM ISSUE A - LEFT OUTER JOIN ( - SELECT USER_NAME, REPOSITORY_NAME, ISSUE_ID, COUNT(COMMENT_ID) AS COMMENT_COUNT FROM ISSUE_COMMENT - WHERE ACTION IN ('comment', 'close_comment', 'reopen_comment') - GROUP BY USER_NAME, REPOSITORY_NAME, ISSUE_ID - ) B - ON (A.USER_NAME = B.USER_NAME AND A.REPOSITORY_NAME = B.REPOSITORY_NAME AND A.ISSUE_ID = B.ISSUE_ID) - LEFT OUTER JOIN ( - SELECT USER_NAME, REPOSITORY_NAME, ISSUE_ID, COUNT(COMMENT_ID) AS COMMENT_COUNT FROM COMMIT_COMMENT - GROUP BY USER_NAME, REPOSITORY_NAME, ISSUE_ID - ) C - ON (A.USER_NAME = C.USER_NAME AND A.REPOSITORY_NAME = C.REPOSITORY_NAME AND A.ISSUE_ID = C.ISSUE_ID); - - -UPDATE COMMIT_COMMENT C SET (ISSUE_ID) = ( - SELECT MAX(P.ISSUE_ID) - FROM PULL_REQUEST P - WHERE - C.USER_NAME = P.USER_NAME AND - C.REPOSITORY_NAME = P.REPOSITORY_NAME AND - C.COMMIT_ID = P.COMMIT_ID_TO -); - -ALTER TABLE COMMIT_COMMENT DROP COLUMN PULL_REQUEST; \ No newline at end of file diff --git a/src/main/resources/update/gitbucket-core_4.0.sql b/src/main/resources/update/gitbucket-core_4.0.sql new file mode 100644 index 000000000..775a77c1c --- /dev/null +++ b/src/main/resources/update/gitbucket-core_4.0.sql @@ -0,0 +1,18 @@ +CREATE OR REPLACE VIEW ISSUE_OUTLINE_VIEW AS + SELECT + A.USER_NAME, + A.REPOSITORY_NAME, + A.ISSUE_ID, + NVL(B.COMMENT_COUNT, 0) + NVL(C.COMMENT_COUNT, 0) AS COMMENT_COUNT + FROM ISSUE A + LEFT OUTER JOIN ( + SELECT USER_NAME, REPOSITORY_NAME, ISSUE_ID, COUNT(COMMENT_ID) AS COMMENT_COUNT FROM ISSUE_COMMENT + WHERE ACTION IN ('comment', 'close_comment', 'reopen_comment') + GROUP BY USER_NAME, REPOSITORY_NAME, ISSUE_ID + ) B + ON (A.USER_NAME = B.USER_NAME AND A.REPOSITORY_NAME = B.REPOSITORY_NAME AND A.ISSUE_ID = B.ISSUE_ID) + LEFT OUTER JOIN ( + SELECT USER_NAME, REPOSITORY_NAME, ISSUE_ID, COUNT(COMMENT_ID) AS COMMENT_COUNT FROM COMMIT_COMMENT + GROUP BY USER_NAME, REPOSITORY_NAME, ISSUE_ID + ) C + ON (A.USER_NAME = C.USER_NAME AND A.REPOSITORY_NAME = C.REPOSITORY_NAME AND A.ISSUE_ID = C.ISSUE_ID); diff --git a/src/main/resources/update/gitbucket-core_4.0.xml b/src/main/resources/update/gitbucket-core_4.0.xml new file mode 100644 index 000000000..36ab26726 --- /dev/null +++ b/src/main/resources/update/gitbucket-core_4.0.xml @@ -0,0 +1,266 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/main/scala/gitbucket/core/GitBucketCoreModule.scala b/src/main/scala/gitbucket/core/GitBucketCoreModule.scala new file mode 100644 index 000000000..3b85c726d --- /dev/null +++ b/src/main/scala/gitbucket/core/GitBucketCoreModule.scala @@ -0,0 +1,11 @@ +package gitbucket.core + +import io.github.gitbucket.solidbase.migration.{SqlMigration, LiquibaseMigration} +import io.github.gitbucket.solidbase.model.{Version, Module} + +object GitBucketCoreModule extends Module("gitbucket-core", + new Version("4.0.0", + new LiquibaseMigration("update/gitbucket-core_4.0.xml"), + new SqlMigration("update/gitbucket-core_4.0.sql") + ) +) diff --git a/src/main/scala/gitbucket/core/plugin/Plugin.scala b/src/main/scala/gitbucket/core/plugin/Plugin.scala index 975791d0c..c092574a9 100644 --- a/src/main/scala/gitbucket/core/plugin/Plugin.scala +++ b/src/main/scala/gitbucket/core/plugin/Plugin.scala @@ -4,7 +4,7 @@ import javax.servlet.ServletContext import gitbucket.core.controller.ControllerBase import gitbucket.core.service.SystemSettingsService.SystemSettings import gitbucket.core.util.ControlUtil._ -import gitbucket.core.util.Version +import io.github.gitbucket.solidbase.model.Version /** * Trait for define plugin interface. diff --git a/src/main/scala/gitbucket/core/plugin/PluginRegistory.scala b/src/main/scala/gitbucket/core/plugin/PluginRegistory.scala index b0323699f..b609f4d5d 100644 --- a/src/main/scala/gitbucket/core/plugin/PluginRegistory.scala +++ b/src/main/scala/gitbucket/core/plugin/PluginRegistory.scala @@ -10,8 +10,9 @@ import gitbucket.core.service.RepositoryService.RepositoryInfo import gitbucket.core.service.SystemSettingsService.SystemSettings import gitbucket.core.util.ControlUtil._ import gitbucket.core.util.Directory._ -import gitbucket.core.util.JDBCUtil._ -import gitbucket.core.util.{Version, Versions} +import io.github.gitbucket.solidbase.Solidbase +import io.github.gitbucket.solidbase.model.Module +import liquibase.database.core.H2Database import org.apache.commons.codec.binary.{Base64, StringUtils} import org.slf4j.LoggerFactory @@ -140,30 +141,15 @@ object PluginRegistry { val plugin = classLoader.loadClass("Plugin").newInstance().asInstanceOf[Plugin] // Migration - val headVersion = plugin.versions.head - val currentVersion = conn.find("SELECT * FROM PLUGIN WHERE PLUGIN_ID = ?", plugin.pluginId)(_.getString("VERSION")) match { - case Some(x) => { - val dim = x.split("\\.") - Version(dim(0).toInt, dim(1).toInt) - } - case None => Version(0, 0) - } - - Versions.update(conn, headVersion, currentVersion, plugin.versions, new URLClassLoader(Array(pluginJar.toURI.toURL))){ conn => - currentVersion.versionString match { - case "0.0" => - conn.update("INSERT INTO PLUGIN (PLUGIN_ID, VERSION) VALUES (?, ?)", plugin.pluginId, headVersion.versionString) - case _ => - conn.update("UPDATE PLUGIN SET VERSION = ? WHERE PLUGIN_ID = ?", headVersion.versionString, plugin.pluginId) - } - } + val solidbase = new Solidbase() + solidbase.migrate(conn, Thread.currentThread.getContextClassLoader, new H2Database(), new Module(plugin.pluginId, plugin.versions: _*)) // Initialize plugin.initialize(instance, context, settings) instance.addPlugin(PluginInfo( pluginId = plugin.pluginId, pluginName = plugin.pluginName, - version = plugin.versions.head.versionString, + version = plugin.versions.head.getVersion, description = plugin.description, pluginClass = plugin )) diff --git a/src/main/scala/gitbucket/core/servlet/AutoUpdate.scala b/src/main/scala/gitbucket/core/servlet/AutoUpdate.scala deleted file mode 100644 index c8987fb38..000000000 --- a/src/main/scala/gitbucket/core/servlet/AutoUpdate.scala +++ /dev/null @@ -1,183 +0,0 @@ -package gitbucket.core.servlet - -import java.io.File -import java.sql.{DriverManager, Connection} -import gitbucket.core.plugin.PluginRegistry -import gitbucket.core.service.SystemSettingsService -import gitbucket.core.util._ -import org.apache.commons.io.FileUtils -import javax.servlet.{ServletContextListener, ServletContextEvent} -import org.slf4j.LoggerFactory -import Directory._ -import ControlUtil._ -import JDBCUtil._ -import org.eclipse.jgit.api.Git -import gitbucket.core.util.Versions -import gitbucket.core.util.Directory - -object AutoUpdate { - - /** - * The history of versions. A head of this sequence is the current BitBucket version. - */ - val versions = Seq( - new Version(3, 10), - new Version(3, 9), - new Version(3, 8), - new Version(3, 7) with SystemSettingsService { - override def update(conn: Connection, cl: ClassLoader): Unit = { - super.update(conn, cl) - val settings = loadSystemSettings() - if(settings.notification){ - saveSystemSettings(settings.copy(useSMTP = true)) - } - } - }, - new Version(3, 6), - new Version(3, 5), - new Version(3, 4), - new Version(3, 3), - new Version(3, 2), - new Version(3, 1), - new Version(3, 0), - new Version(2, 8), - new Version(2, 7) { - override def update(conn: Connection, cl: ClassLoader): Unit = { - super.update(conn, cl) - conn.select("SELECT * FROM REPOSITORY"){ rs => - // Rename attached files directory from /issues to /comments - val userName = rs.getString("USER_NAME") - val repoName = rs.getString("REPOSITORY_NAME") - defining(Directory.getAttachedDir(userName, repoName)){ newDir => - val oldDir = new File(newDir.getParentFile, "issues") - if(oldDir.exists && oldDir.isDirectory){ - oldDir.renameTo(newDir) - } - } - // Update ORIGIN_USER_NAME and ORIGIN_REPOSITORY_NAME if it does not exist - val originalUserName = rs.getString("ORIGIN_USER_NAME") - val originalRepoName = rs.getString("ORIGIN_REPOSITORY_NAME") - if(originalUserName != null && originalRepoName != null){ - if(conn.selectInt("SELECT COUNT(*) FROM REPOSITORY WHERE USER_NAME = ? AND REPOSITORY_NAME = ?", - originalUserName, originalRepoName) == 0){ - conn.update("UPDATE REPOSITORY SET ORIGIN_USER_NAME = NULL, ORIGIN_REPOSITORY_NAME = NULL " + - "WHERE USER_NAME = ? AND REPOSITORY_NAME = ?", userName, repoName) - } - } - // Update PARENT_USER_NAME and PARENT_REPOSITORY_NAME if it does not exist - val parentUserName = rs.getString("PARENT_USER_NAME") - val parentRepoName = rs.getString("PARENT_REPOSITORY_NAME") - if(parentUserName != null && parentRepoName != null){ - if(conn.selectInt("SELECT COUNT(*) FROM REPOSITORY WHERE USER_NAME = ? AND REPOSITORY_NAME = ?", - parentUserName, parentRepoName) == 0){ - conn.update("UPDATE REPOSITORY SET PARENT_USER_NAME = NULL, PARENT_REPOSITORY_NAME = NULL " + - "WHERE USER_NAME = ? AND REPOSITORY_NAME = ?", userName, repoName) - } - } - } - } - }, - new Version(2, 6), - new Version(2, 5), - new Version(2, 4), - new Version(2, 3) { - override def update(conn: Connection, cl: ClassLoader): Unit = { - super.update(conn, cl) - conn.select("SELECT ACTIVITY_ID, ADDITIONAL_INFO FROM ACTIVITY WHERE ACTIVITY_TYPE='push'"){ rs => - val curInfo = rs.getString("ADDITIONAL_INFO") - val newInfo = curInfo.split("\n").filter(_ matches "^[0-9a-z]{40}:.*").mkString("\n") - if (curInfo != newInfo) { - conn.update("UPDATE ACTIVITY SET ADDITIONAL_INFO = ? WHERE ACTIVITY_ID = ?", newInfo, rs.getInt("ACTIVITY_ID")) - } - } - ignore { - FileUtils.deleteDirectory(Directory.getPluginCacheDir()) - //FileUtils.deleteDirectory(new File(Directory.PluginHome)) - } - } - }, - new Version(2, 2), - new Version(2, 1), - new Version(2, 0){ - override def update(conn: Connection, cl: ClassLoader): Unit = { - import eu.medsea.mimeutil.{MimeUtil2, MimeType} - - val mimeUtil = new MimeUtil2() - mimeUtil.registerMimeDetector("eu.medsea.mimeutil.detector.MagicMimeMimeDetector") - - super.update(conn, cl) - conn.select("SELECT USER_NAME, REPOSITORY_NAME FROM REPOSITORY"){ rs => - defining(Directory.getAttachedDir(rs.getString("USER_NAME"), rs.getString("REPOSITORY_NAME"))){ dir => - if(dir.exists && dir.isDirectory){ - dir.listFiles.foreach { file => - if(file.getName.indexOf('.') < 0){ - val mimeType = MimeUtil2.getMostSpecificMimeType(mimeUtil.getMimeTypes(file, new MimeType("application/octet-stream"))).toString - if(mimeType.startsWith("image/")){ - file.renameTo(new File(file.getParent, file.getName + "." + mimeType.split("/")(1))) - } - } - } - } - } - } - } - }, - Version(1, 13), - Version(1, 12), - Version(1, 11), - Version(1, 10), - Version(1, 9), - Version(1, 8), - Version(1, 7), - Version(1, 6), - Version(1, 5), - Version(1, 4), - new Version(1, 3){ - override def update(conn: Connection, cl: ClassLoader): Unit = { - super.update(conn, cl) - // Fix wiki repository configuration - conn.select("SELECT USER_NAME, REPOSITORY_NAME FROM REPOSITORY"){ rs => - using(Git.open(getWikiRepositoryDir(rs.getString("USER_NAME"), rs.getString("REPOSITORY_NAME")))){ git => - defining(git.getRepository.getConfig){ config => - if(!config.getBoolean("http", "receivepack", false)){ - config.setBoolean("http", null, "receivepack", true) - config.save - } - } - } - } - } - }, - Version(1, 2), - Version(1, 1), - Version(1, 0), - Version(0, 0) - ) - - /** - * The head version of BitBucket. - */ - val headVersion = versions.head - - /** - * The version file (GITBUCKET_HOME/version). - */ - lazy val versionFile = new File(GitBucketHome, "version") - - /** - * Returns the current version from the version file. - */ - def getCurrentVersion(): Version = { - if(versionFile.exists){ - FileUtils.readFileToString(versionFile, "UTF-8").trim.split("\\.") match { - case Array(majorVersion, minorVersion) => { - versions.find { v => - v.majorVersion == majorVersion.toInt && v.minorVersion == minorVersion.toInt - }.getOrElse(Version(0, 0)) - } - case _ => Version(0, 0) - } - } else Version(0, 0) - } - -} diff --git a/src/main/scala/gitbucket/core/servlet/InitializeListener.scala b/src/main/scala/gitbucket/core/servlet/InitializeListener.scala index a375b37f7..f9ccc3c36 100644 --- a/src/main/scala/gitbucket/core/servlet/InitializeListener.scala +++ b/src/main/scala/gitbucket/core/servlet/InitializeListener.scala @@ -2,15 +2,15 @@ package gitbucket.core.servlet import akka.event.Logging import com.typesafe.config.ConfigFactory +import gitbucket.core.GitBucketCoreModule import gitbucket.core.plugin.PluginRegistry import gitbucket.core.service.{ActivityService, SystemSettingsService} -import org.apache.commons.io.FileUtils +import io.github.gitbucket.solidbase.Solidbase +import liquibase.database.core.H2Database import javax.servlet.{ServletContextListener, ServletContextEvent} import org.slf4j.LoggerFactory -import gitbucket.core.util.Versions import akka.actor.{Actor, Props, ActorSystem} import com.typesafe.akka.extension.quartz.QuartzSchedulerExtension -import AutoUpdate._ /** * Initialize GitBucket system. @@ -31,13 +31,12 @@ class InitializeListener extends ServletContextListener with SystemSettingsServi val conn = session.conn // Migration - logger.debug("Start schema update") - Versions.update(conn, headVersion, getCurrentVersion(), versions, Thread.currentThread.getContextClassLoader){ conn => - FileUtils.writeStringToFile(versionFile, headVersion.versionString, "UTF-8") - } + logger.info("Start schema update") + val solidbase = new Solidbase() + solidbase.migrate(conn, Thread.currentThread.getContextClassLoader, new H2Database(), GitBucketCoreModule) // Load plugins - logger.debug("Initialize plugins") + logger.info("Initialize plugins") PluginRegistry.initialize(event.getServletContext, loadSystemSettings(), conn) } diff --git a/src/main/scala/gitbucket/core/util/Version.scala b/src/main/scala/gitbucket/core/util/Version.scala deleted file mode 100644 index 3c7ffe631..000000000 --- a/src/main/scala/gitbucket/core/util/Version.scala +++ /dev/null @@ -1,67 +0,0 @@ -package gitbucket.core.util - -import java.sql.Connection - -import org.apache.commons.io.IOUtils -import org.slf4j.LoggerFactory -import ControlUtil._ - -case class Version(majorVersion: Int, minorVersion: Int) { - - private val logger = LoggerFactory.getLogger(classOf[Version]) - - /** - * Execute update/MAJOR_MINOR.sql to update schema to this version. - * If corresponding SQL file does not exist, this method do nothing. - */ - def update(conn: Connection, cl: ClassLoader): Unit = { - val sqlPath = s"update/${majorVersion}_${minorVersion}.sql" - - using(cl.getResourceAsStream(sqlPath)){ in => - if(in != null){ - val sql = IOUtils.toString(in, "UTF-8") - using(conn.createStatement()){ stmt => - logger.debug(sqlPath + "=" + sql) - stmt.executeUpdate(sql) - } - } - } - } - - - /** - * MAJOR.MINOR - */ - val versionString = s"${majorVersion}.${minorVersion}" - -} - -object Versions { - - private val logger = LoggerFactory.getLogger(Versions.getClass) - - def update(conn: Connection, headVersion: Version, currentVersion: Version, versions: Seq[Version], cl: ClassLoader) - (save: Connection => Unit): Unit = { - logger.debug("Start schema update") - try { - if(currentVersion == headVersion){ - logger.debug("No update") - } else if(currentVersion.versionString != "0.0" && !versions.contains(currentVersion)){ - logger.warn(s"Skip migration because ${currentVersion.versionString} is illegal version.") - } else { - versions.takeWhile(_ != currentVersion).reverse.foreach(_.update(conn, cl)) - save(conn) - logger.debug(s"Updated from ${currentVersion.versionString} to ${headVersion.versionString}") - } - } catch { - case ex: Throwable => { - logger.error("Failed to schema update", ex) - ex.printStackTrace() - conn.rollback() - } - } - logger.debug("End schema update") - } - -} - diff --git a/src/main/twirl/gitbucket/core/main.scala.html b/src/main/twirl/gitbucket/core/main.scala.html index 7139b3f04..cb79802fb 100644 --- a/src/main/twirl/gitbucket/core/main.scala.html +++ b/src/main/twirl/gitbucket/core/main.scala.html @@ -1,6 +1,5 @@ @(title: String, repository: Option[gitbucket.core.service.RepositoryService.RepositoryInfo] = None)(body: Html)(implicit context: gitbucket.core.controller.Context) @import gitbucket.core.plugin.PluginRegistry -@import gitbucket.core.servlet.AutoUpdate @import context._ @import gitbucket.core.view.helpers._ @@ -54,9 +53,7 @@ *@ GitBucket - @defining(AutoUpdate.getCurrentVersion){ version => - @version.majorVersion.@version.minorVersion - } + @gitbucket.core.GitBucketCoreModule.getVersions.get(0).getVersion @if(loginAccount.isDefined){ @repository.map { repository => diff --git a/src/test/scala/gitbucket/core/service/ServiceSpecBase.scala b/src/test/scala/gitbucket/core/service/ServiceSpecBase.scala index f5fd9eaad..08f04a78a 100644 --- a/src/test/scala/gitbucket/core/service/ServiceSpecBase.scala +++ b/src/test/scala/gitbucket/core/service/ServiceSpecBase.scala @@ -1,6 +1,5 @@ package gitbucket.core.service -import gitbucket.core.servlet.AutoUpdate import gitbucket.core.util.{ControlUtil, DatabaseConfig, FileUtil} import gitbucket.core.util.ControlUtil._ import gitbucket.core.model._ From f5c1c0703db14f20178f79d907314a7e60867422 Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Wed, 13 Jan 2016 11:47:59 +0900 Subject: [PATCH 02/47] Fix compilation error in testcase --- .../scala/gitbucket/core/service/ServiceSpecBase.scala | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/test/scala/gitbucket/core/service/ServiceSpecBase.scala b/src/test/scala/gitbucket/core/service/ServiceSpecBase.scala index 08f04a78a..c969c18b9 100644 --- a/src/test/scala/gitbucket/core/service/ServiceSpecBase.scala +++ b/src/test/scala/gitbucket/core/service/ServiceSpecBase.scala @@ -1,10 +1,16 @@ package gitbucket.core.service +import gitbucket.core.GitBucketCoreModule import gitbucket.core.util.{ControlUtil, DatabaseConfig, FileUtil} import gitbucket.core.util.ControlUtil._ import gitbucket.core.model._ import gitbucket.core.model.Profile._ +import io.github.gitbucket.solidbase.Solidbase +import liquibase.database.core.H2Database +import liquibase.database.jvm.JdbcConnection import profile.simple._ +import scalaz._ +import Scalaz._ import org.apache.commons.io.FileUtils @@ -21,7 +27,9 @@ trait ServiceSpecBase { val (url, user, pass) = (DatabaseConfig.url(Some(dir.toString)), DatabaseConfig.user, DatabaseConfig.password) org.h2.Driver.load() using(DriverManager.getConnection(url, user, pass)){ conn => - AutoUpdate.versions.reverse.foreach(_.update(conn, Thread.currentThread.getContextClassLoader)) + val solidbase = new Solidbase() + val db = new H2Database() <| { _.setConnection(new JdbcConnection(conn)) } // TODO Remove setConnection in the future + solidbase.migrate(conn, Thread.currentThread.getContextClassLoader, db, GitBucketCoreModule) } Database.forURL(url, user, pass).withSession { session => action(session) From 8b44a002991dc8406887ee380b31039042cbd67d Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Wed, 13 Jan 2016 11:57:36 +0900 Subject: [PATCH 03/47] Fix foreign key cascading --- src/main/resources/update/gitbucket-core_4.0.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/resources/update/gitbucket-core_4.0.xml b/src/main/resources/update/gitbucket-core_4.0.xml index 36ab26726..97877187c 100644 --- a/src/main/resources/update/gitbucket-core_4.0.xml +++ b/src/main/resources/update/gitbucket-core_4.0.xml @@ -187,7 +187,7 @@ - + @@ -205,9 +205,9 @@ - - - + + + @@ -248,7 +248,7 @@ - + From e92d1eae5ad0f4e1f4e7868cec8f2b970456304f Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Sat, 16 Jan 2016 03:19:39 +0900 Subject: [PATCH 04/47] Add migration process from 3.10 --- .../core/servlet/InitializeListener.scala | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/src/main/scala/gitbucket/core/servlet/InitializeListener.scala b/src/main/scala/gitbucket/core/servlet/InitializeListener.scala index f9ccc3c36..75a05a615 100644 --- a/src/main/scala/gitbucket/core/servlet/InitializeListener.scala +++ b/src/main/scala/gitbucket/core/servlet/InitializeListener.scala @@ -1,13 +1,19 @@ package gitbucket.core.servlet +import java.io.File + import akka.event.Logging import com.typesafe.config.ConfigFactory import gitbucket.core.GitBucketCoreModule import gitbucket.core.plugin.PluginRegistry import gitbucket.core.service.{ActivityService, SystemSettingsService} +import gitbucket.core.util.Directory._ +import gitbucket.core.util.JDBCUtil._ import io.github.gitbucket.solidbase.Solidbase +import io.github.gitbucket.solidbase.manager.JDBCVersionManager import liquibase.database.core.H2Database import javax.servlet.{ServletContextListener, ServletContextEvent} +import org.apache.commons.io.FileUtils import org.slf4j.LoggerFactory import akka.actor.{Actor, Props, ActorSystem} import com.typesafe.akka.extension.quartz.QuartzSchedulerExtension @@ -30,7 +36,37 @@ class InitializeListener extends ServletContextListener with SystemSettingsServi Database() withTransaction { session => val conn = session.conn - // Migration + // Check version + val versionFile = new File(GitBucketHome, "version") + + if(versionFile.exists()){ + val version = FileUtils.readFileToString(versionFile, "UTF-8") + if(version == "3.10"){ + // Initialization for GitBucket 3.10 + logger.info("Migration to GitBucket 4.x start") + + // Backup current data + FileUtils.copyFile(new File(GitBucketHome, "data.mv.db"), new File(GitBucketHome, "data.mv.db_3.10")) + FileUtils.copyFile(new File(GitBucketHome, "data.trace.db"), new File(GitBucketHome, "data.trace.db_3.10")) + + // Change form + val manager = new JDBCVersionManager(conn) + manager.initialize() + manager.updateVersion(GitBucketCoreModule.getModuleId, "4.0") + conn.select("SELECT PLUGIN_ID, VERSION FROM PLUGIN"){ rs => + manager.updateVersion(rs.getString("PLUGIN_ID"), rs.getString("VERSION")) + } + conn.update("DROP TABLE PLUGIN") + versionFile.delete() + + logger.info("Migration to GitBucket 4.x completed") + + } else { + throw new Exception("GitBucket can't migrate from this version. Please update to 3.10 at first.") + } + } + + // Run normal migration logger.info("Start schema update") val solidbase = new Solidbase() solidbase.migrate(conn, Thread.currentThread.getContextClassLoader, new H2Database(), GitBucketCoreModule) From 19e74a4fe104cc0acc209c475f9bcea8eb4965f3 Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Sun, 17 Jan 2016 13:29:27 +0900 Subject: [PATCH 05/47] Remove unused model and service for PLUGIN table --- .../controller/SystemSettingsController.scala | 6 ----- .../scala/gitbucket/core/model/Plugin.scala | 19 --------------- .../scala/gitbucket/core/model/Profile.scala | 1 - .../core/service/PluginService.scala | 24 ------------------- 4 files changed, 50 deletions(-) delete mode 100644 src/main/scala/gitbucket/core/model/Plugin.scala delete mode 100644 src/main/scala/gitbucket/core/service/PluginService.scala diff --git a/src/main/scala/gitbucket/core/controller/SystemSettingsController.scala b/src/main/scala/gitbucket/core/controller/SystemSettingsController.scala index 01999f6f9..2bc6ecece 100644 --- a/src/main/scala/gitbucket/core/controller/SystemSettingsController.scala +++ b/src/main/scala/gitbucket/core/controller/SystemSettingsController.scala @@ -55,12 +55,6 @@ trait SystemSettingsControllerBase extends ControllerBase { } else Nil } - private val pluginForm = mapping( - "pluginId" -> list(trim(label("", text()))) - )(PluginForm.apply) - - case class PluginForm(pluginIds: List[String]) - get("/admin/system")(adminOnly { html.system(flash.get("info")) }) diff --git a/src/main/scala/gitbucket/core/model/Plugin.scala b/src/main/scala/gitbucket/core/model/Plugin.scala deleted file mode 100644 index 1e8aac545..000000000 --- a/src/main/scala/gitbucket/core/model/Plugin.scala +++ /dev/null @@ -1,19 +0,0 @@ -package gitbucket.core.model - -trait PluginComponent extends TemplateComponent { self: Profile => - import profile.simple._ - import self._ - - lazy val Plugins = TableQuery[Plugins] - - class Plugins(tag: Tag) extends Table[Plugin](tag, "PLUGIN"){ - val pluginId = column[String]("PLUGIN_ID", O PrimaryKey) - val version = column[String]("VERSION") - def * = (pluginId, version) <> (Plugin.tupled, Plugin.unapply) - } -} - -case class Plugin( - pluginId: String, - version: String -) diff --git a/src/main/scala/gitbucket/core/model/Profile.scala b/src/main/scala/gitbucket/core/model/Profile.scala index e17363059..4fba6a221 100644 --- a/src/main/scala/gitbucket/core/model/Profile.scala +++ b/src/main/scala/gitbucket/core/model/Profile.scala @@ -49,6 +49,5 @@ trait CoreProfile extends ProfileProvider with Profile with SshKeyComponent with WebHookComponent with WebHookEventComponent - with PluginComponent object Profile extends CoreProfile diff --git a/src/main/scala/gitbucket/core/service/PluginService.scala b/src/main/scala/gitbucket/core/service/PluginService.scala deleted file mode 100644 index 99a20d831..000000000 --- a/src/main/scala/gitbucket/core/service/PluginService.scala +++ /dev/null @@ -1,24 +0,0 @@ -package gitbucket.core.service - -import gitbucket.core.model.Plugin -import gitbucket.core.model.Profile._ -import profile.simple._ - -trait PluginService { - - def getPlugins()(implicit s: Session): List[Plugin] = - Plugins.sortBy(_.pluginId).list - - def registerPlugin(plugin: Plugin)(implicit s: Session): Unit = - Plugins.insert(plugin) - - def updatePlugin(plugin: Plugin)(implicit s: Session): Unit = - Plugins.filter(_.pluginId === plugin.pluginId.bind).map(_.version).update(plugin.version) - - def deletePlugin(pluginId: String)(implicit s: Session): Unit = - Plugins.filter(_.pluginId === pluginId.bind).delete - - def getPlugin(pluginId: String)(implicit s: Session): Option[Plugin] = - Plugins.filter(_.pluginId === pluginId.bind).firstOption - -} From 249b27593e2f7b5ff4c7e63afef9831bda1224fa Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Wed, 20 Jan 2016 03:12:30 +0900 Subject: [PATCH 06/47] Apply DDL for branch protection to solidbase migration --- src/main/resources/update/3_11.sql | 25 ------------------- .../resources/update/gitbucket-core_4.0.xml | 20 +++++++++++++++ 2 files changed, 20 insertions(+), 25 deletions(-) delete mode 100644 src/main/resources/update/3_11.sql diff --git a/src/main/resources/update/3_11.sql b/src/main/resources/update/3_11.sql deleted file mode 100644 index 721690aff..000000000 --- a/src/main/resources/update/3_11.sql +++ /dev/null @@ -1,25 +0,0 @@ -DROP TABLE IF EXISTS PROTECTED_BRANCH; - -CREATE TABLE PROTECTED_BRANCH( - USER_NAME VARCHAR(100) NOT NULL, - REPOSITORY_NAME VARCHAR(100) NOT NULL, - BRANCH VARCHAR(100) NOT NULL, - STATUS_CHECK_ADMIN BOOLEAN NOT NULL DEFAULT false -); - -ALTER TABLE PROTECTED_BRANCH ADD CONSTRAINT IDX_PROTECTED_BRANCH_PK PRIMARY KEY (USER_NAME, REPOSITORY_NAME, BRANCH); -ALTER TABLE PROTECTED_BRANCH ADD CONSTRAINT IDX_PROTECTED_BRANCH_FK0 FOREIGN KEY (USER_NAME, REPOSITORY_NAME) REFERENCES REPOSITORY (USER_NAME, REPOSITORY_NAME) - ON DELETE CASCADE ON UPDATE CASCADE; - - -DROP TABLE IF EXISTS PROTECTED_BRANCH_REQUIRE_CONTEXT; -CREATE TABLE PROTECTED_BRANCH_REQUIRE_CONTEXT( - USER_NAME VARCHAR(100) NOT NULL, - REPOSITORY_NAME VARCHAR(100) NOT NULL, - BRANCH VARCHAR(100) NOT NULL, - CONTEXT VARCHAR(255) NOT NULL -); - -ALTER TABLE PROTECTED_BRANCH_REQUIRE_CONTEXT ADD CONSTRAINT IDX_PROTECTED_BRANCH_REQUIRE_CONTEXT_PK PRIMARY KEY (USER_NAME, REPOSITORY_NAME, BRANCH, CONTEXT); -ALTER TABLE PROTECTED_BRANCH_REQUIRE_CONTEXT ADD CONSTRAINT IDX_PROTECTED_BRANCH_REQUIRE_CONTEXT_FK0 FOREIGN KEY (USER_NAME, REPOSITORY_NAME, BRANCH) REFERENCES PROTECTED_BRANCH (USER_NAME, REPOSITORY_NAME, BRANCH) - ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/src/main/resources/update/gitbucket-core_4.0.xml b/src/main/resources/update/gitbucket-core_4.0.xml index 97877187c..bf37f8e02 100644 --- a/src/main/resources/update/gitbucket-core_4.0.xml +++ b/src/main/resources/update/gitbucket-core_4.0.xml @@ -185,6 +185,20 @@ + + + + + + + + + + + + + + @@ -250,6 +264,12 @@ + + + + + + From af7043f4bf2605991d214cdf988f909482e63c14 Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Thu, 21 Jan 2016 11:35:00 +0900 Subject: [PATCH 07/47] Update for GitBucket 3.11 --- .../scala/gitbucket/core/servlet/InitializeListener.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/scala/gitbucket/core/servlet/InitializeListener.scala b/src/main/scala/gitbucket/core/servlet/InitializeListener.scala index 75a05a615..61c756720 100644 --- a/src/main/scala/gitbucket/core/servlet/InitializeListener.scala +++ b/src/main/scala/gitbucket/core/servlet/InitializeListener.scala @@ -41,13 +41,13 @@ class InitializeListener extends ServletContextListener with SystemSettingsServi if(versionFile.exists()){ val version = FileUtils.readFileToString(versionFile, "UTF-8") - if(version == "3.10"){ + if(version == "3.11"){ // Initialization for GitBucket 3.10 logger.info("Migration to GitBucket 4.x start") // Backup current data - FileUtils.copyFile(new File(GitBucketHome, "data.mv.db"), new File(GitBucketHome, "data.mv.db_3.10")) - FileUtils.copyFile(new File(GitBucketHome, "data.trace.db"), new File(GitBucketHome, "data.trace.db_3.10")) + FileUtils.copyFile(new File(GitBucketHome, "data.mv.db"), new File(GitBucketHome, "data.mv.db_3.11")) + FileUtils.copyFile(new File(GitBucketHome, "data.trace.db"), new File(GitBucketHome, "data.trace.db_3.11")) // Change form val manager = new JDBCVersionManager(conn) @@ -62,7 +62,7 @@ class InitializeListener extends ServletContextListener with SystemSettingsServi logger.info("Migration to GitBucket 4.x completed") } else { - throw new Exception("GitBucket can't migrate from this version. Please update to 3.10 at first.") + throw new Exception("GitBucket can't migrate from this version. Please update to 3.11 at first.") } } From 43097b4c1c2cd1aad1450357369e198edd85a153 Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Sun, 6 Mar 2016 13:07:43 +0900 Subject: [PATCH 08/47] Restore amateras-snapshot repository setting --- build.sbt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 60da60326..dbd5567be 100644 --- a/build.sbt +++ b/build.sbt @@ -15,7 +15,8 @@ scalaVersion := "2.11.7" // dependency settings resolvers ++= Seq( Classpaths.typesafeReleases, - "sonatype-snapshot" at "https://oss.sonatype.org/content/repositories/snapshots/" + "sonatype-snapshot" at "https://oss.sonatype.org/content/repositories/snapshots/", + "amateras-snapshot" at "http://amateras.sourceforge.jp/mvn-snapshot/" ) libraryDependencies ++= Seq( "org.eclipse.jgit" % "org.eclipse.jgit.http.server" % "4.1.1.201511131810-r", From c8686f4b34ac668aa786dbb8d7685d29ce90ef85 Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Sun, 6 Mar 2016 13:10:28 +0900 Subject: [PATCH 09/47] Pick adding WEB_HOOK.TOKEN into solidbase integration WEB_HOOK.TOKEN is added in #1127 --- src/main/resources/update/3_13.sql | 1 - src/main/resources/update/gitbucket-core_4.0.xml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 src/main/resources/update/3_13.sql diff --git a/src/main/resources/update/3_13.sql b/src/main/resources/update/3_13.sql deleted file mode 100644 index 8efe1a3ab..000000000 --- a/src/main/resources/update/3_13.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE WEB_HOOK ADD COLUMN TOKEN VARCHAR(100); \ No newline at end of file diff --git a/src/main/resources/update/gitbucket-core_4.0.xml b/src/main/resources/update/gitbucket-core_4.0.xml index bf37f8e02..3fd59f6d7 100644 --- a/src/main/resources/update/gitbucket-core_4.0.xml +++ b/src/main/resources/update/gitbucket-core_4.0.xml @@ -176,6 +176,7 @@ + From 6c2fce1b16cb69d3ece25a9bd9e66393e75341a9 Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Mon, 11 Apr 2016 21:29:58 +0900 Subject: [PATCH 10/47] Switch Slick driver by system property --- src/main/scala/gitbucket/core/model/Profile.scala | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/main/scala/gitbucket/core/model/Profile.scala b/src/main/scala/gitbucket/core/model/Profile.scala index 0badf86f3..de3df9b0b 100644 --- a/src/main/scala/gitbucket/core/model/Profile.scala +++ b/src/main/scala/gitbucket/core/model/Profile.scala @@ -28,7 +28,19 @@ trait Profile { } trait ProfileProvider { self: Profile => - val profile = slick.driver.H2Driver + + private val url = System.getProperty("db.url") +// private val user = System.getProperty("db.user") +// private val password = System.getProperty("db.password") + + val profile = if(url.indexOf("h2") >= 0){ + slick.driver.H2Driver + } else if(url.indexOf("mysql") >= 0) { + slick.driver.MySQLDriver + } else { + throw new ExceptionInInitializerError(s"${url} is not unsupported.") + } + } trait CoreProfile extends ProfileProvider with Profile From 23a482bbbad043087df99c04bfeac63cf26ca1ad Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Mon, 11 Apr 2016 22:16:41 +0900 Subject: [PATCH 11/47] Use HikariCP instead of c3p0 --- build.sbt | 2 +- .../scala/gitbucket/core/model/Profile.scala | 8 ++++---- .../core/servlet/TransactionFilter.scala | 16 ++++++++-------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/build.sbt b/build.sbt index 6283bbcac..85cf1ce00 100644 --- a/build.sbt +++ b/build.sbt @@ -36,7 +36,7 @@ libraryDependencies ++= Seq( "com.novell.ldap" % "jldap" % "2009-10-07", "com.h2database" % "h2" % "1.4.190", "ch.qos.logback" % "logback-classic" % "1.1.1", - "com.mchange" % "c3p0" % "0.9.5.2", + "com.zaxxer" % "HikariCP" % "2.4.5", "com.typesafe" % "config" % "1.3.0", "com.typesafe.akka" %% "akka-actor" % "2.3.14", "fr.brouillard.oss.security.xhub" % "xhub4j-core" % "1.0.0", diff --git a/src/main/scala/gitbucket/core/model/Profile.scala b/src/main/scala/gitbucket/core/model/Profile.scala index de3df9b0b..1fb361d16 100644 --- a/src/main/scala/gitbucket/core/model/Profile.scala +++ b/src/main/scala/gitbucket/core/model/Profile.scala @@ -1,5 +1,7 @@ package gitbucket.core.model +import gitbucket.core.util.DatabaseConfig + trait Profile { val profile: slick.driver.JdbcProfile @@ -29,11 +31,9 @@ trait Profile { trait ProfileProvider { self: Profile => - private val url = System.getProperty("db.url") -// private val user = System.getProperty("db.user") -// private val password = System.getProperty("db.password") + private lazy val url = DatabaseConfig.url - val profile = if(url.indexOf("h2") >= 0){ + lazy val profile = if(url.indexOf("h2") >= 0){ slick.driver.H2Driver } else if(url.indexOf("mysql") >= 0) { slick.driver.MySQLDriver diff --git a/src/main/scala/gitbucket/core/servlet/TransactionFilter.scala b/src/main/scala/gitbucket/core/servlet/TransactionFilter.scala index fc2e45743..e61cc10ae 100644 --- a/src/main/scala/gitbucket/core/servlet/TransactionFilter.scala +++ b/src/main/scala/gitbucket/core/servlet/TransactionFilter.scala @@ -2,7 +2,7 @@ package gitbucket.core.servlet import javax.servlet._ import javax.servlet.http.HttpServletRequest -import com.mchange.v2.c3p0.ComboPooledDataSource +import com.zaxxer.hikari._ import gitbucket.core.util.DatabaseConfig import org.scalatra.ScalatraBase import org.slf4j.LoggerFactory @@ -46,14 +46,14 @@ object Database { private val logger = LoggerFactory.getLogger(Database.getClass) - private val dataSource: ComboPooledDataSource = { - val ds = new ComboPooledDataSource - ds.setDriverClass(DatabaseConfig.driver) - ds.setJdbcUrl(DatabaseConfig.url) - ds.setUser(DatabaseConfig.user) - ds.setPassword(DatabaseConfig.password) + private val dataSource: HikariDataSource = { + val config = new HikariConfig() + config.setDriverClassName(DatabaseConfig.driver) + config.setJdbcUrl(DatabaseConfig.url) + config.setUsername(DatabaseConfig.user) + config.setPassword(DatabaseConfig.password) logger.debug("load database connection pool") - ds + new HikariDataSource(config) } private val db: SlickDatabase = { From 5a0bc127b7191918fcb4b7479637de7dd8e5761e Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Tue, 12 Apr 2016 01:06:12 +0900 Subject: [PATCH 12/47] Add MySQL jdbc driver --- build.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sbt b/build.sbt index 85cf1ce00..c6ee304db 100644 --- a/build.sbt +++ b/build.sbt @@ -35,6 +35,7 @@ libraryDependencies ++= Seq( "com.typesafe.slick" %% "slick" % "2.1.0", "com.novell.ldap" % "jldap" % "2009-10-07", "com.h2database" % "h2" % "1.4.190", + "mysql" % "mysql-connector-java" % "5.1.38", "ch.qos.logback" % "logback-classic" % "1.1.1", "com.zaxxer" % "HikariCP" % "2.4.5", "com.typesafe" % "config" % "1.3.0", From 6f4e94ba9ad1dd3a0ab1e142950666417b145eeb Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Tue, 12 Apr 2016 10:06:38 +0900 Subject: [PATCH 13/47] Update migration scripts for MySQL compatibility --- .../resources/update/gitbucket-core_4.0.sql | 2 +- .../resources/update/gitbucket-core_4.0.xml | 433 ++++++++++-------- 2 files changed, 249 insertions(+), 186 deletions(-) diff --git a/src/main/resources/update/gitbucket-core_4.0.sql b/src/main/resources/update/gitbucket-core_4.0.sql index 775a77c1c..12b5fcf1d 100644 --- a/src/main/resources/update/gitbucket-core_4.0.sql +++ b/src/main/resources/update/gitbucket-core_4.0.sql @@ -3,7 +3,7 @@ CREATE OR REPLACE VIEW ISSUE_OUTLINE_VIEW AS A.USER_NAME, A.REPOSITORY_NAME, A.ISSUE_ID, - NVL(B.COMMENT_COUNT, 0) + NVL(C.COMMENT_COUNT, 0) AS COMMENT_COUNT + IFNULL(B.COMMENT_COUNT, 0) + IFNULL(C.COMMENT_COUNT, 0) AS COMMENT_COUNT FROM ISSUE A LEFT OUTER JOIN ( SELECT USER_NAME, REPOSITORY_NAME, ISSUE_ID, COUNT(COMMENT_ID) AS COMMENT_COUNT FROM ISSUE_COMMENT diff --git a/src/main/resources/update/gitbucket-core_4.0.xml b/src/main/resources/update/gitbucket-core_4.0.xml index 3fd59f6d7..ed0da7b74 100644 --- a/src/main/resources/update/gitbucket-core_4.0.xml +++ b/src/main/resources/update/gitbucket-core_4.0.xml @@ -1,13 +1,9 @@ - - - - - - - + + + @@ -23,133 +19,25 @@ - - - - - - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + @@ -165,123 +53,298 @@ - - - - - - + + - + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - \ No newline at end of file From d50c858a269a1e7a152ee525c9fac6d6f253d307 Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Tue, 12 Apr 2016 11:12:32 +0900 Subject: [PATCH 14/47] Use ${currentDateTime} --- src/main/resources/update/gitbucket-core_4.0.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/update/gitbucket-core_4.0.xml b/src/main/resources/update/gitbucket-core_4.0.xml index ed0da7b74..e3368e771 100644 --- a/src/main/resources/update/gitbucket-core_4.0.xml +++ b/src/main/resources/update/gitbucket-core_4.0.xml @@ -31,8 +31,8 @@ - - + + From 9eff8f248b4da4b7cc7e24ebb0ac19825cb09c25 Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Tue, 12 Apr 2016 11:16:01 +0900 Subject: [PATCH 15/47] Experimental support for MySQL is available! --- src/main/resources/database.conf | 1 - src/main/resources/database_mysql.conf | 5 ++ .../scala/gitbucket/core/model/Profile.scala | 11 +---- .../core/servlet/InitializeListener.scala | 8 ++-- .../core/servlet/TransactionFilter.scala | 2 +- .../gitbucket/core/util/DatabaseConfig.scala | 47 ++++++++++++++++--- 6 files changed, 52 insertions(+), 22 deletions(-) create mode 100644 src/main/resources/database_mysql.conf diff --git a/src/main/resources/database.conf b/src/main/resources/database.conf index 0eca73302..9ce0c897b 100644 --- a/src/main/resources/database.conf +++ b/src/main/resources/database.conf @@ -1,5 +1,4 @@ db { - driver = "org.h2.Driver" url = "jdbc:h2:${DatabaseHome};MVCC=true" user = "sa" password = "sa" diff --git a/src/main/resources/database_mysql.conf b/src/main/resources/database_mysql.conf new file mode 100644 index 000000000..59bafeb90 --- /dev/null +++ b/src/main/resources/database_mysql.conf @@ -0,0 +1,5 @@ +db { + url = "jdbc:mysql://192.168.99.100/gitbucket" + user = "root" + password = "root" +} diff --git a/src/main/scala/gitbucket/core/model/Profile.scala b/src/main/scala/gitbucket/core/model/Profile.scala index 189e0a661..26bb22595 100644 --- a/src/main/scala/gitbucket/core/model/Profile.scala +++ b/src/main/scala/gitbucket/core/model/Profile.scala @@ -2,7 +2,6 @@ package gitbucket.core.model import gitbucket.core.util.DatabaseConfig - trait Profile { val profile: slick.driver.JdbcProfile import profile.simple._ @@ -31,15 +30,7 @@ trait Profile { trait ProfileProvider { self: Profile => - private lazy val url = DatabaseConfig.url - - lazy val profile = if(url.indexOf("h2") >= 0){ - slick.driver.H2Driver - } else if(url.indexOf("mysql") >= 0) { - slick.driver.MySQLDriver - } else { - throw new ExceptionInInitializerError(s"${url} is not unsupported.") - } + lazy val profile = DatabaseConfig.slickDriver } diff --git a/src/main/scala/gitbucket/core/servlet/InitializeListener.scala b/src/main/scala/gitbucket/core/servlet/InitializeListener.scala index 61c756720..7fdc3286f 100644 --- a/src/main/scala/gitbucket/core/servlet/InitializeListener.scala +++ b/src/main/scala/gitbucket/core/servlet/InitializeListener.scala @@ -7,11 +7,11 @@ import com.typesafe.config.ConfigFactory import gitbucket.core.GitBucketCoreModule import gitbucket.core.plugin.PluginRegistry import gitbucket.core.service.{ActivityService, SystemSettingsService} +import gitbucket.core.util.DatabaseConfig import gitbucket.core.util.Directory._ import gitbucket.core.util.JDBCUtil._ import io.github.gitbucket.solidbase.Solidbase import io.github.gitbucket.solidbase.manager.JDBCVersionManager -import liquibase.database.core.H2Database import javax.servlet.{ServletContextListener, ServletContextEvent} import org.apache.commons.io.FileUtils import org.slf4j.LoggerFactory @@ -41,7 +41,7 @@ class InitializeListener extends ServletContextListener with SystemSettingsServi if(versionFile.exists()){ val version = FileUtils.readFileToString(versionFile, "UTF-8") - if(version == "3.11"){ + if(version == "3.13"){ // Initialization for GitBucket 3.10 logger.info("Migration to GitBucket 4.x start") @@ -62,14 +62,14 @@ class InitializeListener extends ServletContextListener with SystemSettingsServi logger.info("Migration to GitBucket 4.x completed") } else { - throw new Exception("GitBucket can't migrate from this version. Please update to 3.11 at first.") + throw new Exception("GitBucket can't migrate from this version. Please update to 3.13 at first.") } } // Run normal migration logger.info("Start schema update") val solidbase = new Solidbase() - solidbase.migrate(conn, Thread.currentThread.getContextClassLoader, new H2Database(), GitBucketCoreModule) + solidbase.migrate(conn, Thread.currentThread.getContextClassLoader, DatabaseConfig.liquiDriver, GitBucketCoreModule) // Load plugins logger.info("Initialize plugins") diff --git a/src/main/scala/gitbucket/core/servlet/TransactionFilter.scala b/src/main/scala/gitbucket/core/servlet/TransactionFilter.scala index e61cc10ae..bad06e1a9 100644 --- a/src/main/scala/gitbucket/core/servlet/TransactionFilter.scala +++ b/src/main/scala/gitbucket/core/servlet/TransactionFilter.scala @@ -48,7 +48,7 @@ object Database { private val dataSource: HikariDataSource = { val config = new HikariConfig() - config.setDriverClassName(DatabaseConfig.driver) + config.setDriverClassName(DatabaseConfig.jdbcDriver) config.setJdbcUrl(DatabaseConfig.url) config.setUsername(DatabaseConfig.user) config.setPassword(DatabaseConfig.password) diff --git a/src/main/scala/gitbucket/core/util/DatabaseConfig.scala b/src/main/scala/gitbucket/core/util/DatabaseConfig.scala index ffe7ceeb9..9ab374d20 100644 --- a/src/main/scala/gitbucket/core/util/DatabaseConfig.scala +++ b/src/main/scala/gitbucket/core/util/DatabaseConfig.scala @@ -2,18 +2,53 @@ package gitbucket.core.util import com.typesafe.config.ConfigFactory import Directory.DatabaseHome +import liquibase.database.AbstractJdbcDatabase +import liquibase.database.core.{MySQLDatabase, H2Database} object DatabaseConfig { - private val config = ConfigFactory.load("database") - private val dbUrl = config.getString("db.url") + private lazy val config = ConfigFactory.load("database") + private lazy val dbUrl = config.getString("db.url") def url(directory: Option[String]): String = dbUrl.replace("${DatabaseHome}", directory.getOrElse(DatabaseHome)) - val url: String = url(None) - val user: String = config.getString("db.user") - val password: String = config.getString("db.password") - val driver: String = config.getString("db.driver") + lazy val url: String = url(None) + lazy val user: String = config.getString("db.user") + lazy val password: String = config.getString("db.password") + lazy val jdbcDriver: String = DatabaseType(url).jdbcDriver + lazy val slickDriver: slick.driver.JdbcProfile = DatabaseType(url).slickDriver + lazy val liquiDriver: AbstractJdbcDatabase = DatabaseType(url).liquiDriver } + +sealed trait DatabaseType { + val jdbcDriver: String + val slickDriver: slick.driver.JdbcProfile + val liquiDriver: AbstractJdbcDatabase +} + +object DatabaseType { + + def apply(url: String): DatabaseType = { + if(url.indexOf("h2") >= 0){ + H2 + } else if(url.indexOf("mysql") >= 0){ + MySQL + } else { + throw new IllegalArgumentException(s"${url} is not supported.") + } + } + + object H2 extends DatabaseType { + val jdbcDriver = "org.h2.Driver" + val slickDriver = slick.driver.H2Driver + val liquiDriver = new H2Database() + } + + object MySQL extends DatabaseType { + val jdbcDriver = "com.mysql.jdbc.Driver" + val slickDriver = slick.driver.MySQLDriver + val liquiDriver = new MySQLDatabase() + } +} From 523ea0d43726aa1d680724ed23084e8957ecd055 Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Tue, 12 Apr 2016 14:46:20 +0900 Subject: [PATCH 16/47] Fix comment --- src/main/scala/gitbucket/core/servlet/InitializeListener.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/gitbucket/core/servlet/InitializeListener.scala b/src/main/scala/gitbucket/core/servlet/InitializeListener.scala index 7fdc3286f..b843072db 100644 --- a/src/main/scala/gitbucket/core/servlet/InitializeListener.scala +++ b/src/main/scala/gitbucket/core/servlet/InitializeListener.scala @@ -42,7 +42,7 @@ class InitializeListener extends ServletContextListener with SystemSettingsServi if(versionFile.exists()){ val version = FileUtils.readFileToString(versionFile, "UTF-8") if(version == "3.13"){ - // Initialization for GitBucket 3.10 + // Initialization for GitBucket 3.13 logger.info("Migration to GitBucket 4.x start") // Backup current data From 3c727fe6784a7b32c8fbb47dd8f2c8c99d8e2cd5 Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Tue, 12 Apr 2016 15:05:59 +0900 Subject: [PATCH 17/47] Backup H2 data files before migration to 4.0 if files exist --- .../gitbucket/core/servlet/InitializeListener.scala | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/scala/gitbucket/core/servlet/InitializeListener.scala b/src/main/scala/gitbucket/core/servlet/InitializeListener.scala index b843072db..02c09c6c5 100644 --- a/src/main/scala/gitbucket/core/servlet/InitializeListener.scala +++ b/src/main/scala/gitbucket/core/servlet/InitializeListener.scala @@ -46,8 +46,14 @@ class InitializeListener extends ServletContextListener with SystemSettingsServi logger.info("Migration to GitBucket 4.x start") // Backup current data - FileUtils.copyFile(new File(GitBucketHome, "data.mv.db"), new File(GitBucketHome, "data.mv.db_3.11")) - FileUtils.copyFile(new File(GitBucketHome, "data.trace.db"), new File(GitBucketHome, "data.trace.db_3.11")) + val dataMvFile = new File(GitBucketHome, "data.mv.db") + if(dataMvFile.exists) { + FileUtils.copyFile(dataMvFile, new File(GitBucketHome, "data.mv.db_3.13")) + } + val dataTraceFile = new File(GitBucketHome, "data.trace.db") + if(dataTraceFile.exists) { + FileUtils.copyFile(dataTraceFile, new File(GitBucketHome, "data.trace.db_3.13")) + } // Change form val manager = new JDBCVersionManager(conn) From 2d225641eea06e30d0d8c54857dd535f831995e3 Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Tue, 12 Apr 2016 22:03:31 +0900 Subject: [PATCH 18/47] Start to implement the database export tool --- .../scala/gitbucket/core/util/JDBCUtil.scala | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/src/main/scala/gitbucket/core/util/JDBCUtil.scala b/src/main/scala/gitbucket/core/util/JDBCUtil.scala index 41a2b1a56..78829a787 100644 --- a/src/main/scala/gitbucket/core/util/JDBCUtil.scala +++ b/src/main/scala/gitbucket/core/util/JDBCUtil.scala @@ -1,6 +1,8 @@ package gitbucket.core.util +import java.io.FileOutputStream import java.sql._ +import java.text.SimpleDateFormat import ControlUtil._ import scala.collection.mutable.ListBuffer @@ -58,6 +60,74 @@ object JDBCUtil { } } + def export(): Unit = { + val dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss") + + using(new FileOutputStream("export.sql")) { out => + val dbMeta = conn.getMetaData + + allTables(dbMeta).foreach { tableName => + val sb = new StringBuilder() + + select(s"SELECT * FROM ${tableName}") { rs => + sb.append(s"INSERT INTO ${tableName} (") + val rsMeta = rs.getMetaData + val columns = (1 to rsMeta.getColumnCount).map { i => + (rsMeta.getColumnName(i), rsMeta.getColumnType(i)) + } + sb.append(columns.map(_._1).mkString(", ")) + sb.append(") VALUES (") + val values = columns.map { case (columnName, columnType) => + columnType match { + case Types.BOOLEAN => rs.getBoolean(columnName) + case Types.VARCHAR | Types.CLOB | Types.CHAR => rs.getString(columnName) + case Types.INTEGER => rs.getInt(columnName) + case Types.TIMESTAMP => rs.getTimestamp(columnName) + } + } + + val columnValues = values.map { value => + value match { + case x: String => "'" + x.replace("'", "''") + "'" + case x: Timestamp => "'" + dateFormat.format(x) + "'" + case null => "NULL" + case x => x + } + } + sb.append(columnValues.mkString(", ")) + sb.append(");\n") + } + + out.write(sb.toString.getBytes("UTF-8")) + } + } + } + + private def parentTables(meta: DatabaseMetaData, tableName: String): Seq[String] = { + using(meta.getImportedKeys(null, null, tableName)) { rs => + val parents = new ListBuffer[String] + while (rs.next) { + val tableName = rs.getString("PKTABLE_NAME") + parents += tableName + parents ++= parentTables(meta, tableName) + } + parents.toSeq + } + } + + private def allTables(meta: DatabaseMetaData): Seq[String] = { + using(meta.getTables(null, null, "%", Seq("TABLE").toArray)) { rs => + val tables = new ListBuffer[(String, Seq[String])] + while (rs.next) { + val name = rs.getString("TABLE_NAME") + if(name != "VERSIONS") { + tables += ((name, parentTables(meta, name))) + } + } + tables.sortWith { (a, b) => b._2.contains(a._1) }.map(_._1).toSeq + } + } + } } From 338946dd3a6e836880a1d95e431d6316ced81ac2 Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Wed, 13 Apr 2016 10:36:03 +0900 Subject: [PATCH 19/47] Start to implement the database export tool --- .../scala/gitbucket/core/util/JDBCUtil.scala | 96 +++++++++++-------- 1 file changed, 55 insertions(+), 41 deletions(-) diff --git a/src/main/scala/gitbucket/core/util/JDBCUtil.scala b/src/main/scala/gitbucket/core/util/JDBCUtil.scala index 78829a787..fd65037dd 100644 --- a/src/main/scala/gitbucket/core/util/JDBCUtil.scala +++ b/src/main/scala/gitbucket/core/util/JDBCUtil.scala @@ -60,49 +60,68 @@ object JDBCUtil { } } - def export(): Unit = { + def export(targetTables: Seq[String]): Unit = { val dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss") using(new FileOutputStream("export.sql")) { out => val dbMeta = conn.getMetaData + val allTablesInDatabase = allTablesOrderByDependencies(dbMeta) - allTables(dbMeta).foreach { tableName => - val sb = new StringBuilder() + allTablesInDatabase.foreach { tableName => + if (targetTables.contains(tableName)) { + val sb = new StringBuilder() + sb.append("DELETE FROM ACCOUNT WHERE USER_NAME = 'root';\n") - select(s"SELECT * FROM ${tableName}") { rs => - sb.append(s"INSERT INTO ${tableName} (") - val rsMeta = rs.getMetaData - val columns = (1 to rsMeta.getColumnCount).map { i => - (rsMeta.getColumnName(i), rsMeta.getColumnType(i)) - } - sb.append(columns.map(_._1).mkString(", ")) - sb.append(") VALUES (") - val values = columns.map { case (columnName, columnType) => - columnType match { - case Types.BOOLEAN => rs.getBoolean(columnName) - case Types.VARCHAR | Types.CLOB | Types.CHAR => rs.getString(columnName) - case Types.INTEGER => rs.getInt(columnName) - case Types.TIMESTAMP => rs.getTimestamp(columnName) + select(s"SELECT * FROM ${tableName}") { rs => + sb.append(s"INSERT INTO ${tableName} (") + + val rsMeta = rs.getMetaData + val columns = (1 to rsMeta.getColumnCount).map { i => + (rsMeta.getColumnName(i), rsMeta.getColumnType(i)) } + sb.append(columns.map(_._1).mkString(", ")) + sb.append(") VALUES (") + + val values = columns.map { case (columnName, columnType) => + columnType match { + case Types.BOOLEAN => rs.getBoolean(columnName) + case Types.VARCHAR | Types.CLOB | Types.CHAR => rs.getString(columnName) + case Types.INTEGER => rs.getInt(columnName) + case Types.TIMESTAMP => rs.getTimestamp(columnName) + } + } + + val columnValues = values.map { value => + value match { + case x: String => "'" + x.replace("'", "''") + "'" + case x: Timestamp => "'" + dateFormat.format(x) + "'" + case null => "NULL" + case x => x + } + } + sb.append(columnValues.mkString(", ")) + sb.append(");\n") } - val columnValues = values.map { value => - value match { - case x: String => "'" + x.replace("'", "''") + "'" - case x: Timestamp => "'" + dateFormat.format(x) + "'" - case null => "NULL" - case x => x - } - } - sb.append(columnValues.mkString(", ")) - sb.append(");\n") + out.write(sb.toString.getBytes("UTF-8")) } - - out.write(sb.toString.getBytes("UTF-8")) } } } + def allTableNames(): Seq[String] = { + using(conn.getMetaData.getTables(null, null, "%", Seq("TABLE").toArray)) { rs => + val tableNames = new ListBuffer[String] + while (rs.next) { + val name = rs.getString("TABLE_NAME") + if (name != "VERSIONS") { + tableNames += name + } + } + tableNames.toSeq + } + } + private def parentTables(meta: DatabaseMetaData, tableName: String): Seq[String] = { using(meta.getImportedKeys(null, null, tableName)) { rs => val parents = new ListBuffer[String] @@ -111,23 +130,18 @@ object JDBCUtil { parents += tableName parents ++= parentTables(meta, tableName) } - parents.toSeq + parents.distinct.toSeq } } - private def allTables(meta: DatabaseMetaData): Seq[String] = { - using(meta.getTables(null, null, "%", Seq("TABLE").toArray)) { rs => - val tables = new ListBuffer[(String, Seq[String])] - while (rs.next) { - val name = rs.getString("TABLE_NAME") - if(name != "VERSIONS") { - tables += ((name, parentTables(meta, name))) - } - } - tables.sortWith { (a, b) => b._2.contains(a._1) }.map(_._1).toSeq + private def allTablesOrderByDependencies(meta: DatabaseMetaData): Seq[String] = { + val tables = allTableNames.map { tableName => + ((tableName, parentTables(meta, tableName))) } + tables.sortWith { (a, b) => + b._2.contains(a._1) || !a._2.contains(b._1) + }.map(_._1) } - } } From 2525bbafa867ab6cdf64f2e996844da9f3c35326 Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Wed, 13 Apr 2016 11:15:42 +0900 Subject: [PATCH 20/47] Database export tool is available --- .../controller/SystemSettingsController.scala | 28 ++++++++++++++++++- .../scala/gitbucket/core/util/JDBCUtil.scala | 11 ++++---- .../gitbucket/core/admin/data.scala.html | 28 +++++++++++++++++++ .../gitbucket/core/admin/menu.scala.html | 3 ++ 4 files changed, 64 insertions(+), 6 deletions(-) create mode 100644 src/main/twirl/gitbucket/core/admin/data.scala.html diff --git a/src/main/scala/gitbucket/core/controller/SystemSettingsController.scala b/src/main/scala/gitbucket/core/controller/SystemSettingsController.scala index 157a584b2..c79d374f3 100644 --- a/src/main/scala/gitbucket/core/controller/SystemSettingsController.scala +++ b/src/main/scala/gitbucket/core/controller/SystemSettingsController.scala @@ -1,5 +1,7 @@ package gitbucket.core.controller +import java.io.FileInputStream + import gitbucket.core.admin.html import gitbucket.core.service.{AccountService, SystemSettingsService, RepositoryService} import gitbucket.core.util.AdminAuthenticator @@ -11,7 +13,7 @@ import gitbucket.core.util.ControlUtil._ import gitbucket.core.util.Directory._ import gitbucket.core.util.StringUtil._ import io.github.gitbucket.scalatra.forms._ -import org.apache.commons.io.FileUtils +import org.apache.commons.io.{IOUtils, FileUtils} import org.scalatra.i18n.Messages class SystemSettingsController extends SystemSettingsControllerBase @@ -74,6 +76,7 @@ trait SystemSettingsControllerBase extends AccountManagementControllerBase { case class PluginForm(pluginIds: List[String]) + case class DataExportForm(tableNames: List[String]) case class NewUserForm(userName: String, password: String, fullName: String, mailAddress: String, isAdmin: Boolean, @@ -268,6 +271,29 @@ trait SystemSettingsControllerBase extends AccountManagementControllerBase { } }) + get("/admin/data")(adminOnly { + import gitbucket.core.util.JDBCUtil._ + val session = request2Session(request) + html.data(session.conn.allTableNames()) + }) + + post("/admin/data")(adminOnly { + import gitbucket.core.util.JDBCUtil._ + val session = request2Session(request) + val file = session.conn.export(request.getParameterValues("tableNames").toSeq) + + contentType = "application/octet-stream" + response.setHeader("Content-Disposition", "attachment; filename=" + file.getName) + response.setContentLength(file.length.toInt) + + using(new FileInputStream(file)){ in => + IOUtils.copy(in, response.outputStream) + } + + () + }) + + private def members: Constraint = new Constraint(){ override def validate(name: String, value: String, messages: Messages): Option[String] = { if(value.split(",").exists { diff --git a/src/main/scala/gitbucket/core/util/JDBCUtil.scala b/src/main/scala/gitbucket/core/util/JDBCUtil.scala index fd65037dd..7c4d19c39 100644 --- a/src/main/scala/gitbucket/core/util/JDBCUtil.scala +++ b/src/main/scala/gitbucket/core/util/JDBCUtil.scala @@ -1,6 +1,6 @@ package gitbucket.core.util -import java.io.FileOutputStream +import java.io._ import java.sql._ import java.text.SimpleDateFormat import ControlUtil._ @@ -60,18 +60,17 @@ object JDBCUtil { } } - def export(targetTables: Seq[String]): Unit = { + def export(targetTables: Seq[String]): File = { val dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss") + val file = File.createTempFile("gitbucket-export-", ".sql") - using(new FileOutputStream("export.sql")) { out => + using(new FileOutputStream(file)) { out => val dbMeta = conn.getMetaData val allTablesInDatabase = allTablesOrderByDependencies(dbMeta) allTablesInDatabase.foreach { tableName => if (targetTables.contains(tableName)) { val sb = new StringBuilder() - sb.append("DELETE FROM ACCOUNT WHERE USER_NAME = 'root';\n") - select(s"SELECT * FROM ${tableName}") { rs => sb.append(s"INSERT INTO ${tableName} (") @@ -107,6 +106,8 @@ object JDBCUtil { } } } + + file } def allTableNames(): Seq[String] = { diff --git a/src/main/twirl/gitbucket/core/admin/data.scala.html b/src/main/twirl/gitbucket/core/admin/data.scala.html new file mode 100644 index 000000000..7cfd90379 --- /dev/null +++ b/src/main/twirl/gitbucket/core/admin/data.scala.html @@ -0,0 +1,28 @@ +@(tableNames: Seq[String])(implicit context: gitbucket.core.controller.Context) +@import context._ +@import gitbucket.core.view.helpers._ +@html.main("Data export / import"){ + @admin.html.menu("data") { +
+
Export
+
+
+ @tableNames.map { tableName => +
+ +
+ } + +
+
+
+ +
+
Import
+
+
+
+ } +} diff --git a/src/main/twirl/gitbucket/core/admin/menu.scala.html b/src/main/twirl/gitbucket/core/admin/menu.scala.html index edc7d7b46..fedc73c61 100644 --- a/src/main/twirl/gitbucket/core/admin/menu.scala.html +++ b/src/main/twirl/gitbucket/core/admin/menu.scala.html @@ -12,6 +12,9 @@ Plugins + + Data export / import +
  • H2 Console
    • From 73228506a53d285047e8555167986933df097c5f Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Wed, 13 Apr 2016 11:34:41 +0900 Subject: [PATCH 21/47] Add data import form --- .../core/controller/SystemSettingsController.scala | 3 +-- src/main/twirl/gitbucket/core/admin/data.scala.html | 6 +++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/scala/gitbucket/core/controller/SystemSettingsController.scala b/src/main/scala/gitbucket/core/controller/SystemSettingsController.scala index c79d374f3..9b897dea1 100644 --- a/src/main/scala/gitbucket/core/controller/SystemSettingsController.scala +++ b/src/main/scala/gitbucket/core/controller/SystemSettingsController.scala @@ -277,7 +277,7 @@ trait SystemSettingsControllerBase extends AccountManagementControllerBase { html.data(session.conn.allTableNames()) }) - post("/admin/data")(adminOnly { + post("/admin/export")(adminOnly { import gitbucket.core.util.JDBCUtil._ val session = request2Session(request) val file = session.conn.export(request.getParameterValues("tableNames").toSeq) @@ -293,7 +293,6 @@ trait SystemSettingsControllerBase extends AccountManagementControllerBase { () }) - private def members: Constraint = new Constraint(){ override def validate(name: String, value: String, messages: Messages): Option[String] = { if(value.split(",").exists { diff --git a/src/main/twirl/gitbucket/core/admin/data.scala.html b/src/main/twirl/gitbucket/core/admin/data.scala.html index 7cfd90379..be7ac1332 100644 --- a/src/main/twirl/gitbucket/core/admin/data.scala.html +++ b/src/main/twirl/gitbucket/core/admin/data.scala.html @@ -6,7 +6,7 @@
    Export
    -
    + @tableNames.map { tableName =>
    From a6790b049d48787ada5db048f29cd7db87312d55 Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Fri, 29 Apr 2016 12:27:38 +0900 Subject: [PATCH 42/47] Specify charset of XML file --- src/main/scala/gitbucket/core/util/JDBCUtil.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/gitbucket/core/util/JDBCUtil.scala b/src/main/scala/gitbucket/core/util/JDBCUtil.scala index 08a62a1cb..631ad88ff 100644 --- a/src/main/scala/gitbucket/core/util/JDBCUtil.scala +++ b/src/main/scala/gitbucket/core/util/JDBCUtil.scala @@ -67,7 +67,7 @@ object JDBCUtil { conn.setAutoCommit(false) try { val factory = XMLInputFactory.newInstance() - using(factory.createXMLStreamReader(in)){ reader => + using(factory.createXMLStreamReader(in, "UTF-8")){ reader => // stateful objects var elementName = "" var insertTable = "" @@ -137,7 +137,7 @@ object JDBCUtil { val file = File.createTempFile("gitbucket-export-", ".xml") val factory = XMLOutputFactory.newInstance() - using(factory.createXMLStreamWriter(new FileOutputStream(file))){ writer => + using(factory.createXMLStreamWriter(new FileOutputStream(file), "UTF-8")){ writer => val dbMeta = conn.getMetaData val allTablesInDatabase = allTablesOrderByDependencies(dbMeta) From 7ec85cbf997f9fbaaa19f0ed2945fa50a4ee6159 Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Fri, 29 Apr 2016 12:52:10 +0900 Subject: [PATCH 43/47] Add error checking for data import --- .../core/controller/FileUploadController.scala | 11 ++++++++--- .../twirl/gitbucket/core/admin/data.scala.html | 15 +++++++++++---- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/main/scala/gitbucket/core/controller/FileUploadController.scala b/src/main/scala/gitbucket/core/controller/FileUploadController.scala index fe9b456c4..f52491ae6 100644 --- a/src/main/scala/gitbucket/core/controller/FileUploadController.scala +++ b/src/main/scala/gitbucket/core/controller/FileUploadController.scala @@ -10,6 +10,7 @@ import gitbucket.core.util.Implicits._ import org.eclipse.jgit.api.Git import org.eclipse.jgit.dircache.DirCache import org.eclipse.jgit.lib.{FileMode, Constants} +import org.scalatra import org.scalatra._ import org.scalatra.servlet.{MultipartConfig, FileUploadSupport, FileItem} import org.apache.commons.io.{IOUtils, FileUtils} @@ -80,9 +81,13 @@ class FileUploadController extends ScalatraServlet with FileUploadSupport with R post("/import") { session.get(Keys.Session.LoginAccount).collect { case loginAccount: Account if loginAccount.isAdmin => execute({ (file, fileId) => - import JDBCUtil._ - val conn = request2Session(request).conn - conn.importAsXML(file.getInputStream) + if(file.getName.endsWith(".xml")){ + import JDBCUtil._ + val conn = request2Session(request).conn + conn.importAsXML(file.getInputStream) + } else { + throw new RuntimeException("Import is available for only the XML file.") + } }, _ => true) } redirect("/admin/data") diff --git a/src/main/twirl/gitbucket/core/admin/data.scala.html b/src/main/twirl/gitbucket/core/admin/data.scala.html index 44f590fdf..4fcbde6af 100644 --- a/src/main/twirl/gitbucket/core/admin/data.scala.html +++ b/src/main/twirl/gitbucket/core/admin/data.scala.html @@ -30,10 +30,10 @@
    -
    Import
    +
    Import (only XML)
    -
    - + +
    @@ -42,7 +42,14 @@ }