Compare commits

..

4 Commits

Author SHA1 Message Date
ᴜɴᴋɴᴡᴏɴ
f0e3cd90f8 release: update version to 0.12.3 2020-10-07 22:39:36 +08:00
ᴜɴᴋɴᴡᴏɴ
fbe34c8c61 lfs: ask client to always send the same value for the HTTP header (#6369)
# Conflicts:
#	go.sum
2020-10-06 13:12:56 +08:00
Jeff
e6b4c467e8 markup: render SHA links without branch prefix (#6350)
Co-authored-by: Zhukov Roman <zhukov.roman@gmail.com>
Co-authored-by: ᴜɴᴋɴᴡᴏɴ <u@gogs.io>
2020-10-06 13:08:27 +08:00
ᴜɴᴋɴᴡᴏɴ
6b6bfe8bb0 cmd/serv: use different log files for ORMs in hook mode (#6361)
# Conflicts:
#	CHANGELOG.md
#	go.sum
#	internal/db/db.go
#	internal/db/models.go
2020-09-29 22:36:02 +08:00
14 changed files with 174 additions and 50 deletions

View File

@@ -20,6 +20,9 @@ All notable changes to Gogs are documented in this file.
- _Regression:_ Submodule with a relative path is linked correctly. [#6319](https://github.com/gogs/gogs/issues/6319)
- Backup can be processed when `--target` is specified on Windows. [#6339](https://github.com/gogs/gogs/issues/6339)
- Commit message contains keywords look like an issue reference no longer fails the push entirely. [#6289](https://github.com/gogs/gogs/issues/6289)
- _Regression:_ When running Gogs on Windows, push commits no longer fail on a daily basis with the error "pre-receive hook declined". [#6316](https://github.com/gogs/gogs/issues/6316)
- Auto-linked commit SHAs now have correct links. [#6300](https://github.com/gogs/gogs/issues/6300)
- Git LFS client (with version >= 2.5.0) wasn't able to upload files with known format (e.g. PNG, JPEG), and the server is expecting the HTTP Header `Content-Type` to be `application/octet-stream`. The server now tells the LFS client to always use `Content-Type: application/octet-stream` when upload files.
### Removed

7
go.sum
View File

@@ -354,7 +354,6 @@ golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslY
golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -401,7 +400,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE=
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
gopkg.in/ini.v1 v1.46.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.55.0 h1:E8yzL5unfpW3M6fz/eB7Cb5MQAYSZ7GKo4Qth+N2sgQ=
gopkg.in/ini.v1 v1.55.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.56.0 h1:DPMeDvGTM54DXbPkVIZsp19fp/I2K7zwA/itHYHKo8Y=
gopkg.in/ini.v1 v1.56.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
@@ -409,10 +407,6 @@ gopkg.in/ldap.v2 v2.5.1 h1:wiu0okdNfjlBzg6UWvd1Hn8Y+Ux17/u/4nlk4CQr6tU=
gopkg.in/ldap.v2 v2.5.1/go.mod h1:oI0cpe/D7HRtBQl8aTg+ZmzFUAvu4lsv3eLXMLGFxWk=
gopkg.in/macaron.v1 v1.3.4/go.mod h1:/RoHTdC8ALpyJ3+QR36mKjwnT1F1dyYtsGM9Ate6ZFI=
gopkg.in/macaron.v1 v1.3.5/go.mod h1:uMZCFccv9yr5TipIalVOyAyZQuOH3OkmXvgcWwhJuP4=
gopkg.in/macaron.v1 v1.3.6 h1:mHw5qRGj2f86sCjBV/S5OOPvPbiZpu0OEs1F5l66fYI=
gopkg.in/macaron.v1 v1.3.6/go.mod h1:uMZCFccv9yr5TipIalVOyAyZQuOH3OkmXvgcWwhJuP4=
gopkg.in/macaron.v1 v1.3.8 h1:45bQT4dDF+5SLd75qlj+dulQsuFQ9aFyLlvZX6YUJWE=
gopkg.in/macaron.v1 v1.3.8/go.mod h1:uMZCFccv9yr5TipIalVOyAyZQuOH3OkmXvgcWwhJuP4=
gopkg.in/macaron.v1 v1.3.9 h1:Dw+DDRYdXgQyEsPlfAfKz+UA5qVUrH3KPD7JhmZ9MFc=
gopkg.in/macaron.v1 v1.3.9/go.mod h1:uMZCFccv9yr5TipIalVOyAyZQuOH3OkmXvgcWwhJuP4=
gopkg.in/redis.v2 v2.3.2 h1:GPVIIB/JnL1wvfULefy3qXmPu1nfNu2d0yA09FHgwfs=
@@ -422,7 +416,6 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

View File

@@ -18,7 +18,7 @@ import (
)
func init() {
conf.App.Version = "0.12.2"
conf.App.Version = "0.12.3"
}
func main() {

View File

@@ -66,7 +66,7 @@ func runHookPreReceive(c *cli.Context) error {
if len(os.Getenv("SSH_ORIGINAL_COMMAND")) == 0 {
return nil
}
setup(c, "hooks/pre-receive.log", true)
setup(c, "pre-receive.log", true)
isWiki := strings.Contains(os.Getenv(db.ENV_REPO_CUSTOM_HOOKS_PATH), ".wiki.git/")
@@ -159,7 +159,7 @@ func runHookUpdate(c *cli.Context) error {
if len(os.Getenv("SSH_ORIGINAL_COMMAND")) == 0 {
return nil
}
setup(c, "hooks/update.log", false)
setup(c, "update.log", false)
args := c.Args()
if len(args) != 3 {
@@ -193,7 +193,7 @@ func runHookPostReceive(c *cli.Context) error {
if len(os.Getenv("SSH_ORIGINAL_COMMAND")) == 0 {
return nil
}
setup(c, "hooks/post-receive.log", true)
setup(c, "post-receive.log", true)
// Post-receive hook does more than just gather Git information,
// so we need to setup additional services for email notifications.

View File

@@ -38,7 +38,7 @@ var Serv = cli.Command{
// logs error message on the server side. When not in "prod" mode,
// error message is also printed to the client for easier debugging.
func fail(userMessage, errMessage string, args ...interface{}) {
fmt.Fprintln(os.Stderr, "Gogs:", userMessage)
_, _ = fmt.Fprintln(os.Stderr, "Gogs:", userMessage)
if len(errMessage) > 0 {
if !conf.IsProdMode() {
@@ -47,10 +47,11 @@ func fail(userMessage, errMessage string, args ...interface{}) {
log.Error(errMessage, args...)
}
log.Stop()
os.Exit(1)
}
func setup(c *cli.Context, logPath string, connectDB bool) {
func setup(c *cli.Context, logFile string, connectDB bool) {
conf.HookMode = true
var customConf string
@@ -73,7 +74,7 @@ func setup(c *cli.Context, logPath string, connectDB bool) {
err = log.NewFile(log.FileConfig{
Level: level,
Filename: filepath.Join(conf.Log.RootPath, logPath),
Filename: filepath.Join(conf.Log.RootPath, "hooks", logFile),
FileRotationConfig: log.FileRotationConfig{
Rotate: true,
Daily: true,

View File

@@ -128,7 +128,7 @@ var tables = []interface{}{
new(LFSObject), new(LoginSource),
}
func Init() (*gorm.DB, error) {
func Init(w io.Writer) (*gorm.DB, error) {
db, err := openDB(conf.Database)
if err != nil {
return nil, errors.Wrap(err, "open database")
@@ -138,10 +138,6 @@ func Init() (*gorm.DB, error) {
db.DB().SetMaxIdleConns(conf.Database.MaxIdleConns)
db.DB().SetConnMaxLifetime(time.Minute)
w, err := getLogWriter()
if err != nil {
return nil, errors.Wrap(err, "get log writer")
}
db.SetLogger(&dbutil.Writer{Writer: w})
if !conf.IsProdMode() {
db = db.LogMode(true)

View File

@@ -7,13 +7,16 @@ package db
import (
"database/sql"
"fmt"
"io"
"net/url"
"os"
"path"
"path/filepath"
"strings"
"time"
"github.com/jinzhu/gorm"
"github.com/pkg/errors"
log "unknwon.dev/clog/v2"
"xorm.io/core"
"xorm.io/xorm"
@@ -130,16 +133,21 @@ func SetEngine() (*gorm.DB, error) {
x.SetMapper(core.GonicMapper{})
// WARNING: for serv command, MUST remove the output to os.stdout,
// so use log file to instead print to stdout.
var logPath string
if conf.HookMode {
logPath = filepath.Join(conf.Log.RootPath, "hooks", "xorm.log")
} else {
logPath = filepath.Join(conf.Log.RootPath, "xorm.log")
}
sec := conf.File.Section("log.xorm")
logger, err := log.NewFileWriter(path.Join(conf.Log.RootPath, "xorm.log"),
fileWriter, err := log.NewFileWriter(logPath,
log.FileRotationConfig{
Rotate: sec.Key("ROTATE").MustBool(true),
Daily: sec.Key("ROTATE_DAILY").MustBool(true),
MaxSize: sec.Key("MAX_SIZE").MustInt64(100) * 1024 * 1024,
MaxDays: sec.Key("MAX_DAYS").MustInt64(3),
})
},
)
if err != nil {
return nil, fmt.Errorf("create 'xorm.log': %v", err)
}
@@ -149,12 +157,22 @@ func SetEngine() (*gorm.DB, error) {
x.SetConnMaxLifetime(time.Second)
if conf.IsProdMode() {
x.SetLogger(xorm.NewSimpleLogger3(logger, xorm.DEFAULT_LOG_PREFIX, xorm.DEFAULT_LOG_FLAG, core.LOG_WARNING))
x.SetLogger(xorm.NewSimpleLogger3(fileWriter, xorm.DEFAULT_LOG_PREFIX, xorm.DEFAULT_LOG_FLAG, core.LOG_WARNING))
} else {
x.SetLogger(xorm.NewSimpleLogger(logger))
x.SetLogger(xorm.NewSimpleLogger(fileWriter))
}
x.ShowSQL(true)
return Init()
var w io.Writer
if conf.HookMode {
w = fileWriter
} else {
w, err = getLogWriter()
if err != nil {
return nil, errors.Wrap(err, "get log writer")
}
}
return Init(w)
}
func NewEngine() (err error) {

View File

@@ -426,24 +426,29 @@ func (repo *Repository) UpdateSize() error {
return nil
}
// ComposeMetas composes a map of metas for rendering external issue tracker URL.
// ComposeMetas composes a map of metas for rendering SHA1 URL and external issue tracker URL.
func (repo *Repository) ComposeMetas() map[string]string {
if !repo.EnableExternalTracker {
return nil
} else if repo.ExternalMetas == nil {
repo.ExternalMetas = map[string]string{
"format": repo.ExternalTrackerFormat,
"user": repo.MustOwner().Name,
"repo": repo.Name,
}
if repo.ExternalMetas != nil {
return repo.ExternalMetas
}
repo.ExternalMetas = map[string]string{
"repoLink": repo.Link(),
}
if repo.EnableExternalTracker {
repo.ExternalMetas["user"] = repo.MustOwner().Name
repo.ExternalMetas["repo"] = repo.Name
repo.ExternalMetas["format"] = repo.ExternalTrackerFormat
switch repo.ExternalTrackerStyle {
case markup.ISSUE_NAME_STYLE_ALPHANUMERIC:
repo.ExternalMetas["style"] = markup.ISSUE_NAME_STYLE_ALPHANUMERIC
default:
repo.ExternalMetas["style"] = markup.ISSUE_NAME_STYLE_NUMERIC
}
}
return repo.ExternalMetas
}

View File

@@ -19,14 +19,19 @@ func TestRepository_ComposeMetas(t *testing.T) {
t.Run("no external tracker is configured", func(t *testing.T) {
repo.EnableExternalTracker = false
assert.Equal(t, map[string]string(nil), repo.ComposeMetas())
// Should be nil even if other settings are present
repo.ExternalTrackerStyle = markup.ISSUE_NAME_STYLE_NUMERIC
assert.Equal(t, map[string]string(nil), repo.ComposeMetas())
metas := repo.ComposeMetas()
assert.Equal(t, metas["repoLink"], repo.Link())
// Should no format and style if no external tracker is configured
_, ok := metas["format"]
assert.False(t, ok)
_, ok = metas["style"]
assert.False(t, ok)
})
t.Run("an external issue tracker is configured", func(t *testing.T) {
repo.ExternalMetas = nil
repo.EnableExternalTracker = true
// Default to numeric issue style

View File

@@ -145,7 +145,8 @@ func RenderSha1CurrentPattern(rawBytes []byte, urlPrefix string) []byte {
if com.StrTo(m).MustInt() > 0 {
return m
}
return fmt.Sprintf(`<a href="%s/commit/%s"><code>%s</code></a>`, urlPrefix, m, tool.ShortSHA1(string(m)))
return fmt.Sprintf(`<a href="%s/commit/%s"><code>%s</code></a>`, urlPrefix, m, tool.ShortSHA1(m))
}))
}
@@ -160,7 +161,7 @@ func RenderSpecialLink(rawBytes []byte, urlPrefix string, metas map[string]strin
rawBytes = RenderIssueIndexPattern(rawBytes, urlPrefix, metas)
rawBytes = RenderCrossReferenceIssueIndexPattern(rawBytes, urlPrefix, metas)
rawBytes = RenderSha1CurrentPattern(rawBytes, urlPrefix)
rawBytes = RenderSha1CurrentPattern(rawBytes, metas["repoLink"])
return rawBytes
}

View File

@@ -215,3 +215,41 @@ func Test_RenderIssueIndexPattern(t *testing.T) {
})
})
}
func TestRenderSha1CurrentPattern(t *testing.T) {
metas := map[string]string{
"repoLink": "/someuser/somerepo",
}
tests := []struct {
desc string
input string
prefix string
expVal string
}{
{
desc: "Full SHA (40 symbols)",
input: "ad8ced4f57d9068cb2874557245be3c7f341149d",
prefix: metas["repoLink"],
expVal: `<a href="/someuser/somerepo/commit/ad8ced4f57d9068cb2874557245be3c7f341149d"><code>ad8ced4f57</code></a>`,
},
{
desc: "Short SHA (8 symbols)",
input: "ad8ced4f",
prefix: metas["repoLink"],
expVal: `<a href="/someuser/somerepo/commit/ad8ced4f"><code>ad8ced4f</code></a>`,
},
{
desc: "9 digits",
input: "123456789",
prefix: metas["repoLink"],
expVal: "123456789",
},
}
for _, test := range tests {
t.Run(test.desc, func(t *testing.T) {
assert.Equal(t, test.expVal, string(RenderSha1CurrentPattern([]byte(test.input), test.prefix)))
})
}
}

View File

@@ -44,6 +44,11 @@ func serveBatch(c *macaron.Context, owner *db.User, repo *db.Repository) {
actions = batchActions{
Upload: &batchAction{
Href: fmt.Sprintf("%s/%s", baseHref, obj.Oid),
Header: map[string]string{
// NOTE: git-lfs v2.5.0 sets the Content-Type based on the uploaded file.
// This ensures that the client always uses the designated value for the header.
"Content-Type": "application/octet-stream",
},
},
Verify: &batchAction{
Href: fmt.Sprintf("%s/verify", baseHref),
@@ -136,7 +141,8 @@ type batchError struct {
}
type batchAction struct {
Href string `json:"href"`
Href string `json:"href"`
Header map[string]string `json:"header,omitempty"`
}
type batchActions struct {

View File

@@ -6,6 +6,7 @@ package lfs
import (
"bytes"
"encoding/json"
"io/ioutil"
"net/http"
"net/http/httptest"
@@ -42,7 +43,7 @@ func Test_serveBatch(t *testing.T) {
name: "unrecognized operation",
body: `{"operation": "update"}`,
expStatusCode: http.StatusBadRequest,
expBody: `{"message":"Operation not recognized"}` + "\n",
expBody: `{"message": "Operation not recognized"}` + "\n",
},
{
name: "upload: contains invalid oid",
@@ -53,7 +54,25 @@ func Test_serveBatch(t *testing.T) {
{"oid": "ef797c8118f02dfb649607dd5d3f8c7623048c9c063d532cc95c5ed7a898a64f", "size": 123}
]}`,
expStatusCode: http.StatusOK,
expBody: `{"transfer":"basic","objects":[{"oid":"bad_oid","size":123,"actions":{"error":{"code":422,"message":"Object has invalid oid"}}},{"oid":"ef797c8118f02dfb649607dd5d3f8c7623048c9c063d532cc95c5ed7a898a64f","size":123,"actions":{"upload":{"href":"https://gogs.example.com/owner/repo.git/info/lfs/objects/basic/ef797c8118f02dfb649607dd5d3f8c7623048c9c063d532cc95c5ed7a898a64f"},"verify":{"href":"https://gogs.example.com/owner/repo.git/info/lfs/objects/basic/verify"}}}]}` + "\n",
expBody: `{
"transfer": "basic",
"objects": [
{"oid": "bad_oid", "size":123, "actions": {"error": {"code": 422, "message": "Object has invalid oid"}}},
{
"oid": "ef797c8118f02dfb649607dd5d3f8c7623048c9c063d532cc95c5ed7a898a64f",
"size": 123,
"actions": {
"upload": {
"href": "https://gogs.example.com/owner/repo.git/info/lfs/objects/basic/ef797c8118f02dfb649607dd5d3f8c7623048c9c063d532cc95c5ed7a898a64f",
"header": {"Content-Type": "application/octet-stream"}
},
"verify": {
"href": "https://gogs.example.com/owner/repo.git/info/lfs/objects/basic/verify"
}
}
}
]
}` + "\n",
},
{
name: "download: contains non-existent oid and mismatched size",
@@ -78,7 +97,26 @@ func Test_serveBatch(t *testing.T) {
},
},
expStatusCode: http.StatusOK,
expBody: `{"transfer":"basic","objects":[{"oid":"bad_oid","size":123,"actions":{"error":{"code":404,"message":"Object does not exist"}}},{"oid":"ef797c8118f02dfb649607dd5d3f8c7623048c9c063d532cc95c5ed7a898a64f","size":123,"actions":{"error":{"code":422,"message":"Object size mismatch"}}},{"oid":"5cac0a318669fadfee734fb340a5f5b70b428ac57a9f4b109cb6e150b2ba7e57","size":456,"actions":{"download":{"href":"https://gogs.example.com/owner/repo.git/info/lfs/objects/basic/5cac0a318669fadfee734fb340a5f5b70b428ac57a9f4b109cb6e150b2ba7e57"}}}]}` + "\n",
expBody: `{
"transfer": "basic",
"objects": [
{"oid": "bad_oid", "size": 123, "actions": {"error": {"code": 404, "message": "Object does not exist"}}},
{
"oid": "ef797c8118f02dfb649607dd5d3f8c7623048c9c063d532cc95c5ed7a898a64f",
"size": 123,
"actions": {"error": {"code": 422, "message": "Object size mismatch"}}
},
{
"oid": "5cac0a318669fadfee734fb340a5f5b70b428ac57a9f4b109cb6e150b2ba7e57",
"size": 456,
"actions": {
"download": {
"href": "https://gogs.example.com/owner/repo.git/info/lfs/objects/basic/5cac0a318669fadfee734fb340a5f5b70b428ac57a9f4b109cb6e150b2ba7e57"
}
}
}
]
}` + "\n",
},
}
for _, test := range tests {
@@ -100,7 +138,20 @@ func Test_serveBatch(t *testing.T) {
if err != nil {
t.Fatal(err)
}
assert.Equal(t, test.expBody, string(body))
var expBody bytes.Buffer
err = json.Indent(&expBody, []byte(test.expBody), "", " ")
if err != nil {
t.Fatal(err)
}
var gotBody bytes.Buffer
err = json.Indent(&gotBody, body, "", " ")
if err != nil {
t.Fatal(err)
}
assert.Equal(t, expBody.String(), gotBody.String())
})
}
}

View File

@@ -135,6 +135,8 @@ func authorize(mode db.AccessMode) macaron.Handler {
return
}
log.Trace("[LFS] Authorized user %q to %q", actor.Name, username+"/"+reponame)
c.Map(owner) // NOTE: Override actor
c.Map(repo)
}
@@ -144,10 +146,15 @@ func authorize(mode db.AccessMode) macaron.Handler {
// When not, response given "failCode" as status code.
func verifyHeader(key, value string, failCode int) macaron.Handler {
return func(c *macaron.Context) {
if !strings.Contains(c.Req.Header.Get(key), value) {
c.Status(failCode)
return
vals := c.Req.Header.Values(key)
for _, val := range vals {
if strings.Contains(val, value) {
return
}
}
log.Trace("[LFS] HTTP header %q does not contain value %q", key, value)
c.Status(failCode)
}
}