Nemcio/Gitea
1
0
Fork 0
mirror of https://github.com/go-gitea/gitea.git synced 2025-10-31 19:06:18 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fix all possible setting error related storages and added some tests (#23911)

Browse Source 
Follow up #22405

Fix #20703 

This PR rewrites storage configuration read sequences with some breaks
and tests. It becomes more strict than before and also fixed some
inherit problems.

- Move storage's MinioConfig struct into setting, so after the
configuration loading, the values will be stored into the struct but not
still on some section.
- All storages configurations should be stored on one section,
configuration items cannot be overrided by multiple sections. The
prioioty of configuration is `[attachment]` > `[storage.attachments]` |
`[storage.customized]` > `[storage]` > `default`
- For extra override configuration items, currently are `SERVE_DIRECT`,
`MINIO_BASE_PATH`, `MINIO_BUCKET`, which could be configured in another
section. The prioioty of the override configuration is `[attachment]` >
`[storage.attachments]` > `default`.
- Add more tests for storages configurations.
- Update the storage documentations.

---------

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
This commit is contained in:
Lunny Xiao
2023-06-14 11:42:38 +08:00
committed by GitHub
parent dc0a7168f1
commit d6dd6d641b
41 changed files with 1152 additions and 452 deletions
Download Patch File Download Diff File Expand all files Collapse all files

215
modules/setting/storage.go
View File

@@ -4,87 +4,182 @@
package setting
import (
"errors"
"fmt"
"path/filepath"
"reflect"
)
// StorageType is a type of Storage
type StorageType string
const (
// LocalStorageType is the type descriptor for local storage
LocalStorageType StorageType = "local"
// MinioStorageType is the type descriptor for minio storage
MinioStorageType StorageType = "minio"
)
var storageTypes = []StorageType{
LocalStorageType,
MinioStorageType,
}
// IsValidStorageType returns true if the given storage type is valid
func IsValidStorageType(storageType StorageType) bool {
for _, t := range storageTypes {
if t == storageType {
return true
}
}
return false
}
// MinioStorageConfig represents the configuration for a minio storage
type MinioStorageConfig struct {
Endpoint string `ini:"MINIO_ENDPOINT" json:",omitempty"`
AccessKeyID string `ini:"MINIO_ACCESS_KEY_ID" json:",omitempty"`
SecretAccessKey string `ini:"MINIO_SECRET_ACCESS_KEY" json:",omitempty"`
Bucket string `ini:"MINIO_BUCKET" json:",omitempty"`
Location string `ini:"MINIO_LOCATION" json:",omitempty"`
BasePath string `ini:"MINIO_BASE_PATH" json:",omitempty"`
UseSSL bool `ini:"MINIO_USE_SSL"`
InsecureSkipVerify bool `ini:"MINIO_INSECURE_SKIP_VERIFY"`
ChecksumAlgorithm string `ini:"MINIO_CHECKSUM_ALGORITHM" json:",omitempty"`
ServeDirect bool `ini:"SERVE_DIRECT"`
}
// Storage represents configuration of storages
type Storage struct {
Type string
Path string
Section ConfigSection
ServeDirect bool
Type StorageType // local or minio
Path string `json:",omitempty"` // for local type
TemporaryPath string `json:",omitempty"`
MinioConfig MinioStorageConfig // for minio type
}
// MapTo implements the Mappable interface
func (s *Storage) MapTo(v interface{}) error {
pathValue := reflect.ValueOf(v).Elem().FieldByName("Path")
if pathValue.IsValid() && pathValue.Kind() == reflect.String {
pathValue.SetString(s.Path)
func (storage *Storage) ToShadowCopy() Storage {
shadowStorage := *storage
if shadowStorage.MinioConfig.AccessKeyID != "" {
shadowStorage.MinioConfig.AccessKeyID = "******"
}
if s.Section != nil {
return s.Section.MapTo(v)
if shadowStorage.MinioConfig.SecretAccessKey != "" {
shadowStorage.MinioConfig.SecretAccessKey = "******"
}
return nil
return shadowStorage
}
func getStorage(rootCfg ConfigProvider, name, typ string, targetSec ConfigSection) Storage {
const sectionName = "storage"
sec := rootCfg.Section(sectionName)
const storageSectionName = "storage"
func getDefaultStorageSection(rootCfg ConfigProvider) ConfigSection {
storageSec := rootCfg.Section(storageSectionName)
// Global Defaults
sec.Key("MINIO_ENDPOINT").MustString("localhost:9000")
sec.Key("MINIO_ACCESS_KEY_ID").MustString("")
sec.Key("MINIO_SECRET_ACCESS_KEY").MustString("")
sec.Key("MINIO_BUCKET").MustString("gitea")
sec.Key("MINIO_LOCATION").MustString("us-east-1")
sec.Key("MINIO_USE_SSL").MustBool(false)
sec.Key("MINIO_INSECURE_SKIP_VERIFY").MustBool(false)
sec.Key("MINIO_CHECKSUM_ALGORITHM").MustString("default")
storageSec.Key("STORAGE_TYPE").MustString("local")
storageSec.Key("MINIO_ENDPOINT").MustString("localhost:9000")
storageSec.Key("MINIO_ACCESS_KEY_ID").MustString("")
storageSec.Key("MINIO_SECRET_ACCESS_KEY").MustString("")
storageSec.Key("MINIO_BUCKET").MustString("gitea")
storageSec.Key("MINIO_LOCATION").MustString("us-east-1")
storageSec.Key("MINIO_USE_SSL").MustBool(false)
storageSec.Key("MINIO_INSECURE_SKIP_VERIFY").MustBool(false)
storageSec.Key("MINIO_CHECKSUM_ALGORITHM").MustString("default")
return storageSec
}
func getStorage(rootCfg ConfigProvider, name, typ string, sec ConfigSection) (*Storage, error) {
if name == "" {
return nil, errors.New("no name for storage")
}
var targetSec ConfigSection
if typ != "" {
var err error
targetSec, err = rootCfg.GetSection(storageSectionName + "." + typ)
if err != nil {
if !IsValidStorageType(StorageType(typ)) {
return nil, fmt.Errorf("get section via storage type %q failed: %v", typ, err)
}
}
if targetSec != nil {
targetType := targetSec.Key("STORAGE_TYPE").String()
if targetType == "" {
if !IsValidStorageType(StorageType(typ)) {
return nil, fmt.Errorf("unknow storage type %q", typ)
}
targetSec.Key("STORAGE_TYPE").SetValue(typ)
} else if !IsValidStorageType(StorageType(targetType)) {
return nil, fmt.Errorf("unknow storage type %q for section storage.%v", targetType, typ)
}
}
}
storageNameSec, _ := rootCfg.GetSection(storageSectionName + "." + name)
if targetSec == nil {
targetSec, _ = rootCfg.NewSection(name)
targetSec = sec
}
if targetSec == nil {
targetSec = storageNameSec
}
if targetSec == nil {
targetSec = getDefaultStorageSection(rootCfg)
} else {
targetType := targetSec.Key("STORAGE_TYPE").String()
switch {
case targetType == "":
if targetSec.Key("PATH").String() == "" {
targetSec = getDefaultStorageSection(rootCfg)
} else {
targetSec.Key("STORAGE_TYPE").SetValue("local")
}
default:
newTargetSec, _ := rootCfg.GetSection(storageSectionName + "." + targetType)
if newTargetSec == nil {
if !IsValidStorageType(StorageType(targetType)) {
return nil, fmt.Errorf("invalid storage section %s.%q", storageSectionName, targetType)
}
} else {
targetSec = newTargetSec
if IsValidStorageType(StorageType(targetType)) {
tp := targetSec.Key("STORAGE_TYPE").String()
if tp == "" {
targetSec.Key("STORAGE_TYPE").SetValue(targetType)
}
}
}
}
}
targetType := targetSec.Key("STORAGE_TYPE").String()
if !IsValidStorageType(StorageType(targetType)) {
return nil, fmt.Errorf("invalid storage type %q", targetType)
}
var storage Storage
storage.Section = targetSec
storage.Type = typ
storage.Type = StorageType(targetType)
overrides := make([]ConfigSection, 0, 3)
nameSec, err := rootCfg.GetSection(sectionName + "." + name)
if err == nil {
overrides = append(overrides, nameSec)
}
switch targetType {
case string(LocalStorageType):
storage.Path = ConfigSectionKeyString(targetSec, "PATH", filepath.Join(AppDataPath, name))
if !filepath.IsAbs(storage.Path) {
storage.Path = filepath.Join(AppWorkPath, storage.Path)
}
case string(MinioStorageType):
storage.MinioConfig.BasePath = name + "/"
typeSec, err := rootCfg.GetSection(sectionName + "." + typ)
if err == nil {
overrides = append(overrides, typeSec)
nextType := typeSec.Key("STORAGE_TYPE").String()
if len(nextType) > 0 {
storage.Type = nextType // Support custom STORAGE_TYPE
if err := targetSec.MapTo(&storage.MinioConfig); err != nil {
return nil, fmt.Errorf("map minio config failed: %v", err)
}
// extra config section will be read SERVE_DIRECT, PATH, MINIO_BASE_PATH to override the targetsec
extraConfigSec := sec
if extraConfigSec == nil {
extraConfigSec = storageNameSec
}
if extraConfigSec != nil {
storage.MinioConfig.ServeDirect = ConfigSectionKeyBool(extraConfigSec, "SERVE_DIRECT", storage.MinioConfig.ServeDirect)
storage.MinioConfig.BasePath = ConfigSectionKeyString(extraConfigSec, "MINIO_BASE_PATH", storage.MinioConfig.BasePath)
storage.MinioConfig.Bucket = ConfigSectionKeyString(extraConfigSec, "MINIO_BUCKET", storage.MinioConfig.Bucket)
}
}
overrides = append(overrides, sec)
for _, override := range overrides {
for _, key := range override.Keys() {
if !targetSec.HasKey(key.Name()) {
_, _ = targetSec.NewKey(key.Name(), key.Value())
}
}
if len(storage.Type) == 0 {
storage.Type = override.Key("STORAGE_TYPE").String()
}
}
storage.ServeDirect = storage.Section.Key("SERVE_DIRECT").MustBool(false)
// Specific defaults
storage.Path = storage.Section.Key("PATH").MustString(filepath.Join(AppDataPath, name))
if !filepath.IsAbs(storage.Path) {
storage.Path = filepath.Join(AppWorkPath, storage.Path)
storage.Section.Key("PATH").SetValue(storage.Path)
}
storage.Section.Key("MINIO_BASE_PATH").MustString(name + "/")
return storage
return &storage, nil
}