From f9acad82ca231b2a094879e53134b0d91815ddf0 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 18 Aug 2021 21:10:39 +0800 Subject: [PATCH] Add proxy settings and support for migration and webhook (#16704) * Add proxy settings and support for migration and webhook * Fix default value * Add newline for example ini * Add lfs proxy support * Fix lint * Follow @zeripath's review * Fix git clone * Fix test * missgin http requests for proxy * use empty Co-authored-by: zeripath Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: zeripath --- custom/conf/app.example.ini | 8 ++ .../doc/advanced/config-cheat-sheet.en-us.md | 18 +++- .../doc/advanced/config-cheat-sheet.zh-cn.md | 14 ++++ modules/git/command.go | 51 ++++++++---- modules/git/repo.go | 28 ++++++- modules/lfs/client.go | 4 +- modules/lfs/client_test.go | 4 +- modules/lfs/http_client.go | 11 ++- modules/migrations/gitea_downloader.go | 22 ++++- modules/migrations/github.go | 14 +++- modules/migrations/gitlab.go | 20 ++++- modules/migrations/gogs.go | 6 +- modules/proxy/proxy.go | 83 +++++++++++++++++++ modules/repository/repo.go | 6 +- modules/setting/migrations.go | 2 + modules/setting/proxy.go | 40 +++++++++ modules/setting/setting.go | 1 + services/mirror/mirror_pull.go | 2 +- services/mirror/mirror_push.go | 6 +- services/webhook/deliver.go | 3 +- 20 files changed, 302 insertions(+), 41 deletions(-) create mode 100644 modules/proxy/proxy.go create mode 100644 modules/setting/proxy.go diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 95dd8073a4..d0fe6150d6 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2127,3 +2127,11 @@ PATH = ;; ;; Minio enabled ssl only available when STORAGE_TYPE is `minio` ;MINIO_USE_SSL = false + +;[proxy] +;; Enable the proxy, all requests to external via HTTP will be affected +;PROXY_ENABLED = false +;; Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy/no_proxy +;PROXY_URL = +;; Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts. +;PROXY_HOSTS = diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 572e33af7f..23f125f4da 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -549,8 +549,8 @@ Define allowed algorithms and their minimum key length (use -1 to disable a type - `DELIVER_TIMEOUT`: **5**: Delivery timeout (sec) for shooting webhooks. - `SKIP_TLS_VERIFY`: **false**: Allow insecure certification. - `PAGING_NUM`: **10**: Number of webhook history events that are shown in one page. -- `PROXY_URL`: ****: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy -- `PROXY_HOSTS`: ****: Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts. +- `PROXY_URL`: **\**: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy. If not given, will use global proxy setting. +- `PROXY_HOSTS`: **\`**: Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts. If not given, will use global proxy setting. ## Mailer (`mailer`) @@ -950,6 +950,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf - `ALLOWED_DOMAINS`: **\**: Domains allowlist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas. - `BLOCKED_DOMAINS`: **\**: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWED_DOMAINS` is not blank, this option will be ignored. - `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291 +- `SKIP_TLS_VERIFY`: **false**: Allow skip tls verify ## Mirror (`mirror`) @@ -1023,6 +1024,19 @@ is `data/repo-archive` and the default of `MINIO_BASE_PATH` is `repo-archive/`. - `MINIO_BASE_PATH`: **repo-archive/**: Minio base path on the bucket only available when `STORAGE_TYPE` is `minio` - `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio` +## Proxy (`proxy`) + +- `PROXY_ENABLED`: **false**: Enable the proxy if true, all requests to external via HTTP will be affected, if false, no proxy will be used even environment http_proxy/https_proxy +- `PROXY_URL`: **\**: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy +- `PROXY_HOSTS`: **\**: Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts. + +i.e. +```ini +PROXY_ENABLED = true +PROXY_URL = socks://127.0.0.1:1080 +PROXY_HOSTS = *.github.com +``` + ## Other (`other`) - `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer. diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md index 31b34db2f7..5c3d69ecfd 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md @@ -332,6 +332,7 @@ IS_INPUT_FILE = false - `ALLOWED_DOMAINS`: **\**: 迁移仓库的域名白名单，默认为空，表示允许从任意域名迁移仓库，多个域名用逗号分隔。 - `BLOCKED_DOMAINS`: **\**: 迁移仓库的域名黑名单，默认为空，多个域名用逗号分隔。如果 `ALLOWED_DOMAINS` 不为空，此选项将会被忽略。 - `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918 +- `SKIP_TLS_VERIFY`: **false**: 允许忽略 TLS 认证 ## LFS (`lfs`) @@ -397,6 +398,19 @@ Repository archive 的存储配置。 如果 `STORAGE_TYPE` 为空，则此配 - `MINIO_BASE_PATH`: **repo-archive/**: Minio base path ，仅当 `STORAGE_TYPE` 为 `minio` 时有效。 - `MINIO_USE_SSL`: **false**: Minio 是否启用 ssl ，仅当 `STORAGE_TYPE` 为 `minio` 时有效。 +## Proxy (`proxy`) + +- `PROXY_ENABLED`: **false**: 是否启用全局代理。如果为否，则不使用代理，环境变量中的代理也不使用 +- `PROXY_URL`: **\**: 代理服务器地址，支持 http://, https//, socks://，为空则不启用代理而使用环境变量中的 http_proxy/https_proxy +- `PROXY_HOSTS`: **\**: 逗号分隔的多个需要代理的网址，支持 * 号匹配符号， ** 表示匹配所有网站 + +i.e. +```ini +PROXY_ENABLED = true +PROXY_URL = socks://127.0.0.1:1080 +PROXY_HOSTS = *.github.com +``` + ## Other (`other`) - `SHOW_FOOTER_BRANDING`: 为真则在页面底部显示Gitea的字样。 diff --git a/modules/git/command.go b/modules/git/command.go index d83c42fdc2..e7496f072c 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -110,24 +110,47 @@ func (c *Command) RunInDirTimeoutEnvFullPipeline(env []string, timeout time.Dura // RunInDirTimeoutEnvFullPipelineFunc executes the command in given directory with given timeout, // it pipes stdout and stderr to given io.Writer and passes in an io.Reader as stdin. Between cmd.Start and cmd.Wait the passed in function is run. func (c *Command) RunInDirTimeoutEnvFullPipelineFunc(env []string, timeout time.Duration, dir string, stdout, stderr io.Writer, stdin io.Reader, fn func(context.Context, context.CancelFunc) error) error { - if timeout == -1 { - timeout = defaultCommandExecutionTimeout + return c.RunWithContext(&RunContext{ + Env: env, + Timeout: timeout, + Dir: dir, + Stdout: stdout, + Stderr: stderr, + Stdin: stdin, + PipelineFunc: fn, + }) +} + +// RunContext represents parameters to run the command +type RunContext struct { + Env []string + Timeout time.Duration + Dir string + Stdout, Stderr io.Writer + Stdin io.Reader + PipelineFunc func(context.Context, context.CancelFunc) error +} + +// RunWithContext run the command with context +func (c *Command) RunWithContext(rc *RunContext) error { + if rc.Timeout == -1 { + rc.Timeout = defaultCommandExecutionTimeout } - if len(dir) == 0 { + if len(rc.Dir) == 0 { log.Debug("%s", c) } else { - log.Debug("%s: %v", dir, c) + log.Debug("%s: %v", rc.Dir, c) } - ctx, cancel := context.WithTimeout(c.parentContext, timeout) + ctx, cancel := context.WithTimeout(c.parentContext, rc.Timeout) defer cancel() cmd := exec.CommandContext(ctx, c.name, c.args...) - if env == nil { + if rc.Env == nil { cmd.Env = os.Environ() } else { - cmd.Env = env + cmd.Env = rc.Env } cmd.Env = append( @@ -141,23 +164,23 @@ func (c *Command) RunInDirTimeoutEnvFullPipelineFunc(env []string, timeout time. if goVersionLessThan115 { cmd.Env = append(cmd.Env, "GODEBUG=asyncpreemptoff=1") } - cmd.Dir = dir - cmd.Stdout = stdout - cmd.Stderr = stderr - cmd.Stdin = stdin + cmd.Dir = rc.Dir + cmd.Stdout = rc.Stdout + cmd.Stderr = rc.Stderr + cmd.Stdin = rc.Stdin if err := cmd.Start(); err != nil { return err } desc := c.desc if desc == "" { - desc = fmt.Sprintf("%s %s %s [repo_path: %s]", GitExecutable, c.name, strings.Join(c.args, " "), dir) + desc = fmt.Sprintf("%s %s %s [repo_path: %s]", GitExecutable, c.name, strings.Join(c.args, " "), rc.Dir) } pid := process.GetManager().Add(desc, cancel) defer process.GetManager().Remove(pid) - if fn != nil { - err := fn(ctx, cancel) + if rc.PipelineFunc != nil { + err := rc.PipelineFunc(ctx, cancel) if err != nil { cancel() _ = cmd.Wait() diff --git a/modules/git/repo.go b/modules/git/repo.go index 4e6f90c3ef..f2bbbf4716 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -9,11 +9,15 @@ import ( "bytes" "context" "fmt" + "io" + "net/url" "os" "path" "strconv" "strings" "time" + + "code.gitea.io/gitea/modules/proxy" ) // GPGSettings represents the default GPG settings for this repository @@ -99,12 +103,12 @@ type CloneRepoOptions struct { } // Clone clones original repository to target path. -func Clone(from, to string, opts CloneRepoOptions) (err error) { +func Clone(from, to string, opts CloneRepoOptions) error { return CloneWithContext(DefaultContext, from, to, opts) } // CloneWithContext clones original repository to target path. -func CloneWithContext(ctx context.Context, from, to string, opts CloneRepoOptions) (err error) { +func CloneWithContext(ctx context.Context, from, to string, opts CloneRepoOptions) error { cargs := make([]string, len(GlobalCommandArgs)) copy(cargs, GlobalCommandArgs) return CloneWithArgs(ctx, from, to, cargs, opts) @@ -146,8 +150,24 @@ func CloneWithArgs(ctx context.Context, from, to string, args []string, opts Clo opts.Timeout = -1 } - _, err = cmd.RunTimeout(opts.Timeout) - return err + var envs = os.Environ() + u, err := url.Parse(from) + if err == nil && (strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https")) { + if proxy.Match(u.Host) { + envs = append(envs, fmt.Sprintf("https_proxy=%s", proxy.GetProxyURL())) + } + } + + var stderr = new(bytes.Buffer) + if err = cmd.RunWithContext(&RunContext{ + Timeout: opts.Timeout, + Env: envs, + Stdout: io.Discard, + Stderr: stderr, + }); err != nil { + return ConcatenateError(err, stderr.String()) + } + return nil } // PullRemoteOptions options when pull from remote diff --git a/modules/lfs/client.go b/modules/lfs/client.go index 0a21440f73..81b047c5bd 100644 --- a/modules/lfs/client.go +++ b/modules/lfs/client.go @@ -24,9 +24,9 @@ type Client interface { } // NewClient creates a LFS client -func NewClient(endpoint *url.URL) Client { +func NewClient(endpoint *url.URL, skipTLSVerify bool) Client { if endpoint.Scheme == "file" { return newFilesystemClient(endpoint) } - return newHTTPClient(endpoint) + return newHTTPClient(endpoint, skipTLSVerify) } diff --git a/modules/lfs/client_test.go b/modules/lfs/client_test.go index 1040b39925..ee6b7a59fc 100644 --- a/modules/lfs/client_test.go +++ b/modules/lfs/client_test.go @@ -13,10 +13,10 @@ import ( func TestNewClient(t *testing.T) { u, _ := url.Parse("file:///test") - c := NewClient(u) + c := NewClient(u, true) assert.IsType(t, &FilesystemClient{}, c) u, _ = url.Parse("https://test.com/lfs") - c = NewClient(u) + c = NewClient(u, true) assert.IsType(t, &HTTPClient{}, c) } diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go index 31c67903a8..5df5ed33a9 100644 --- a/modules/lfs/http_client.go +++ b/modules/lfs/http_client.go @@ -7,6 +7,7 @@ package lfs import ( "bytes" "context" + "crypto/tls" "errors" "fmt" "net/http" @@ -15,6 +16,7 @@ import ( "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/proxy" ) const batchSize = 20 @@ -32,8 +34,13 @@ func (c *HTTPClient) BatchSize() int { return batchSize } -func newHTTPClient(endpoint *url.URL) *HTTPClient { - hc := &http.Client{} +func newHTTPClient(endpoint *url.URL, skipTLSVerify bool) *HTTPClient { + hc := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: skipTLSVerify}, + Proxy: proxy.Proxy(), + }, + } client := &HTTPClient{ client: hc, diff --git a/modules/migrations/gitea_downloader.go b/modules/migrations/gitea_downloader.go index 2ed6c9113d..23ede93a42 100644 --- a/modules/migrations/gitea_downloader.go +++ b/modules/migrations/gitea_downloader.go @@ -6,6 +6,7 @@ package migrations import ( "context" + "crypto/tls" "errors" "fmt" "io" @@ -17,6 +18,8 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/proxy" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" gitea_sdk "code.gitea.io/sdk/gitea" @@ -87,6 +90,12 @@ func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, passwo gitea_sdk.SetToken(token), gitea_sdk.SetBasicAuth(username, password), gitea_sdk.SetContext(ctx), + gitea_sdk.SetHTTPClient(&http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}, + Proxy: proxy.Proxy(), + }, + }), ) if err != nil { log.Error(fmt.Sprintf("Failed to create NewGiteaDownloader for: %s. Error: %v", baseURL, err)) @@ -266,6 +275,13 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele Created: rel.CreatedAt, } + httpClient := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}, + Proxy: proxy.Proxy(), + }, + } + for _, asset := range rel.Attachments { size := int(asset.Size) dlCount := int(asset.DownloadCount) @@ -282,7 +298,11 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele return nil, err } // FIXME: for a private download? - resp, err := http.Get(asset.DownloadURL) + req, err := http.NewRequest("GET", asset.DownloadURL, nil) + if err != nil { + return nil, err + } + resp, err := httpClient.Do(req) if err != nil { return nil, err } diff --git a/modules/migrations/github.go b/modules/migrations/github.go index cc5279e38f..f6063b0661 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -7,6 +7,7 @@ package migrations import ( "context" + "crypto/tls" "fmt" "io" "net/http" @@ -17,6 +18,8 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/proxy" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" @@ -90,7 +93,7 @@ func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, tok Transport: &http.Transport{ Proxy: func(req *http.Request) (*url.URL, error) { req.SetBasicAuth(userName, password) - return nil, nil + return proxy.Proxy()(req) }, }, } @@ -269,6 +272,13 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) r.Published = rel.PublishedAt.Time } + httpClient := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}, + Proxy: proxy.Proxy(), + }, + } + for _, asset := range rel.Assets { var assetID = *asset.ID // Don't optimize this, for closure we need a local variable r.Assets = append(r.Assets, &base.ReleaseAsset{ @@ -295,7 +305,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) if err != nil { return nil, err } - resp, err := http.DefaultClient.Do(req) + resp, err := httpClient.Do(req) err1 := g.RefreshRate() if err1 != nil { log.Error("g.client.RateLimits: %s", err1) diff --git a/modules/migrations/gitlab.go b/modules/migrations/gitlab.go index 1050ffd0c9..28e9eac63c 100644 --- a/modules/migrations/gitlab.go +++ b/modules/migrations/gitlab.go @@ -6,6 +6,7 @@ package migrations import ( "context" + "crypto/tls" "errors" "fmt" "io" @@ -17,6 +18,8 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/proxy" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" "github.com/xanzy/go-gitlab" @@ -77,7 +80,12 @@ type GitlabDownloader struct { // Use either a username/password, personal token entered into the username field, or anonymous/public access // Note: Public access only allows very basic access func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GitlabDownloader, error) { - gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL)) + gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(&http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}, + Proxy: proxy.Proxy(), + }, + })) // Only use basic auth if token is blank and password is NOT // Basic auth will fail with empty strings, but empty token will allow anonymous public API usage if token == "" && password != "" { @@ -295,6 +303,13 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea PublisherName: rel.Author.Username, } + httpClient := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}, + Proxy: proxy.Proxy(), + }, + } + for k, asset := range rel.Assets.Links { r.Assets = append(r.Assets, &base.ReleaseAsset{ ID: int64(asset.ID), @@ -313,8 +328,7 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea return nil, err } req = req.WithContext(g.ctx) - - resp, err := http.DefaultClient.Do(req) + resp, err := httpClient.Do(req) if err != nil { return nil, err } diff --git a/modules/migrations/gogs.go b/modules/migrations/gogs.go index 388020c88a..2c7fa76146 100644 --- a/modules/migrations/gogs.go +++ b/modules/migrations/gogs.go @@ -6,6 +6,7 @@ package migrations import ( "context" + "crypto/tls" "fmt" "net/http" "net/url" @@ -14,6 +15,8 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/proxy" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" "github.com/gogs/go-gogs-client" @@ -95,9 +98,10 @@ func NewGogsDownloader(ctx context.Context, baseURL, userName, password, token, downloader.userName = token } else { downloader.transport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}, Proxy: func(req *http.Request) (*url.URL, error) { req.SetBasicAuth(userName, password) - return nil, nil + return proxy.Proxy()(req) }, } diff --git a/modules/proxy/proxy.go b/modules/proxy/proxy.go new file mode 100644 index 0000000000..0ab6ed3341 --- /dev/null +++ b/modules/proxy/proxy.go @@ -0,0 +1,83 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package proxy + +import ( + "net/http" + "net/url" + "os" + "sync" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + + "github.com/gobwas/glob" +) + +var ( + once sync.Once + hostMatchers []glob.Glob +) + +// GetProxyURL returns proxy url +func GetProxyURL() string { + if !setting.Proxy.Enabled { + return "" + } + + if setting.Proxy.ProxyURL == "" { + if os.Getenv("http_proxy") != "" { + return os.Getenv("http_proxy") + } + return os.Getenv("https_proxy") + } + return setting.Proxy.ProxyURL +} + +// Match return true if url needs to be proxied +func Match(u string) bool { + if !setting.Proxy.Enabled { + return false + } + + // enforce do once + Proxy() + + for _, v := range hostMatchers { + if v.Match(u) { + return true + } + } + return false +} + +// Proxy returns the system proxy +func Proxy() func(req *http.Request) (*url.URL, error) { + if !setting.Proxy.Enabled { + return nil + } + if setting.Proxy.ProxyURL == "" { + return http.ProxyFromEnvironment + } + + once.Do(func() { + for _, h := range setting.Proxy.ProxyHosts { + if g, err := glob.Compile(h); err == nil { + hostMatchers = append(hostMatchers, g) + } else { + log.Error("glob.Compile %s failed: %v", h, err) + } + } + }) + + return func(req *http.Request) (*url.URL, error) { + for _, v := range hostMatchers { + if v.Match(req.URL.Host) { + return http.ProxyURL(setting.Proxy.ProxyURLFixed)(req) + } + } + return http.ProxyFromEnvironment(req) + } +} diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 08531c04ed..6b87039775 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -126,7 +126,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models. if opts.LFS { ep := lfs.DetermineEndpoint(opts.CloneAddr, opts.LFSEndpoint) - if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, ep); err != nil { + if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, ep, setting.Migrations.SkipTLSVerify); err != nil { log.Error("Failed to store missing LFS objects for repository: %v", err) } } @@ -316,8 +316,8 @@ func PushUpdateAddTag(repo *models.Repository, gitRepo *git.Repository, tagName } // StoreMissingLfsObjectsInRepository downloads missing LFS objects -func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, endpoint *url.URL) error { - client := lfs.NewClient(endpoint) +func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, endpoint *url.URL, skipTLSVerify bool) error { + client := lfs.NewClient(endpoint, skipTLSVerify) contentStore := lfs.NewContentStore() pointerChan := make(chan lfs.PointerBlob) diff --git a/modules/setting/migrations.go b/modules/setting/migrations.go index 7808df5280..b663b52f89 100644 --- a/modules/setting/migrations.go +++ b/modules/setting/migrations.go @@ -16,6 +16,7 @@ var ( AllowedDomains []string BlockedDomains []string AllowLocalNetworks bool + SkipTLSVerify bool }{ MaxAttempts: 3, RetryBackoff: 3, @@ -37,4 +38,5 @@ func newMigrationsService() { } Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false) + Migrations.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool(false) } diff --git a/modules/setting/proxy.go b/modules/setting/proxy.go new file mode 100644 index 0000000000..b99237a398 --- /dev/null +++ b/modules/setting/proxy.go @@ -0,0 +1,40 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package setting + +import ( + "net/url" + + "code.gitea.io/gitea/modules/log" +) + +var ( + // Proxy settings + Proxy = struct { + Enabled bool + ProxyURL string + ProxyURLFixed *url.URL + ProxyHosts []string + }{ + Enabled: false, + ProxyURL: "", + ProxyHosts: []string{}, + } +) + +func newProxyService() { + sec := Cfg.Section("proxy") + Proxy.Enabled = sec.Key("PROXY_ENABLED").MustBool(false) + Proxy.ProxyURL = sec.Key("PROXY_URL").MustString("") + if Proxy.ProxyURL != "" { + var err error + Proxy.ProxyURLFixed, err = url.Parse(Proxy.ProxyURL) + if err != nil { + log.Error("Global PROXY_URL is not valid") + Proxy.ProxyURL = "" + } + } + Proxy.ProxyHosts = sec.Key("PROXY_HOSTS").Strings(",") +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index d584ed3d4d..441bceda20 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -1195,6 +1195,7 @@ func NewServices() { newMailService() newRegisterMailService() newNotifyMailService() + newProxyService() newWebhookService() newMigrationsService() newIndexerService() diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index 89b5df4638..5bd08fa9bf 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -196,7 +196,7 @@ func runSync(ctx context.Context, m *models.Mirror) ([]*mirrorSyncResult, bool) if m.LFS && setting.LFS.StartServer { log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo) ep := lfs.DetermineEndpoint(remoteAddr.String(), m.LFSEndpoint) - if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, ep); err != nil { + if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, ep, false); err != nil { log.Error("Failed to synchronize LFS objects for repository: %v", err) } } diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go index de81303689..c1f53196e3 100644 --- a/services/mirror/mirror_push.go +++ b/services/mirror/mirror_push.go @@ -134,7 +134,7 @@ func runPushSync(ctx context.Context, m *models.PushMirror) error { defer gitRepo.Close() ep := lfs.DetermineEndpoint(remoteAddr.String(), "") - if err := pushAllLFSObjects(ctx, gitRepo, ep); err != nil { + if err := pushAllLFSObjects(ctx, gitRepo, ep, false); err != nil { return util.NewURLSanitizedError(err, remoteAddr, true) } } @@ -176,8 +176,8 @@ func runPushSync(ctx context.Context, m *models.PushMirror) error { return nil } -func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, endpoint *url.URL) error { - client := lfs.NewClient(endpoint) +func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, endpoint *url.URL, skipTLSVerify bool) error { + client := lfs.NewClient(endpoint, skipTLSVerify) contentStore := lfs.NewContentStore() pointerChan := make(chan lfs.PointerBlob) diff --git a/services/webhook/deliver.go b/services/webhook/deliver.go index 8243fde1bb..d0e115b136 100644 --- a/services/webhook/deliver.go +++ b/services/webhook/deliver.go @@ -25,6 +25,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/proxy" "code.gitea.io/gitea/modules/setting" "github.com/gobwas/glob" ) @@ -260,7 +261,7 @@ var ( func webhookProxy() func(req *http.Request) (*url.URL, error) { if setting.Webhook.ProxyURL == "" { - return http.ProxyFromEnvironment + return proxy.Proxy() } once.Do(func() {