Add integrations tests from git cli (#3377)

* test: integration add git cli tests

Extracted form for easing review process and debug #3152

* test: integration add git cli big file commit

* fix:  Don't rewrite key if internal server

integrations/git_test.go

@@ -6,8 +6,9 @@ package integrations
 
 import (
 	"context"
+	"crypto/rand"
+	"fmt"
 	"io/ioutil"
-	"math/rand"
 	"net"
 	"net/http"
 	"net/url"
@@ -17,6 +18,7 @@ import (
 	"time"
 
 	"code.gitea.io/git"
+	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/setting"
 	api "code.gitea.io/sdk/gitea"
 
@@ -24,7 +26,13 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func onGiteaWebRun(t *testing.T, callback func(*testing.T, *url.URL)) {
+const (
+	littleSize = 1024              //1ko
+	bigSize    = 128 * 1024 * 1024 //128Mo
+)
+
+func onGiteaRun(t *testing.T, callback func(*testing.T, *url.URL)) {
+	prepareTestEnv(t)
 	s := http.Server{
 		Handler: mac,
 	}
@@ -35,151 +43,241 @@ func onGiteaWebRun(t *testing.T, callback func(*testing.T, *url.URL)) {
 	assert.NoError(t, err)
 
 	defer func() {
-		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
 		s.Shutdown(ctx)
 		cancel()
 	}()
 
 	go s.Serve(listener)
+	//Started by config go ssh.Listen(setting.SSH.ListenHost, setting.SSH.ListenPort, setting.SSH.ServerCiphers, setting.SSH.ServerKeyExchanges, setting.SSH.ServerMACs)
 
 	callback(t, u)
 }
 
 func TestGit(t *testing.T) {
-	prepareTestEnv(t)
+	onGiteaRun(t, func(t *testing.T, u *url.URL) {
-
-	onGiteaWebRun(t, func(t *testing.T, u *url.URL) {
-		dstPath, err := ioutil.TempDir("", "repo-tmp-17")
-		assert.NoError(t, err)
-		defer os.RemoveAll(dstPath)
 		u.Path = "user2/repo1.git"
 
-		t.Run("Standard", func(t *testing.T) {
+		t.Run("HTTP", func(t *testing.T) {
-
+			dstPath, err := ioutil.TempDir("", "repo-tmp-17")
-			t.Run("CloneNoLogin", func(t *testing.T) {
+			assert.NoError(t, err)
-				dstLocalPath, err := ioutil.TempDir("", "repo1")
+			defer os.RemoveAll(dstPath)
-				assert.NoError(t, err)
+			t.Run("Standard", func(t *testing.T) {
-				defer os.RemoveAll(dstLocalPath)
+				t.Run("CloneNoLogin", func(t *testing.T) {
-				err = git.Clone(u.String(), dstLocalPath, git.CloneRepoOptions{})
+					dstLocalPath, err := ioutil.TempDir("", "repo1")
-				assert.NoError(t, err)
+					assert.NoError(t, err)
-				assert.True(t, com.IsExist(filepath.Join(dstLocalPath, "README.md")))
+					defer os.RemoveAll(dstLocalPath)
-			})
+					err = git.Clone(u.String(), dstLocalPath, git.CloneRepoOptions{})
-
+					assert.NoError(t, err)
-			t.Run("CreateRepo", func(t *testing.T) {
+					assert.True(t, com.IsExist(filepath.Join(dstLocalPath, "README.md")))
-				session := loginUser(t, "user2")
-				req := NewRequestWithJSON(t, "POST", "/api/v1/user/repos", &api.CreateRepoOption{
-					AutoInit:    true,
-					Description: "Temporary repo",
-					Name:        "repo-tmp-17",
-					Private:     false,
-					Gitignores:  "",
-					License:     "WTFPL",
-					Readme:      "Default",
 				})
-				session.MakeRequest(t, req, http.StatusCreated)
-			})
 
-			u.Path = "user2/repo-tmp-17.git"
+				t.Run("CreateRepo", func(t *testing.T) {
-			u.User = url.UserPassword("user2", userPassword)
+					session := loginUser(t, "user2")
-			t.Run("Clone", func(t *testing.T) {
+					req := NewRequestWithJSON(t, "POST", "/api/v1/user/repos", &api.CreateRepoOption{
-				err = git.Clone(u.String(), dstPath, git.CloneRepoOptions{})
+						AutoInit:    true,
-				assert.NoError(t, err)
+						Description: "Temporary repo",
-				assert.True(t, com.IsExist(filepath.Join(dstPath, "README.md")))
+						Name:        "repo-tmp-17",
-			})
+						Private:     false,
-
+						Gitignores:  "",
-			t.Run("PushCommit", func(t *testing.T) {
+						License:     "WTFPL",
-				data := make([]byte, 1024)
+						Readme:      "Default",
-				_, err := rand.Read(data)
+					})
-				assert.NoError(t, err)
+					session.MakeRequest(t, req, http.StatusCreated)
-				tmpFile, err := ioutil.TempFile(dstPath, "data-file-")
-				defer tmpFile.Close()
-				_, err = tmpFile.Write(data)
-				assert.NoError(t, err)
-
-				//Commit
-				err = git.AddChanges(dstPath, false, filepath.Base(tmpFile.Name()))
-				assert.NoError(t, err)
-				err = git.CommitChanges(dstPath, git.CommitChangesOptions{
-					Committer: &git.Signature{
-						Email: "user2@example.com",
-						Name:  "User Two",
-						When:  time.Now(),
-					},
-					Author: &git.Signature{
-						Email: "user2@example.com",
-						Name:  "User Two",
-						When:  time.Now(),
-					},
-					Message: "Testing commit",
 				})
-				assert.NoError(t, err)
 
-				//Push
+				u.Path = "user2/repo-tmp-17.git"
-				err = git.Push(dstPath, git.PushOptions{
+				u.User = url.UserPassword("user2", userPassword)
-					Branch: "master",
+				t.Run("Clone", func(t *testing.T) {
-					Remote: u.String(),
+					err = git.Clone(u.String(), dstPath, git.CloneRepoOptions{})
-					Force:  false,
+					assert.NoError(t, err)
+					assert.True(t, com.IsExist(filepath.Join(dstPath, "README.md")))
+				})
+
+				t.Run("PushCommit", func(t *testing.T) {
+					t.Run("Little", func(t *testing.T) {
+						commitAndPush(t, littleSize, dstPath)
+					})
+					t.Run("Big", func(t *testing.T) {
+						commitAndPush(t, bigSize, dstPath)
+					})
+				})
+			})
+			t.Run("LFS", func(t *testing.T) {
+				t.Run("PushCommit", func(t *testing.T) {
+					//Setup git LFS
+					_, err = git.NewCommand("lfs").AddArguments("install").RunInDir(dstPath)
+					assert.NoError(t, err)
+					_, err = git.NewCommand("lfs").AddArguments("track", "data-file-*").RunInDir(dstPath)
+					assert.NoError(t, err)
+					err = git.AddChanges(dstPath, false, ".gitattributes")
+					assert.NoError(t, err)
+
+					t.Run("Little", func(t *testing.T) {
+						commitAndPush(t, littleSize, dstPath)
+					})
+					t.Run("Big", func(t *testing.T) {
+						commitAndPush(t, bigSize, dstPath)
+					})
+				})
+				t.Run("Locks", func(t *testing.T) {
+					lockTest(t, u.String(), dstPath)
 				})
-				assert.NoError(t, err)
 			})
 		})
-		t.Run("LFS", func(t *testing.T) {
+		t.Run("SSH", func(t *testing.T) {
-			t.Run("PushCommit", func(t *testing.T) {
+			//Setup remote link
-				/* Generate random file */
+			u.Scheme = "ssh"
-				data := make([]byte, 1024)
+			u.User = url.User("git")
-				_, err := rand.Read(data)
+			u.Host = fmt.Sprintf("%s:%d", setting.SSH.ListenHost, setting.SSH.ListenPort)
-				assert.NoError(t, err)
+			u.Path = "user2/repo-tmp-18.git"
-				tmpFile, err := ioutil.TempFile(dstPath, "data-file-")
-				defer tmpFile.Close()
-				_, err = tmpFile.Write(data)
-				assert.NoError(t, err)
 
-				//Setup git LFS
+			//Setup key
-				_, err = git.NewCommand("lfs").AddArguments("install").RunInDir(dstPath)
+			keyFile := filepath.Join(setting.AppDataPath, "my-testing-key")
-				assert.NoError(t, err)
+			_, _, err := com.ExecCmd("ssh-keygen", "-f", keyFile, "-t", "rsa", "-N", "")
-				_, err = git.NewCommand("lfs").AddArguments("track", "data-file-*").RunInDir(dstPath)
+			assert.NoError(t, err)
-				assert.NoError(t, err)
+			defer os.RemoveAll(keyFile)
+			defer os.RemoveAll(keyFile + ".pub")
 
-				//Commit
+			session := loginUser(t, "user1")
-				err = git.AddChanges(dstPath, false, ".gitattributes", filepath.Base(tmpFile.Name()))
+			keyOwner := models.AssertExistsAndLoadBean(t, &models.User{Name: "user2"}).(*models.User)
-				assert.NoError(t, err)
+			urlStr := fmt.Sprintf("/api/v1/admin/users/%s/keys", keyOwner.Name)
-				err = git.CommitChanges(dstPath, git.CommitChangesOptions{
-					Committer: &git.Signature{
-						Email: "user2@example.com",
-						Name:  "User Two",
-						When:  time.Now(),
-					},
-					Author: &git.Signature{
-						Email: "user2@example.com",
-						Name:  "User Two",
-						When:  time.Now(),
-					},
-					Message: "Testing LFS ",
-				})
-				assert.NoError(t, err)
 
-				//Push
+			dataPubKey, err := ioutil.ReadFile(keyFile + ".pub")
-				u.User = url.UserPassword("user2", userPassword)
+			assert.NoError(t, err)
-				err = git.Push(dstPath, git.PushOptions{
+			req := NewRequestWithValues(t, "POST", urlStr, map[string]string{
-					Branch: "master",
+				"key":   string(dataPubKey),
-					Remote: u.String(),
+				"title": "test-key",
-					Force:  false,
-				})
-				assert.NoError(t, err)
-			})
-			t.Run("Locks", func(t *testing.T) {
-				_, err = git.NewCommand("remote").AddArguments("set-url", "origin", u.String()).RunInDir(dstPath) //TODO add test ssh git-lfs-creds
-				assert.NoError(t, err)
-				_, err = git.NewCommand("lfs").AddArguments("locks").RunInDir(dstPath)
-				assert.NoError(t, err)
-				_, err = git.NewCommand("lfs").AddArguments("lock", "README.md").RunInDir(dstPath)
-				assert.NoError(t, err)
-				_, err = git.NewCommand("lfs").AddArguments("locks").RunInDir(dstPath)
-				assert.NoError(t, err)
-				_, err = git.NewCommand("lfs").AddArguments("unlock", "README.md").RunInDir(dstPath)
-				assert.NoError(t, err)
 			})
+			session.MakeRequest(t, req, http.StatusCreated)
 
+			//Setup ssh wrapper
+			sshWrapper, err := ioutil.TempFile(setting.AppDataPath, "tmp-ssh-wrapper")
+			sshWrapper.WriteString("#!/bin/sh\n\n")
+			sshWrapper.WriteString("ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i \"" + filepath.Join(setting.AppWorkPath, keyFile) + "\" $* \n\n")
+			err = sshWrapper.Chmod(os.ModePerm)
+			assert.NoError(t, err)
+			sshWrapper.Close()
+			defer os.RemoveAll(sshWrapper.Name())
+
+			//Setup clone folder
+			dstPath, err := ioutil.TempDir("", "repo-tmp-18")
+			assert.NoError(t, err)
+			defer os.RemoveAll(dstPath)
+
+			t.Run("Standard", func(t *testing.T) {
+				t.Run("CreateRepo", func(t *testing.T) {
+					session := loginUser(t, "user2")
+					req := NewRequestWithJSON(t, "POST", "/api/v1/user/repos", &api.CreateRepoOption{
+						AutoInit:    true,
+						Description: "Temporary repo",
+						Name:        "repo-tmp-18",
+						Private:     false,
+						Gitignores:  "",
+						License:     "WTFPL",
+						Readme:      "Default",
+					})
+					session.MakeRequest(t, req, http.StatusCreated)
+				})
+				//TODO get url from api
+				t.Run("Clone", func(t *testing.T) {
+					_, err = git.NewCommand("clone").AddArguments("--config", "core.sshCommand="+filepath.Join(setting.AppWorkPath, sshWrapper.Name()), u.String(), dstPath).Run()
+					assert.NoError(t, err)
+					assert.True(t, com.IsExist(filepath.Join(dstPath, "README.md")))
+				})
+				//time.Sleep(5 * time.Minute)
+				t.Run("PushCommit", func(t *testing.T) {
+					t.Run("Little", func(t *testing.T) {
+						commitAndPush(t, littleSize, dstPath)
+					})
+					t.Run("Big", func(t *testing.T) {
+						commitAndPush(t, bigSize, dstPath)
+					})
+				})
+			})
+			t.Run("LFS", func(t *testing.T) {
+				os.Setenv("GIT_SSH_COMMAND", filepath.Join(setting.AppWorkPath, sshWrapper.Name())) //TODO remove when fixed https://github.com/git-lfs/git-lfs/issues/2215
+				defer os.Unsetenv("GIT_SSH_COMMAND")
+				t.Run("PushCommit", func(t *testing.T) {
+					//Setup git LFS
+					_, err = git.NewCommand("lfs").AddArguments("install").RunInDir(dstPath)
+					assert.NoError(t, err)
+					_, err = git.NewCommand("lfs").AddArguments("track", "data-file-*").RunInDir(dstPath)
+					assert.NoError(t, err)
+					err = git.AddChanges(dstPath, false, ".gitattributes")
+					assert.NoError(t, err)
+
+					t.Run("Little", func(t *testing.T) {
+						commitAndPush(t, littleSize, dstPath)
+					})
+					t.Run("Big", func(t *testing.T) {
+						commitAndPush(t, bigSize, dstPath)
+					})
+				})
+				/* Failed without #3152. TODO activate with fix.
+				t.Run("Locks", func(t *testing.T) {
+				  lockTest(t, u.String(), dstPath)
+				})
+				*/
+			})
 		})
 	})
 }
+
+func lockTest(t *testing.T, remote, repoPath string) {
+	_, err := git.NewCommand("remote").AddArguments("set-url", "origin", remote).RunInDir(repoPath) //TODO add test ssh git-lfs-creds
+	assert.NoError(t, err)
+	_, err = git.NewCommand("lfs").AddArguments("locks").RunInDir(repoPath)
+	assert.NoError(t, err)
+	_, err = git.NewCommand("lfs").AddArguments("lock", "README.md").RunInDir(repoPath)
+	assert.NoError(t, err)
+	_, err = git.NewCommand("lfs").AddArguments("locks").RunInDir(repoPath)
+	assert.NoError(t, err)
+	_, err = git.NewCommand("lfs").AddArguments("unlock", "README.md").RunInDir(repoPath)
+	assert.NoError(t, err)
+}
+
+func commitAndPush(t *testing.T, size int, repoPath string) {
+	err := generateCommitWithNewData(size, repoPath, "user2@example.com", "User Two")
+	assert.NoError(t, err)
+	_, err = git.NewCommand("push").RunInDir(repoPath) //Push
+	assert.NoError(t, err)
+}
+
+func generateCommitWithNewData(size int, repoPath, email, fullName string) error {
+	//Generate random file
+	data := make([]byte, size)
+	_, err := rand.Read(data)
+	if err != nil {
+		return err
+	}
+	tmpFile, err := ioutil.TempFile(repoPath, "data-file-")
+	if err != nil {
+		return err
+	}
+	defer tmpFile.Close()
+	_, err = tmpFile.Write(data)
+	if err != nil {
+		return err
+	}
+
+	//Commit
+	err = git.AddChanges(repoPath, false, filepath.Base(tmpFile.Name()))
+	if err != nil {
+		return err
+	}
+	err = git.CommitChanges(repoPath, git.CommitChangesOptions{
+		Committer: &git.Signature{
+			Email: email,
+			Name:  fullName,
+			When:  time.Now(),
+		},
+		Author: &git.Signature{
+			Email: email,
+			Name:  fullName,
+			When:  time.Now(),
+		},
+		Message: fmt.Sprintf("Testing commit @ %v", time.Now()),
+	})
+	return err
+}

integrations/mysql.ini.tmpl

@@ -8,7 +8,6 @@ NAME     = {{TEST_MYSQL_DBNAME}}
 USER     = {{TEST_MYSQL_USERNAME}}
 PASSWD   = {{TEST_MYSQL_PASSWORD}}
 SSL_MODE = disable
-PATH     = data/gitea.db
 
 [indexer]
 ISSUE_INDEXER_PATH = integrations/indexers-mysql/issues.bleve
@@ -27,10 +26,14 @@ SSH_DOMAIN       = localhost
 HTTP_PORT        = 3001
 ROOT_URL         = http://localhost:3001/
 DISABLE_SSH      = false
-SSH_PORT         = 22
+SSH_LISTEN_HOST  = localhost
+SSH_PORT         = 2201
+START_SSH_SERVER = true
 LFS_START_SERVER = true
 LFS_CONTENT_PATH = data/lfs-mysql
 OFFLINE_MODE     = false
+LFS_JWT_SECRET   = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w
+APP_DATA_PATH    = integrations/gitea-integration-mysql/data
 
 [mailer]
 ENABLED = false

integrations/pgsql.ini.tmpl

@@ -8,7 +8,6 @@ NAME     = {{TEST_PGSQL_DBNAME}}
 USER     = {{TEST_PGSQL_USERNAME}}
 PASSWD   = {{TEST_PGSQL_PASSWORD}}
 SSL_MODE = disable
-PATH     = data/gitea.db
 
 [indexer]
 ISSUE_INDEXER_PATH = integrations/indexers-pgsql/issues.bleve
@@ -27,23 +26,27 @@ SSH_DOMAIN       = localhost
 HTTP_PORT        = 3002
 ROOT_URL         = http://localhost:3002/
 DISABLE_SSH      = false
-SSH_PORT         = 22
+SSH_LISTEN_HOST  = localhost
+SSH_PORT         = 2202
+START_SSH_SERVER = true
 LFS_START_SERVER = true
 LFS_CONTENT_PATH = data/lfs-pgsql
 OFFLINE_MODE     = false
+LFS_JWT_SECRET   = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w
+APP_DATA_PATH    = integrations/gitea-integration-pgsql/data
 
 [mailer]
 ENABLED = false
 
 [service]
-REGISTER_EMAIL_CONFIRM     = false
+REGISTER_EMAIL_CONFIRM            = false
-ENABLE_NOTIFY_MAIL         = false
+ENABLE_NOTIFY_MAIL                = false
-DISABLE_REGISTRATION       = false
+DISABLE_REGISTRATION              = false
-ENABLE_CAPTCHA             = false
+ENABLE_CAPTCHA                    = false
-REQUIRE_SIGNIN_VIEW        = false
+REQUIRE_SIGNIN_VIEW               = false
-DEFAULT_KEEP_EMAIL_PRIVATE = false
+DEFAULT_KEEP_EMAIL_PRIVATE        = false
 DEFAULT_ALLOW_CREATE_ORGANIZATION = true
-NO_REPLY_ADDRESS           = noreply.example.org
+NO_REPLY_ADDRESS                  = noreply.example.org
 
 [picture]
 DISABLE_GRAVATAR        = false
@@ -54,7 +57,7 @@ PROVIDER = file
 PROVIDER_CONFIG = data/sessions-pgsql
 
 [log]
-MODE = console,file
+MODE      = console,file
 ROOT_PATH = pgsql-log
 
 [log.console]
@@ -64,6 +67,6 @@ LEVEL = Warn
 LEVEL = Debug
 
 [security]
-INSTALL_LOCK = true
+INSTALL_LOCK   = true
-SECRET_KEY   = 9pCviYTWSb
+SECRET_KEY     = 9pCviYTWSb
-INTERNAL_TOKEN = test
+INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.hhSVGOANkaKk3vfCd2jDOIww4pUk0xtg9JRde5UogyQ

integrations/repo_test.go

@@ -71,6 +71,6 @@ func TestViewRepo1CloneLinkAuthorized(t *testing.T) {
 	assert.Equal(t, setting.AppURL+"user2/repo1.git", link)
 	link, exists = htmlDoc.doc.Find("#repo-clone-ssh").Attr("data-link")
 	assert.True(t, exists, "The template has changed")
-	sshURL := fmt.Sprintf("%s@%s:user2/repo1.git", setting.RunUser, setting.SSH.Domain)
+	sshURL := fmt.Sprintf("ssh://%s@%s:%d/user2/repo1.git", setting.RunUser, setting.SSH.Domain, setting.SSH.Port)
 	assert.Equal(t, sshURL, link)
 }

integrations/sqlite.ini

@@ -3,7 +3,7 @@ RUN_MODE = prod
 
 [database]
 DB_TYPE = sqlite3
-PATH    = :memory:
+PATH    = integrations/gitea-integration-sqlite/gitea.db
 
 [indexer]
 ISSUE_INDEXER_PATH   = integrations/indexers-sqlite/issues.bleve
@@ -22,11 +22,14 @@ SSH_DOMAIN       = localhost
 HTTP_PORT        = 3003
 ROOT_URL         = http://localhost:3003/
 DISABLE_SSH      = false
-SSH_PORT         = 22
+SSH_LISTEN_HOST  = localhost
+SSH_PORT         = 2203
+START_SSH_SERVER = true
 LFS_START_SERVER = true
 LFS_CONTENT_PATH = data/lfs-sqlite
 OFFLINE_MODE     = false
 LFS_JWT_SECRET   = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w
+APP_DATA_PATH    = integrations/gitea-integration-sqlite/data
 
 [mailer]
 ENABLED = false
@@ -62,4 +65,3 @@ LEVEL = Debug
 INSTALL_LOCK   = true
 SECRET_KEY     = 9pCviYTWSb
 INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTI3OTU5ODN9.OQkH5UmzID2XBdwQ9TAI6Jj2t1X-wElVTjbE7aoN4I8
-

models/ssh_key.go

@@ -304,6 +304,11 @@ func CheckPublicKeyString(content string) (_ string, err error) {
 
 // appendAuthorizedKeysToFile appends new SSH keys' content to authorized_keys file.
 func appendAuthorizedKeysToFile(keys ...*PublicKey) error {
+	// Don't need to rewrite this file if builtin SSH server is enabled.
+	if setting.SSH.StartBuiltinServer {
+		return nil
+	}
+
 	sshOpLocker.Lock()
 	defer sshOpLocker.Unlock()
 
@@ -532,6 +537,11 @@ func DeletePublicKey(doer *User, id int64) (err error) {
 // Note: x.Iterate does not get latest data after insert/delete, so we have to call this function
 // outside any session scope independently.
 func RewriteAllPublicKeys() error {
+	//Don't rewrite key if internal server
+	if setting.SSH.StartBuiltinServer {
+		return nil
+	}
+
 	sshOpLocker.Lock()
 	defer sshOpLocker.Unlock()