From fac4e27882f412faf3ddf8661bdba76732a35292 Mon Sep 17 00:00:00 2001 From: Vladimir Vissoultchev Date: Tue, 28 Jul 2015 19:48:36 +0300 Subject: [PATCH 1/5] Allow spaces in file/dir names on parsing git diff --- models/git_diff.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/git_diff.go b/models/git_diff.go index 9d34ed6271..0ff2c3a728 100644 --- a/models/git_diff.go +++ b/models/git_diff.go @@ -167,8 +167,8 @@ func ParsePatch(pid int64, maxlines int, cmd *exec.Cmd, reader io.Reader) (*Diff return diff, nil } - fs := strings.Split(line[len(DIFF_HEAD):], " ") - a := fs[0] + beg := len(DIFF_HEAD) + a := line[beg : (len(line)-beg)/2+beg] curFile = &DiffFile{ Name: a[strings.Index(a, "/")+1:], From 43bfee0d4872932c1da185e8367e21795bc4ceb5 Mon Sep 17 00:00:00 2001 From: Vladimir Vissoultchev Date: Tue, 28 Jul 2015 19:50:35 +0300 Subject: [PATCH 2/5] Raw text file view returns correct charset in content-type header if not utf-8 --- routers/repo/download.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/routers/repo/download.go b/routers/repo/download.go index b1c5fbc84d..8e9efba31b 100644 --- a/routers/repo/download.go +++ b/routers/repo/download.go @@ -26,10 +26,17 @@ func ServeBlob(ctx *middleware.Context, blob *git.Blob) error { } _, isTextFile := base.IsTextFile(buf) - _, isImageFile := base.IsImageFile(buf) - if !isTextFile && !isImageFile { - ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+path.Base(ctx.Repo.TreeName)) - ctx.Resp.Header().Set("Content-Transfer-Encoding", "binary") + if isTextFile { + charset, _ := base.DetectEncoding(buf) + if charset != "utf-8" { + ctx.Resp.Header().Set("Content-Type", "text/plain; charset="+charset) + } + } else { + _, isImageFile := base.IsImageFile(buf) + if !isImageFile { + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+path.Base(ctx.Repo.TreeName)) + ctx.Resp.Header().Set("Content-Transfer-Encoding", "binary") + } } ctx.Resp.Write(buf) _, err = io.Copy(ctx.Resp, dataRc) From 2bb1fb8f44bcd39c1f54b33377212636ae8c9558 Mon Sep 17 00:00:00 2001 From: Vladimir Vissoultchev Date: Tue, 28 Jul 2015 19:51:40 +0300 Subject: [PATCH 3/5] Use EscapePound in link hrefs in diff/file views --- templates/repo/diff.tmpl | 16 ++++++++-------- templates/repo/home.tmpl | 3 ++- templates/repo/view_file.tmpl | 6 +++--- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/templates/repo/diff.tmpl b/templates/repo/diff.tmpl index b9dc87de7c..268bbc166e 100644 --- a/templates/repo/diff.tmpl +++ b/templates/repo/diff.tmpl @@ -6,7 +6,7 @@ {{if .IsDiffCompare }}
- {{.i18n.Tr "repo.diff.browse_source"}} + {{.i18n.Tr "repo.diff.browse_source"}}

{{ShortSha .BeforeCommitId}} ... {{ShortSha .AfterCommitId}}

@@ -16,7 +16,7 @@ {{else}}
- {{.i18n.Tr "repo.diff.browse_source"}} + {{.i18n.Tr "repo.diff.browse_source"}}

{{RenderCommitMessage .Commit.Message $.RepoLink}}

@@ -37,7 +37,7 @@ {{.Commit.Author.Name}} {{end}} - {{TimeSince .Commit.Author.When $.Lang}} + {{TimeSince .Commit.Author.When $.Lang}}

@@ -90,9 +90,9 @@ {{end}}
{{if $file.IsDeleted}} - {{$.i18n.Tr "repo.diff.view_file"}} + {{$.i18n.Tr "repo.diff.view_file"}} {{else}} - {{$.i18n.Tr "repo.diff.view_file"}} + {{$.i18n.Tr "repo.diff.view_file"}} {{end}} {{$file.Name}}
@@ -100,7 +100,7 @@
{{if $isImage}}
- +
{{else}} @@ -114,7 +114,7 @@ - + @@ -126,7 +126,7 @@ {{end}} -
+
{{end}} {{end}} diff --git a/templates/repo/home.tmpl b/templates/repo/home.tmpl index 7a18ed985d..831a3962d8 100644 --- a/templates/repo/home.tmpl +++ b/templates/repo/home.tmpl @@ -50,7 +50,8 @@ {{if eq $i $l}} {{$v}} {{else}} - {{$v}} + {{ $p := index $.Paths $i}} + {{$v}} {{end}} {{end}} diff --git a/templates/repo/view_file.tmpl b/templates/repo/view_file.tmpl index 4d11170bd0..274c7df22e 100644 --- a/templates/repo/view_file.tmpl +++ b/templates/repo/view_file.tmpl @@ -13,11 +13,11 @@ {{end}} {{if not .ReadmeInList}} {{if not .IsCommit}} - + {{end}} - + @@ -47,4 +47,4 @@
{{if $line.RightIdx}}{{$line.RightIdx}}{{end}} 
{{$line.Content}}
{{end}}
- \ No newline at end of file + From 4917d29c121af221c3768871efabc5f99b88c4b8 Mon Sep 17 00:00:00 2001 From: Vladimir Vissoultchev Date: Wed, 29 Jul 2015 17:55:01 +0300 Subject: [PATCH 4/5] Partial impl of git diff encoding --- models/git_diff.go | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/models/git_diff.go b/models/git_diff.go index 0ff2c3a728..9681b3f465 100644 --- a/models/git_diff.go +++ b/models/git_diff.go @@ -87,7 +87,7 @@ func ParsePatch(pid int64, maxlines int, cmd *exec.Cmd, reader io.Reader) (*Diff leftLine, rightLine int isTooLong bool - // FIXME: use first 30 lines to detect file encoding. Should use cache in the future. + // FIXME: Should use cache in the future. buf bytes.Buffer ) @@ -106,16 +106,10 @@ func ParsePatch(pid int64, maxlines int, cmd *exec.Cmd, reader io.Reader) (*Diff i = i + 1 - // FIXME: use first 30 lines to detect file encoding. - if i <= 30 { - buf.WriteString(line) - } - // Diff data too large, we only show the first about maxlines lines if i == maxlines { isTooLong = true log.Warn("Diff data too large") - //return &Diff{}, nil } switch { @@ -127,7 +121,7 @@ func ParsePatch(pid int64, maxlines int, cmd *exec.Cmd, reader io.Reader) (*Diff continue case line[0] == '@': if isTooLong { - return diff, nil + break } curSection = &DiffSection{} @@ -137,9 +131,14 @@ func ParsePatch(pid int64, maxlines int, cmd *exec.Cmd, reader io.Reader) (*Diff curSection.Lines = append(curSection.Lines, diffLine) // Parse line number. - ranges := strings.Split(ss[len(ss)-2][1:], " ") + ranges := strings.Split(ss[1][1:], " ") leftLine, _ = com.StrTo(strings.Split(ranges[0], ",")[0][1:]).Int() - rightLine, _ = com.StrTo(strings.Split(ranges[1], ",")[0]).Int() + if len(ranges) > 1 { + rightLine, _ = com.StrTo(strings.Split(ranges[1], ",")[0]).Int() + } else { + log.Warn("Parse line number failed: %v", line) + rightLine = leftLine + } continue case line[0] == '+': curFile.Addition++ @@ -164,7 +163,7 @@ func ParsePatch(pid int64, maxlines int, cmd *exec.Cmd, reader io.Reader) (*Diff // Get new file. if strings.HasPrefix(line, DIFF_HEAD) { if isTooLong { - return diff, nil + break } beg := len(DIFF_HEAD) @@ -201,14 +200,19 @@ func ParsePatch(pid int64, maxlines int, cmd *exec.Cmd, reader io.Reader) (*Diff } } - // FIXME: use first 30 lines to detect file encoding. - charsetLabel, err := base.DetectEncoding(buf.Bytes()) - if charsetLabel != "utf8" && err == nil { - encoding, _ := charset.Lookup(charsetLabel) - - if encoding != nil { - d := encoding.NewDecoder() - for _, f := range diff.Files { + for _, f := range diff.Files { + buf.Reset() + for _, sec := range f.Sections { + for _, l := range sec.Lines { + buf.WriteString(l.Content) + buf.WriteString("\n") + } + } + charsetLabel, err := base.DetectEncoding(buf.Bytes()) + if charsetLabel != "UTF-8" && err == nil { + encoding, _ := charset.Lookup(charsetLabel) + if encoding != nil { + d := encoding.NewDecoder() for _, sec := range f.Sections { for _, l := range sec.Lines { if c, _, err := transform.String(d, l.Content); err == nil { @@ -219,7 +223,6 @@ func ParsePatch(pid int64, maxlines int, cmd *exec.Cmd, reader io.Reader) (*Diff } } } - return diff, nil } From 2cc050e21ed35d95d824cc22d129fc22ee1318a4 Mon Sep 17 00:00:00 2001 From: Vladimir Vissoultchev Date: Wed, 29 Jul 2015 17:58:03 +0300 Subject: [PATCH 5/5] Fix UTF-8 in upper-case, use ansi charset for all non UTF-8 encodings --- modules/base/template.go | 4 ++-- modules/setting/setting.go | 2 +- routers/repo/download.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/base/template.go b/modules/base/template.go index f0a2e0329c..2a81a34d26 100644 --- a/modules/base/template.go +++ b/modules/base/template.go @@ -55,7 +55,7 @@ func ShortSha(sha1 string) string { func DetectEncoding(content []byte) (string, error) { detector := chardet.NewTextDetector() result, err := detector.DetectBest(content) - if result.Charset == "ISO-8859-1" { + if result.Charset != "UTF-8" && len(setting.AnsiCharset) > 0 { return setting.AnsiCharset, err } return result.Charset, err @@ -67,7 +67,7 @@ func ToUtf8WithErr(content []byte) (error, string) { return err, "" } - if charsetLabel == "utf8" { + if charsetLabel == "UTF-8" { return nil, string(content) } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 896e60ad13..f826a3a41a 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -313,7 +313,7 @@ func NewConfigContext() { RepoRootPath = path.Clean(RepoRootPath) } ScriptType = sec.Key("SCRIPT_TYPE").MustString("bash") - AnsiCharset = sec.Key("ANSI_CHARSET").MustString("ISO-8859-1") + AnsiCharset = sec.Key("ANSI_CHARSET").MustString("") // UI settings. IssuePagingNum = Cfg.Section("ui").Key("ISSUE_PAGING_NUM").MustInt(10) diff --git a/routers/repo/download.go b/routers/repo/download.go index 8e9efba31b..c71f8d293e 100644 --- a/routers/repo/download.go +++ b/routers/repo/download.go @@ -28,7 +28,7 @@ func ServeBlob(ctx *middleware.Context, blob *git.Blob) error { _, isTextFile := base.IsTextFile(buf) if isTextFile { charset, _ := base.DetectEncoding(buf) - if charset != "utf-8" { + if charset != "UTF-8" { ctx.Resp.Header().Set("Content-Type", "text/plain; charset="+charset) } } else {