Add warning for BIDI characters in page renders and in diffs (#17562)

Fix #17514

Given the comments I've adjusted this somewhat. The numbers of characters detected are increased and include things like the use of U+300 to make à instead of à and non-breaking spaces.

There is a button which can be used to escape the content to show it.

Signed-off-by: Andrew Thornton <art27@cantab.net>
Co-authored-by: Gwyneth Morgan <gwymor@tilde.club>
Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

modules/charset/escape.go

@@ -0,0 +1,230 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package charset
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+
+	"golang.org/x/text/unicode/bidi"
+)
+
+// EscapeStatus represents the findings of the unicode escaper
+type EscapeStatus struct {
+	Escaped      bool
+	HasError     bool
+	HasBadRunes  bool
+	HasControls  bool
+	HasSpaces    bool
+	HasMarks     bool
+	HasBIDI      bool
+	BadBIDI      bool
+	HasRTLScript bool
+	HasLTRScript bool
+}
+
+// Or combines two EscapeStatus structs into one representing the conjunction of the two
+func (status EscapeStatus) Or(other EscapeStatus) EscapeStatus {
+	st := status
+	st.Escaped = st.Escaped || other.Escaped
+	st.HasError = st.HasError || other.HasError
+	st.HasBadRunes = st.HasBadRunes || other.HasBadRunes
+	st.HasControls = st.HasControls || other.HasControls
+	st.HasSpaces = st.HasSpaces || other.HasSpaces
+	st.HasMarks = st.HasMarks || other.HasMarks
+	st.HasBIDI = st.HasBIDI || other.HasBIDI
+	st.BadBIDI = st.BadBIDI || other.BadBIDI
+	st.HasRTLScript = st.HasRTLScript || other.HasRTLScript
+	st.HasLTRScript = st.HasLTRScript || other.HasLTRScript
+	return st
+}
+
+// EscapeControlString escapes the unicode control sequences in a provided string and returns the findings as an EscapeStatus and the escaped string
+func EscapeControlString(text string) (EscapeStatus, string) {
+	sb := &strings.Builder{}
+	escaped, _ := EscapeControlReader(strings.NewReader(text), sb)
+	return escaped, sb.String()
+}
+
+// EscapeControlBytes escapes the unicode control sequences  a provided []byte and returns the findings as an EscapeStatus and the escaped []byte
+func EscapeControlBytes(text []byte) (EscapeStatus, []byte) {
+	buf := &bytes.Buffer{}
+	escaped, _ := EscapeControlReader(bytes.NewReader(text), buf)
+	return escaped, buf.Bytes()
+}
+
+// EscapeControlReader escapes the unicode control sequences  a provided Reader writing the escaped output to the output and returns the findings as an EscapeStatus and an error
+func EscapeControlReader(text io.Reader, output io.Writer) (escaped EscapeStatus, err error) {
+	buf := make([]byte, 4096)
+	readStart := 0
+	var n int
+	var writePos int
+
+	lineHasBIDI := false
+	lineHasRTLScript := false
+	lineHasLTRScript := false
+
+readingloop:
+	for err == nil {
+		n, err = text.Read(buf[readStart:])
+		bs := buf[:n+readStart]
+		i := 0
+
+		for i < len(bs) {
+			r, size := utf8.DecodeRune(bs[i:])
+			// Now handle the codepoints
+			switch {
+			case r == utf8.RuneError:
+				if writePos < i {
+					if _, err = output.Write(bs[writePos:i]); err != nil {
+						escaped.HasError = true
+						return
+					}
+					writePos = i
+				}
+				// runes can be at most 4 bytes - so...
+				if len(bs)-i <= 3 {
+					// if not request more data
+					copy(buf, bs[i:])
+					readStart = n - i
+					writePos = 0
+					continue readingloop
+				}
+				// this is a real broken rune
+				escaped.HasBadRunes = true
+				escaped.Escaped = true
+				if err = writeBroken(output, bs[i:i+size]); err != nil {
+					escaped.HasError = true
+					return
+				}
+				writePos += size
+			case r == '\n':
+				if lineHasBIDI && !lineHasRTLScript && lineHasLTRScript {
+					escaped.BadBIDI = true
+				}
+				lineHasBIDI = false
+				lineHasRTLScript = false
+				lineHasLTRScript = false
+
+			case r == '\r' || r == '\t' || r == ' ':
+				// These are acceptable control characters and space characters
+			case unicode.IsSpace(r):
+				escaped.HasSpaces = true
+				escaped.Escaped = true
+				if writePos < i {
+					if _, err = output.Write(bs[writePos:i]); err != nil {
+						escaped.HasError = true
+						return
+					}
+				}
+				if err = writeEscaped(output, r); err != nil {
+					escaped.HasError = true
+					return
+				}
+				writePos = i + size
+			case unicode.Is(unicode.Bidi_Control, r):
+				escaped.Escaped = true
+				escaped.HasBIDI = true
+				if writePos < i {
+					if _, err = output.Write(bs[writePos:i]); err != nil {
+						escaped.HasError = true
+						return
+					}
+				}
+				lineHasBIDI = true
+				if err = writeEscaped(output, r); err != nil {
+					escaped.HasError = true
+					return
+				}
+				writePos = i + size
+			case unicode.Is(unicode.C, r):
+				escaped.Escaped = true
+				escaped.HasControls = true
+				if writePos < i {
+					if _, err = output.Write(bs[writePos:i]); err != nil {
+						escaped.HasError = true
+						return
+					}
+				}
+				if err = writeEscaped(output, r); err != nil {
+					escaped.HasError = true
+					return
+				}
+				writePos = i + size
+			case unicode.Is(unicode.M, r):
+				escaped.Escaped = true
+				escaped.HasMarks = true
+				if writePos < i {
+					if _, err = output.Write(bs[writePos:i]); err != nil {
+						escaped.HasError = true
+						return
+					}
+				}
+				if err = writeEscaped(output, r); err != nil {
+					escaped.HasError = true
+					return
+				}
+				writePos = i + size
+			default:
+				p, _ := bidi.Lookup(bs[i : i+size])
+				c := p.Class()
+				if c == bidi.R || c == bidi.AL {
+					lineHasRTLScript = true
+					escaped.HasRTLScript = true
+				} else if c == bidi.L {
+					lineHasLTRScript = true
+					escaped.HasLTRScript = true
+				}
+			}
+			i += size
+		}
+		if n > 0 {
+			// we read something...
+			// write everything unwritten
+			if writePos < i {
+				if _, err = output.Write(bs[writePos:i]); err != nil {
+					escaped.HasError = true
+					return
+				}
+			}
+
+			// reset the starting positions for the next read
+			readStart = 0
+			writePos = 0
+		}
+	}
+	if readStart > 0 {
+		// this means that there is an incomplete or broken rune at 0-readStart and we read nothing on the last go round
+		escaped.Escaped = true
+		escaped.HasBadRunes = true
+		if err = writeBroken(output, buf[:readStart]); err != nil {
+			escaped.HasError = true
+			return
+		}
+	}
+	if err == io.EOF {
+		if lineHasBIDI && !lineHasRTLScript && lineHasLTRScript {
+			escaped.BadBIDI = true
+		}
+		err = nil
+		return
+	}
+	escaped.HasError = true
+	return
+}
+
+func writeBroken(output io.Writer, bs []byte) (err error) {
+	_, err = fmt.Fprintf(output, `<span class="broken-code-point">&lt;%X&gt;</span>`, bs)
+	return
+}
+
+func writeEscaped(output io.Writer, r rune) (err error) {
+	_, err = fmt.Fprintf(output, `<span class="escaped-code-point" data-escaped="[U+%04X]"><span class="char">%c</span></span>`, r, r)
+	return
+}

modules/charset/escape_test.go

@@ -0,0 +1,202 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package charset
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+)
+
+type escapeControlTest struct {
+	name   string
+	text   string
+	status EscapeStatus
+	result string
+}
+
+var escapeControlTests = []escapeControlTest{
+	{
+		name: "<empty>",
+	},
+	{
+		name:   "single line western",
+		text:   "single line western",
+		result: "single line western",
+		status: EscapeStatus{HasLTRScript: true},
+	},
+	{
+		name:   "multi line western",
+		text:   "single line western\nmulti line western\n",
+		result: "single line western\nmulti line western\n",
+		status: EscapeStatus{HasLTRScript: true},
+	},
+	{
+		name:   "multi line western non-breaking space",
+		text:   "single line western\nmulti line western\n",
+		result: `single line<span class="escaped-code-point" data-escaped="[U+00A0]"><span class="char"> </span></span>western` + "\n" + `multi line<span class="escaped-code-point" data-escaped="[U+00A0]"><span class="char"> </span></span>western` + "\n",
+		status: EscapeStatus{Escaped: true, HasLTRScript: true, HasSpaces: true},
+	},
+	{
+		name:   "mixed scripts: western + japanese",
+		text:   "日属秘ぞしちゅ。Then some western.",
+		result: "日属秘ぞしちゅ。Then some western.",
+		status: EscapeStatus{HasLTRScript: true},
+	},
+	{
+		name:   "japanese",
+		text:   "日属秘ぞしちゅ。",
+		result: "日属秘ぞしちゅ。",
+		status: EscapeStatus{HasLTRScript: true},
+	},
+	{
+		name:   "hebrew",
+		text:   "עד תקופת יוון העתיקה היה העיסוק במתמטיקה תכליתי בלבד: היא שימשה כאוסף של נוסחאות לחישוב קרקע, אוכלוסין וכו'. פריצת הדרך של היוונים, פרט לתרומותיהם הגדולות לידע המתמטי, הייתה בלימוד המתמטיקה כשלעצמה, מתוקף ערכה הרוחני. יחסם של חלק מהיוונים הקדמונים למתמטיקה היה דתי - למשל, הכת שאסף סביבו פיתגורס האמינה כי המתמטיקה היא הבסיס לכל הדברים. היוונים נחשבים ליוצרי מושג ההוכחה המתמטית, וכן לראשונים שעסקו במתמטיקה לשם עצמה, כלומר כתחום מחקרי עיוני ומופשט ולא רק כעזר שימושי. עם זאת, לצדה",
+		result: "עד תקופת יוון העתיקה היה העיסוק במתמטיקה תכליתי בלבד: היא שימשה כאוסף של נוסחאות לחישוב קרקע, אוכלוסין וכו'. פריצת הדרך של היוונים, פרט לתרומותיהם הגדולות לידע המתמטי, הייתה בלימוד המתמטיקה כשלעצמה, מתוקף ערכה הרוחני. יחסם של חלק מהיוונים הקדמונים למתמטיקה היה דתי - למשל, הכת שאסף סביבו פיתגורס האמינה כי המתמטיקה היא הבסיס לכל הדברים. היוונים נחשבים ליוצרי מושג ההוכחה המתמטית, וכן לראשונים שעסקו במתמטיקה לשם עצמה, כלומר כתחום מחקרי עיוני ומופשט ולא רק כעזר שימושי. עם זאת, לצדה",
+		status: EscapeStatus{HasRTLScript: true},
+	},
+	{
+		name: "more hebrew",
+		text: `בתקופה מאוחרת יותר, השתמשו היוונים בשיטת סימון מתקדמת יותר, שבה הוצגו המספרים לפי 22 אותיות האלפבית היווני. לסימון המספרים בין 1 ל-9 נקבעו תשע האותיות הראשונות, בתוספת גרש ( ' ) בצד ימין של האות, למעלה; תשע האותיות הבאות ייצגו את העשרות מ-10 עד 90, והבאות את המאות. לסימון הספרות בין 1000 ל-900,000, השתמשו היוונים באותן אותיות, אך הוסיפו לאותיות את הגרש דווקא מצד שמאל של האותיות, למטה. ממיליון ומעלה, כנראה השתמשו היוונים בשני תגים במקום אחד.
+
+			המתמטיקאי הבולט הראשון ביוון העתיקה, ויש האומרים בתולדות האנושות, הוא תאלס (624 לפנה"ס - 546 לפנה"ס בקירוב).[1] לא יהיה זה משולל יסוד להניח שהוא האדם הראשון שהוכיח משפט מתמטי, ולא רק גילה אותו. תאלס הוכיח שישרים מקבילים חותכים מצד אחד של שוקי זווית קטעים בעלי יחסים שווים (משפט תאלס הראשון), שהזווית המונחת על קוטר במעגל היא זווית ישרה (משפט תאלס השני), שהקוטר מחלק את המעגל לשני חלקים שווים, ושזוויות הבסיס במשולש שווה-שוקיים שוות זו לזו. מיוחסות לו גם שיטות למדידת גובהן של הפירמידות בעזרת מדידת צילן ולקביעת מיקומה של ספינה הנראית מן החוף.
+
+			בשנים 582 לפנה"ס עד 496 לפנה"ס, בקירוב, חי מתמטיקאי חשוב במיוחד - פיתגורס. המקורות הראשוניים עליו מועטים, וההיסטוריונים מתקשים להפריד את העובדות משכבת המסתורין והאגדות שנקשרו בו. ידוע שסביבו התקבצה האסכולה הפיתגוראית מעין כת פסבדו-מתמטית שהאמינה ש"הכל מספר", או ליתר דיוק הכל ניתן לכימות, וייחסה למספרים משמעויות מיסטיות. ככל הנראה הפיתגוראים ידעו לבנות את הגופים האפלטוניים, הכירו את הממוצע האריתמטי, הממוצע הגאומטרי והממוצע ההרמוני והגיעו להישגים חשובים נוספים. ניתן לומר שהפיתגוראים גילו את היותו של השורש הריבועי של 2, שהוא גם האלכסון בריבוע שאורך צלעותיו 1, אי רציונלי, אך תגליתם הייתה למעשה רק שהקטעים "חסרי מידה משותפת", ומושג המספר האי רציונלי מאוחר יותר.[2] אזכור ראשון לקיומם של קטעים חסרי מידה משותפת מופיע בדיאלוג "תאיטיטוס" של אפלטון, אך רעיון זה היה מוכר עוד קודם לכן, במאה החמישית לפנה"ס להיפאסוס, בן האסכולה הפיתגוראית, ואולי לפיתגורס עצמו.[3]`,
+		result: `בתקופה מאוחרת יותר, השתמשו היוונים בשיטת סימון מתקדמת יותר, שבה הוצגו המספרים לפי 22 אותיות האלפבית היווני. לסימון המספרים בין 1 ל-9 נקבעו תשע האותיות הראשונות, בתוספת גרש ( ' ) בצד ימין של האות, למעלה; תשע האותיות הבאות ייצגו את העשרות מ-10 עד 90, והבאות את המאות. לסימון הספרות בין 1000 ל-900,000, השתמשו היוונים באותן אותיות, אך הוסיפו לאותיות את הגרש דווקא מצד שמאל של האותיות, למטה. ממיליון ומעלה, כנראה השתמשו היוונים בשני תגים במקום אחד.
+
+			המתמטיקאי הבולט הראשון ביוון העתיקה, ויש האומרים בתולדות האנושות, הוא תאלס (624 לפנה"ס - 546 לפנה"ס בקירוב).[1] לא יהיה זה משולל יסוד להניח שהוא האדם הראשון שהוכיח משפט מתמטי, ולא רק גילה אותו. תאלס הוכיח שישרים מקבילים חותכים מצד אחד של שוקי זווית קטעים בעלי יחסים שווים (משפט תאלס הראשון), שהזווית המונחת על קוטר במעגל היא זווית ישרה (משפט תאלס השני), שהקוטר מחלק את המעגל לשני חלקים שווים, ושזוויות הבסיס במשולש שווה-שוקיים שוות זו לזו. מיוחסות לו גם שיטות למדידת גובהן של הפירמידות בעזרת מדידת צילן ולקביעת מיקומה של ספינה הנראית מן החוף.
+
+			בשנים 582 לפנה"ס עד 496 לפנה"ס, בקירוב, חי מתמטיקאי חשוב במיוחד - פיתגורס. המקורות הראשוניים עליו מועטים, וההיסטוריונים מתקשים להפריד את העובדות משכבת המסתורין והאגדות שנקשרו בו. ידוע שסביבו התקבצה האסכולה הפיתגוראית מעין כת פסבדו-מתמטית שהאמינה ש"הכל מספר", או ליתר דיוק הכל ניתן לכימות, וייחסה למספרים משמעויות מיסטיות. ככל הנראה הפיתגוראים ידעו לבנות את הגופים האפלטוניים, הכירו את הממוצע האריתמטי, הממוצע הגאומטרי והממוצע ההרמוני והגיעו להישגים חשובים נוספים. ניתן לומר שהפיתגוראים גילו את היותו של השורש הריבועי של 2, שהוא גם האלכסון בריבוע שאורך צלעותיו 1, אי רציונלי, אך תגליתם הייתה למעשה רק שהקטעים "חסרי מידה משותפת", ומושג המספר האי רציונלי מאוחר יותר.[2] אזכור ראשון לקיומם של קטעים חסרי מידה משותפת מופיע בדיאלוג "תאיטיטוס" של אפלטון, אך רעיון זה היה מוכר עוד קודם לכן, במאה החמישית לפנה"ס להיפאסוס, בן האסכולה הפיתגוראית, ואולי לפיתגורס עצמו.[3]`,
+		status: EscapeStatus{HasRTLScript: true},
+	},
+	{
+		name: "Mixed RTL+LTR",
+		text: `Many computer programs fail to display bidirectional text correctly.
+For example, the Hebrew name Sarah (שרה) is spelled: sin (ש) (which appears rightmost),
+then resh (ר), and finally heh (ה) (which should appear leftmost).`,
+		result: `Many computer programs fail to display bidirectional text correctly.
+For example, the Hebrew name Sarah (שרה) is spelled: sin (ש) (which appears rightmost),
+then resh (ר), and finally heh (ה) (which should appear leftmost).`,
+		status: EscapeStatus{
+			HasRTLScript: true,
+			HasLTRScript: true,
+		},
+	},
+	{
+		name: "Mixed RTL+LTR+BIDI",
+		text: `Many computer programs fail to display bidirectional text correctly.
+			For example, the Hebrew name Sarah ` + "\u2067" + `שרה` + "\u2066\n" +
+			`sin (ש) (which appears rightmost), then resh (ר), and finally heh (ה) (which should appear leftmost).`,
+		result: `Many computer programs fail to display bidirectional text correctly.
+			For example, the Hebrew name Sarah <span class="escaped-code-point" data-escaped="[U+2067]"><span class="char">` + "\u2067" + `</span></span>שרה<span class="escaped-code-point" data-escaped="[U+2066]"><span class="char">` + "\u2066" + `</span></span>` + "\n" +
+			`sin (ש) (which appears rightmost), then resh (ר), and finally heh (ה) (which should appear leftmost).`,
+		status: EscapeStatus{
+			Escaped:      true,
+			HasBIDI:      true,
+			HasRTLScript: true,
+			HasLTRScript: true,
+		},
+	},
+	{
+		name:   "Accented characters",
+		text:   string([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}),
+		result: string([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}),
+		status: EscapeStatus{HasLTRScript: true},
+	},
+	{
+		name:   "Program",
+		text:   "string([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})",
+		result: "string([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})",
+		status: EscapeStatus{HasLTRScript: true},
+	},
+	{
+		name:   "CVE testcase",
+		text:   "if access_level != \"user\u202E \u2066// Check if admin\u2069 \u2066\" {",
+		result: `if access_level != "user<span class="escaped-code-point" data-escaped="[U+202E]"><span class="char">` + "\u202e" + `</span></span> <span class="escaped-code-point" data-escaped="[U+2066]"><span class="char">` + "\u2066" + `</span></span>// Check if admin<span class="escaped-code-point" data-escaped="[U+2069]"><span class="char">` + "\u2069" + `</span></span> <span class="escaped-code-point" data-escaped="[U+2066]"><span class="char">` + "\u2066" + `</span></span>" {`,
+		status: EscapeStatus{Escaped: true, HasBIDI: true, BadBIDI: true, HasLTRScript: true},
+	},
+	{
+		name: "Mixed testcase with fail",
+		text: `Many computer programs fail to display bidirectional text correctly.
+			For example, the Hebrew name Sarah ` + "\u2067" + `שרה` + "\u2066\n" +
+			`sin (ש) (which appears rightmost), then resh (ר), and finally heh (ה) (which should appear leftmost).` +
+			"\nif access_level != \"user\u202E \u2066// Check if admin\u2069 \u2066\" {\n",
+		result: `Many computer programs fail to display bidirectional text correctly.
+			For example, the Hebrew name Sarah <span class="escaped-code-point" data-escaped="[U+2067]"><span class="char">` + "\u2067" + `</span></span>שרה<span class="escaped-code-point" data-escaped="[U+2066]"><span class="char">` + "\u2066" + `</span></span>` + "\n" +
+			`sin (ש) (which appears rightmost), then resh (ר), and finally heh (ה) (which should appear leftmost).` +
+			"\n" + `if access_level != "user<span class="escaped-code-point" data-escaped="[U+202E]"><span class="char">` + "\u202e" + `</span></span> <span class="escaped-code-point" data-escaped="[U+2066]"><span class="char">` + "\u2066" + `</span></span>// Check if admin<span class="escaped-code-point" data-escaped="[U+2069]"><span class="char">` + "\u2069" + `</span></span> <span class="escaped-code-point" data-escaped="[U+2066]"><span class="char">` + "\u2066" + `</span></span>" {` + "\n",
+		status: EscapeStatus{Escaped: true, HasBIDI: true, BadBIDI: true, HasLTRScript: true, HasRTLScript: true},
+	},
+}
+
+func TestEscapeControlString(t *testing.T) {
+	for _, tt := range escapeControlTests {
+		t.Run(tt.name, func(t *testing.T) {
+			status, result := EscapeControlString(tt.text)
+			if !reflect.DeepEqual(status, tt.status) {
+				t.Errorf("EscapeControlString() status = %v, wanted= %v", status, tt.status)
+			}
+			if result != tt.result {
+				t.Errorf("EscapeControlString()\nresult= %v,\nwanted= %v", result, tt.result)
+			}
+		})
+	}
+}
+
+func TestEscapeControlBytes(t *testing.T) {
+	for _, tt := range escapeControlTests {
+		t.Run(tt.name, func(t *testing.T) {
+			status, result := EscapeControlBytes([]byte(tt.text))
+			if !reflect.DeepEqual(status, tt.status) {
+				t.Errorf("EscapeControlBytes() status = %v, wanted= %v", status, tt.status)
+			}
+			if string(result) != tt.result {
+				t.Errorf("EscapeControlBytes()\nresult= %v,\nwanted= %v", result, tt.result)
+			}
+		})
+	}
+}
+
+func TestEscapeControlReader(t *testing.T) {
+	// lets add some control characters to the tests
+	tests := make([]escapeControlTest, 0, len(escapeControlTests)*3)
+	copy(tests, escapeControlTests)
+	for _, test := range escapeControlTests {
+		test.name += " (+Control)"
+		test.text = "\u001E" + test.text
+		test.result = `<span class="escaped-code-point" data-escaped="[U+001E]"><span class="char">` + "\u001e" + `</span></span>` + test.result
+		test.status.Escaped = true
+		test.status.HasControls = true
+		tests = append(tests, test)
+	}
+
+	for _, test := range escapeControlTests {
+		test.name += " (+Mark)"
+		test.text = "\u0300" + test.text
+		test.result = `<span class="escaped-code-point" data-escaped="[U+0300]"><span class="char">` + "\u0300" + `</span></span>` + test.result
+		test.status.Escaped = true
+		test.status.HasMarks = true
+		tests = append(tests, test)
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			input := strings.NewReader(tt.text)
+			output := &strings.Builder{}
+			status, err := EscapeControlReader(input, output)
+			result := output.String()
+			if err != nil {
+				t.Errorf("EscapeControlReader(): err = %v", err)
+			}
+
+			if !reflect.DeepEqual(status, tt.status) {
+				t.Errorf("EscapeControlReader() status = %v, wanted= %v", status, tt.status)
+			}
+			if result != tt.result {
+				t.Errorf("EscapeControlReader()\nresult= %v,\nwanted= %v", result, tt.result)
+			}
+		})
+	}
+}

options/locale/locale_en-US.ini

@@ -1005,6 +1005,16 @@ file_view_rendered = View Rendered
 file_view_raw = View Raw
 file_permalink = Permalink
 file_too_large = The file is too large to be shown.
+bidi_bad_header = `This file contains unexpected Bidirectional Unicode characters!`
+bidi_bad_description = `This file contains unexpected Bidirectional Unicode characters that may be processed differently from what appears below. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to reveal hidden characters.`
+bidi_bad_description_escaped = `This file contains unexpected Bidirectional Unicode characters. Hidden unicode characters are escaped below. Use the Unescape button to show how they render.`
+unicode_header = `This file contains hidden Unicode characters!`
+unicode_description = `This file contains hidden Unicode characters that may be processed differently from what appears below. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to reveal hidden characters.`
+unicode_description_escaped = `This file contains hidden Unicode characters. Hidden unicode characters are escaped below. Use the Unescape button to show how they render.`
+line_unicode = `This line has hidden unicode characters`
+
+escape_control_characters = Escape
+unescape_control_characters = Unescape
 file_copy_permalink = Copy Permalink
 video_not_supported_in_browser = Your browser does not support the HTML5 'video' tag.
 audio_not_supported_in_browser = Your browser does not support the HTML5 'audio' tag.
@@ -2101,6 +2111,7 @@ diff.protected = Protected
 diff.image.side_by_side = Side by Side
 diff.image.swipe = Swipe
 diff.image.overlay = Overlay
+diff.has_escaped = This line has hidden Unicode characters
 
 releases.desc = Track project versions and downloads.
 release.releases = Releases

routers/web/repo/blame.go

@@ -14,6 +14,7 @@ import (
 	repo_model "code.gitea.io/gitea/models/repo"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/base"
+	"code.gitea.io/gitea/modules/charset"
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/highlight"
@@ -39,6 +40,7 @@ type blameRow struct {
 	CommitMessage  string
 	CommitSince    gotemplate.HTML
 	Code           gotemplate.HTML
+	EscapeStatus   charset.EscapeStatus
 }
 
 // RefBlame render blame page
@@ -233,6 +235,7 @@ func renderBlame(ctx *context.Context, blameParts []git.BlamePart, commitNames m
 	}
 	var lines = make([]string, 0)
 	rows := make([]*blameRow, 0)
+	escapeStatus := charset.EscapeStatus{}
 
 	var i = 0
 	var commitCnt = 0
@@ -277,11 +280,14 @@ func renderBlame(ctx *context.Context, blameParts []git.BlamePart, commitNames m
 			fileName := fmt.Sprintf("%v", ctx.Data["FileName"])
 			line = highlight.Code(fileName, language, line)
 
+			br.EscapeStatus, line = charset.EscapeControlString(line)
 			br.Code = gotemplate.HTML(line)
 			rows = append(rows, br)
+			escapeStatus = escapeStatus.Or(br.EscapeStatus)
 		}
 	}
 
+	ctx.Data["EscapeStatus"] = escapeStatus
 	ctx.Data["BlameRows"] = rows
 	ctx.Data["CommitCnt"] = commitCnt
 }

routers/web/repo/lfs.go

@@ -300,10 +300,11 @@ func LFSFileGet(ctx *context.Context) {
 		rd := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc))
 
 		// Building code view blocks with line number on server side.
-		fileContent, _ := io.ReadAll(rd)
+		escapedContent := &bytes.Buffer{}
+		ctx.Data["EscapeStatus"], _ = charset.EscapeControlReader(rd, escapedContent)
 
 		var output bytes.Buffer
-		lines := strings.Split(string(fileContent), "\n")
+		lines := strings.Split(escapedContent.String(), "\n")
 		//Remove blank line at the end of file
 		if len(lines) > 0 && lines[len(lines)-1] == "" {
 			lines = lines[:len(lines)-1]

routers/web/repo/view.go

@@ -339,21 +339,24 @@ func renderDirectory(ctx *context.Context, treeLink string) {
 					}, rd, &result)
 					if err != nil {
 						log.Error("Render failed: %v then fallback", err)
-						bs, _ := io.ReadAll(rd)
+						buf := &bytes.Buffer{}
+						ctx.Data["EscapeStatus"], _ = charset.EscapeControlReader(rd, buf)
 						ctx.Data["FileContent"] = strings.ReplaceAll(
-							gotemplate.HTMLEscapeString(string(bs)), "\n", `<br>`,
+							gotemplate.HTMLEscapeString(buf.String()), "\n", `<br>`,
 						)
 					} else {
-						ctx.Data["FileContent"] = result.String()
+						ctx.Data["EscapeStatus"], ctx.Data["FileContent"] = charset.EscapeControlString(result.String())
 					}
 				} else {
 					ctx.Data["IsRenderedHTML"] = true
-					buf, err = io.ReadAll(rd)
+					buf := &bytes.Buffer{}
+					ctx.Data["EscapeStatus"], err = charset.EscapeControlReader(rd, buf)
 					if err != nil {
-						log.Error("ReadAll failed: %v", err)
+						log.Error("Read failed: %v", err)
 					}
+
 					ctx.Data["FileContent"] = strings.ReplaceAll(
-						gotemplate.HTMLEscapeString(string(buf)), "\n", `<br>`,
+						gotemplate.HTMLEscapeString(buf.String()), "\n", `<br>`,
 					)
 				}
 			}
@@ -502,12 +505,15 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
 				ctx.ServerError("Render", err)
 				return
 			}
-			ctx.Data["FileContent"] = result.String()
+			ctx.Data["EscapeStatus"], ctx.Data["FileContent"] = charset.EscapeControlString(result.String())
 		} else if readmeExist {
-			buf, _ := io.ReadAll(rd)
+			buf := &bytes.Buffer{}
 			ctx.Data["IsRenderedHTML"] = true
+
+			ctx.Data["EscapeStatus"], _ = charset.EscapeControlReader(rd, buf)
+
 			ctx.Data["FileContent"] = strings.ReplaceAll(
-				gotemplate.HTMLEscapeString(string(buf)), "\n", `<br>`,
+				gotemplate.HTMLEscapeString(buf.String()), "\n", `<br>`,
 			)
 		} else {
 			buf, _ := io.ReadAll(rd)
@@ -540,7 +546,15 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
 					language = ""
 				}
 			}
-			ctx.Data["FileContent"] = highlight.File(lineNums, blob.Name(), language, buf)
+			fileContent := highlight.File(lineNums, blob.Name(), language, buf)
+			status, _ := charset.EscapeControlReader(bytes.NewReader(buf), io.Discard)
+			ctx.Data["EscapeStatus"] = status
+			statuses := make([]charset.EscapeStatus, len(fileContent))
+			for i, line := range fileContent {
+				statuses[i], fileContent[i] = charset.EscapeControlString(line)
+			}
+			ctx.Data["FileContent"] = fileContent
+			ctx.Data["LineEscapeStatus"] = statuses
 		}
 		if !isLFSFile {
 			if ctx.Repo.CanEnableEditor() {
@@ -588,7 +602,8 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
 				ctx.ServerError("Render", err)
 				return
 			}
-			ctx.Data["FileContent"] = result.String()
+
+			ctx.Data["EscapeStatus"], ctx.Data["FileContent"] = charset.EscapeControlString(result.String())
 		}
 	}
 

routers/web/repo/wiki.go

@@ -17,6 +17,7 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/models/unit"
 	"code.gitea.io/gitea/modules/base"
+	"code.gitea.io/gitea/modules/charset"
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
@@ -232,7 +233,8 @@ func renderViewPage(ctx *context.Context) (*git.Repository, *git.TreeEntry) {
 		ctx.ServerError("Render", err)
 		return nil, nil
 	}
-	ctx.Data["content"] = buf.String()
+
+	ctx.Data["EscapeStatus"], ctx.Data["content"] = charset.EscapeControlString(buf.String())
 
 	buf.Reset()
 	if err := markdown.Render(rctx, bytes.NewReader(sidebarContent), &buf); err != nil {
@@ -243,7 +245,7 @@ func renderViewPage(ctx *context.Context) (*git.Repository, *git.TreeEntry) {
 		return nil, nil
 	}
 	ctx.Data["sidebarPresent"] = sidebarContent != nil
-	ctx.Data["sidebarContent"] = buf.String()
+	ctx.Data["sidebarEscapeStatus"], ctx.Data["sidebarContent"] = charset.EscapeControlString(buf.String())
 
 	buf.Reset()
 	if err := markdown.Render(rctx, bytes.NewReader(footerContent), &buf); err != nil {
@@ -254,7 +256,7 @@ func renderViewPage(ctx *context.Context) (*git.Repository, *git.TreeEntry) {
 		return nil, nil
 	}
 	ctx.Data["footerPresent"] = footerContent != nil
-	ctx.Data["footerContent"] = buf.String()
+	ctx.Data["footerEscapeStatus"], ctx.Data["footerContent"] = charset.EscapeControlString(buf.String())
 
 	// get commit count - wiki revisions
 	commitsCount, _ := wikiRepo.FileCommitsCount("master", pageFilename)

services/gitdiff/gitdiff.go

@@ -169,11 +169,11 @@ func getDiffLineSectionInfo(treePath, line string, lastLeftIdx, lastRightIdx int
 }
 
 // escape a line's content or return <br> needed for copy/paste purposes
-func getLineContent(content string) string {
+func getLineContent(content string) DiffInline {
 	if len(content) > 0 {
-		return html.EscapeString(content)
+		return DiffInlineWithUnicodeEscape(template.HTML(html.EscapeString(content)))
 	}
-	return "<br>"
+	return DiffInline{Content: "<br>"}
 }
 
 // DiffSection represents a section of a DiffFile.
@@ -411,7 +411,7 @@ func fixupBrokenSpans(diffs []diffmatchpatch.Diff) []diffmatchpatch.Diff {
 	return fixedup
 }
 
-func diffToHTML(fileName string, diffs []diffmatchpatch.Diff, lineType DiffLineType) template.HTML {
+func diffToHTML(fileName string, diffs []diffmatchpatch.Diff, lineType DiffLineType) DiffInline {
 	buf := bytes.NewBuffer(nil)
 	match := ""
 
@@ -483,7 +483,7 @@ func diffToHTML(fileName string, diffs []diffmatchpatch.Diff, lineType DiffLineT
 			buf.Write(codeTagSuffix)
 		}
 	}
-	return template.HTML(buf.Bytes())
+	return DiffInlineWithUnicodeEscape(template.HTML(buf.String()))
 }
 
 // GetLine gets a specific line by type (add or del) and file line number
@@ -535,10 +535,28 @@ func init() {
 	diffMatchPatch.DiffEditCost = 100
 }
 
+// DiffInline is a struct that has a content and escape status
+type DiffInline struct {
+	EscapeStatus charset.EscapeStatus
+	Content      template.HTML
+}
+
+// DiffInlineWithUnicodeEscape makes a DiffInline with hidden unicode characters escaped
+func DiffInlineWithUnicodeEscape(s template.HTML) DiffInline {
+	status, content := charset.EscapeControlString(string(s))
+	return DiffInline{EscapeStatus: status, Content: template.HTML(content)}
+}
+
+// DiffInlineWithHighlightCode makes a DiffInline with code highlight and hidden unicode characters escaped
+func DiffInlineWithHighlightCode(fileName, language, code string) DiffInline {
+	status, content := charset.EscapeControlString(highlight.Code(fileName, language, code))
+	return DiffInline{EscapeStatus: status, Content: template.HTML(content)}
+}
+
 // GetComputedInlineDiffFor computes inline diff for the given line.
-func (diffSection *DiffSection) GetComputedInlineDiffFor(diffLine *DiffLine) template.HTML {
+func (diffSection *DiffSection) GetComputedInlineDiffFor(diffLine *DiffLine) DiffInline {
 	if setting.Git.DisableDiffHighlight {
-		return template.HTML(getLineContent(diffLine.Content[1:]))
+		return getLineContent(diffLine.Content[1:])
 	}
 
 	var (
@@ -555,26 +573,26 @@ func (diffSection *DiffSection) GetComputedInlineDiffFor(diffLine *DiffLine) tem
 	// try to find equivalent diff line. ignore, otherwise
 	switch diffLine.Type {
 	case DiffLineSection:
-		return template.HTML(getLineContent(diffLine.Content[1:]))
+		return getLineContent(diffLine.Content[1:])
 	case DiffLineAdd:
 		compareDiffLine = diffSection.GetLine(DiffLineDel, diffLine.RightIdx)
 		if compareDiffLine == nil {
-			return template.HTML(highlight.Code(diffSection.FileName, language, diffLine.Content[1:]))
+			return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content[1:])
 		}
 		diff1 = compareDiffLine.Content
 		diff2 = diffLine.Content
 	case DiffLineDel:
 		compareDiffLine = diffSection.GetLine(DiffLineAdd, diffLine.LeftIdx)
 		if compareDiffLine == nil {
-			return template.HTML(highlight.Code(diffSection.FileName, language, diffLine.Content[1:]))
+			return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content[1:])
 		}
 		diff1 = diffLine.Content
 		diff2 = compareDiffLine.Content
 	default:
 		if strings.IndexByte(" +-", diffLine.Content[0]) > -1 {
-			return template.HTML(highlight.Code(diffSection.FileName, language, diffLine.Content[1:]))
+			return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content[1:])
 		}
-		return template.HTML(highlight.Code(diffSection.FileName, language, diffLine.Content))
+		return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content)
 	}
 
 	diffRecord := diffMatchPatch.DiffMain(highlight.Code(diffSection.FileName, language, diff1[1:]), highlight.Code(diffSection.FileName, language, diff2[1:]), true)

services/gitdiff/gitdiff_test.go

@@ -38,14 +38,14 @@ func TestDiffToHTML(t *testing.T) {
 		{Type: dmp.DiffInsert, Text: "bar"},
 		{Type: dmp.DiffDelete, Text: " baz"},
 		{Type: dmp.DiffEqual, Text: " biz"},
-	}, DiffLineAdd))
+	}, DiffLineAdd).Content)
 
 	assertEqual(t, "foo <span class=\"removed-code\">bar</span> biz", diffToHTML("", []dmp.Diff{
 		{Type: dmp.DiffEqual, Text: "foo "},
 		{Type: dmp.DiffDelete, Text: "bar"},
 		{Type: dmp.DiffInsert, Text: " baz"},
 		{Type: dmp.DiffEqual, Text: " biz"},
-	}, DiffLineDel))
+	}, DiffLineDel).Content)
 
 	assertEqual(t, "<span class=\"k\">if</span> <span class=\"p\">!</span><span class=\"nx\">nohl</span> <span class=\"o\">&amp;&amp;</span> <span class=\"added-code\"><span class=\"p\">(</span></span><span class=\"nx\">lexer</span> <span class=\"o\">!=</span> <span class=\"kc\">nil</span><span class=\"added-code\"> <span class=\"o\">||</span> <span class=\"nx\">r</span><span class=\"p\">.</span><span class=\"nx\">GuessLanguage</span><span class=\"p\">)</span></span> <span class=\"p\">{</span>", diffToHTML("", []dmp.Diff{
 		{Type: dmp.DiffEqual, Text: "<span class=\"k\">if</span> <span class=\"p\">!</span><span class=\"nx\">nohl</span> <span class=\"o\">&amp;&amp;</span> <span class=\""},
@@ -53,7 +53,7 @@ func TestDiffToHTML(t *testing.T) {
 		{Type: dmp.DiffEqual, Text: "nx\">lexer</span> <span class=\"o\">!=</span> <span class=\"kc\">nil"},
 		{Type: dmp.DiffInsert, Text: "</span> <span class=\"o\">||</span> <span class=\"nx\">r</span><span class=\"p\">.</span><span class=\"nx\">GuessLanguage</span><span class=\"p\">)"},
 		{Type: dmp.DiffEqual, Text: "</span> <span class=\"p\">{</span>"},
-	}, DiffLineAdd))
+	}, DiffLineAdd).Content)
 
 	assertEqual(t, "<span class=\"nx\">tagURL</span> <span class=\"o\">:=</span> <span class=\"removed-code\"><span class=\"nx\">fmt</span><span class=\"p\">.</span><span class=\"nf\">Sprintf</span><span class=\"p\">(</span><span class=\"s\">&#34;## [%s](%s/%s/%s/%s?q=&amp;type=all&amp;state=closed&amp;milestone=%d) - %s&#34;</span><span class=\"p\">,</span> <span class=\"nx\">ge</span><span class=\"p\">.</span><span class=\"nx\">Milestone\"</span></span><span class=\"p\">,</span> <span class=\"nx\">ge</span><span class=\"p\">.</span><span class=\"nx\">BaseURL</span><span class=\"p\">,</span> <span class=\"nx\">ge</span><span class=\"p\">.</span><span class=\"nx\">Owner</span><span class=\"p\">,</span> <span class=\"nx\">ge</span><span class=\"p\">.</span><span class=\"nx\">Repo</span><span class=\"p\">,</span> <span class=\"removed-code\"><span class=\"nx\">from</span><span class=\"p\">,</span> <span class=\"nx\">milestoneID</span><span class=\"p\">,</span> <span class=\"nx\">time</span><span class=\"p\">.</span><span class=\"nf\">Now</span><span class=\"p\">(</span><span class=\"p\">)</span><span class=\"p\">.</span><span class=\"nf\">Format</span><span class=\"p\">(</span><span class=\"s\">&#34;2006-01-02&#34;</span><span class=\"p\">)</span></span><span class=\"p\">)</span>", diffToHTML("", []dmp.Diff{
 		{Type: dmp.DiffEqual, Text: "<span class=\"nx\">tagURL</span> <span class=\"o\">:=</span> <span class=\"n"},
@@ -63,7 +63,7 @@ func TestDiffToHTML(t *testing.T) {
 		{Type: dmp.DiffDelete, Text: "from</span><span class=\"p\">,</span> <span class=\"nx\">milestoneID</span><span class=\"p\">,</span> <span class=\"nx\">time</span><span class=\"p\">.</span><span class=\"nf\">Now</span><span class=\"p\">(</span><span class=\"p\">)</span><span class=\"p\">.</span><span class=\"nf\">Format</span><span class=\"p\">(</span><span class=\"s\">&#34;2006-01-02&#34;</span><span class=\"p\">)"},
 		{Type: dmp.DiffInsert, Text: "ge</span><span class=\"p\">.</span><span class=\"nx\">Milestone</span><span class=\"p\">,</span> <span class=\"nx\">from</span><span class=\"p\">,</span> <span class=\"nx\">milestoneID"},
 		{Type: dmp.DiffEqual, Text: "</span><span class=\"p\">)</span>"},
-	}, DiffLineDel))
+	}, DiffLineDel).Content)
 
 	assertEqual(t, "<span class=\"nx\">r</span><span class=\"p\">.</span><span class=\"nf\">WrapperRenderer</span><span class=\"p\">(</span><span class=\"nx\">w</span><span class=\"p\">,</span> <span class=\"removed-code\"><span class=\"nx\">language</span><span class=\"p\">,</span> <span class=\"kc\">true</span><span class=\"p\">,</span> <span class=\"nx\">attrs</span></span><span class=\"p\">,</span> <span class=\"kc\">false</span><span class=\"p\">)</span>", diffToHTML("", []dmp.Diff{
 		{Type: dmp.DiffEqual, Text: "<span class=\"nx\">r</span><span class=\"p\">.</span><span class=\"nf\">WrapperRenderer</span><span class=\"p\">(</span><span class=\"nx\">w</span><span class=\"p\">,</span> <span class=\"nx\">"},
@@ -71,14 +71,14 @@ func TestDiffToHTML(t *testing.T) {
 		{Type: dmp.DiffEqual, Text: "c"},
 		{Type: dmp.DiffDelete, Text: "lass=\"p\">,</span> <span class=\"kc\">true</span><span class=\"p\">,</span> <span class=\"nx\">attrs"},
 		{Type: dmp.DiffEqual, Text: "</span><span class=\"p\">,</span> <span class=\"kc\">false</span><span class=\"p\">)</span>"},
-	}, DiffLineDel))
+	}, DiffLineDel).Content)
 
 	assertEqual(t, "<span class=\"added-code\">language</span><span class=\"p\">,</span> <span class=\"kc\">true</span><span class=\"p\">,</span> <span class=\"nx\">attrs</span></span><span class=\"p\">,</span> <span class=\"kc\">false</span><span class=\"p\">)</span>", diffToHTML("", []dmp.Diff{
 		{Type: dmp.DiffInsert, Text: "language</span><span "},
 		{Type: dmp.DiffEqual, Text: "c"},
 		{Type: dmp.DiffInsert, Text: "lass=\"p\">,</span> <span class=\"kc\">true</span><span class=\"p\">,</span> <span class=\"nx\">attrs"},
 		{Type: dmp.DiffEqual, Text: "</span><span class=\"p\">,</span> <span class=\"kc\">false</span><span class=\"p\">)</span>"},
-	}, DiffLineAdd))
+	}, DiffLineAdd).Content)
 
 	assertEqual(t, "<span class=\"k\">print</span><span class=\"added-code\"><span class=\"p\">(</span></span><span class=\"sa\"></span><span class=\"s2\">&#34;</span><span class=\"s2\">// </span><span class=\"s2\">&#34;</span><span class=\"p\">,</span> <span class=\"n\">sys</span><span class=\"o\">.</span><span class=\"n\">argv</span><span class=\"added-code\"><span class=\"p\">)</span></span>", diffToHTML("", []dmp.Diff{
 		{Type: dmp.DiffEqual, Text: "<span class=\"k\">print</span>"},
@@ -87,14 +87,14 @@ func TestDiffToHTML(t *testing.T) {
 		{Type: dmp.DiffInsert, Text: "class=\"p\">(</span>"},
 		{Type: dmp.DiffEqual, Text: "<span class=\"sa\"></span><span class=\"s2\">&#34;</span><span class=\"s2\">// </span><span class=\"s2\">&#34;</span><span class=\"p\">,</span> <span class=\"n\">sys</span><span class=\"o\">.</span><span class=\"n\">argv</span>"},
 		{Type: dmp.DiffInsert, Text: "<span class=\"p\">)</span>"},
-	}, DiffLineAdd))
+	}, DiffLineAdd).Content)
 
 	assertEqual(t, "sh <span class=\"added-code\">&#39;useradd -u $(stat -c &#34;%u&#34; .gitignore) jenkins&#39;</span>", diffToHTML("", []dmp.Diff{
 		{Type: dmp.DiffEqual, Text: "sh &#3"},
 		{Type: dmp.DiffDelete, Text: "4;useradd -u 111 jenkins&#34"},
 		{Type: dmp.DiffInsert, Text: "9;useradd -u $(stat -c &#34;%u&#34; .gitignore) jenkins&#39"},
 		{Type: dmp.DiffEqual, Text: ";"},
-	}, DiffLineAdd))
+	}, DiffLineAdd).Content)
 
 	assertEqual(t, "<span class=\"x\">							&lt;h<span class=\"added-code\">4 class=&#34;release-list-title df ac&#34;</span>&gt;</span>", diffToHTML("", []dmp.Diff{
 		{Type: dmp.DiffEqual, Text: "<span class=\"x\">							&lt;h"},
@@ -102,7 +102,7 @@ func TestDiffToHTML(t *testing.T) {
 		{Type: dmp.DiffEqual, Text: "3"},
 		{Type: dmp.DiffInsert, Text: "4;release-list-title df ac&#34;"},
 		{Type: dmp.DiffEqual, Text: "&gt;</span>"},
-	}, DiffLineAdd))
+	}, DiffLineAdd).Content)
 }
 
 func TestParsePatch_skipTo(t *testing.T) {
@@ -718,7 +718,7 @@ func TestDiffToHTML_14231(t *testing.T) {
 	expected := `		<span class="n">run</span><span class="added-code"><span class="o">(</span><span class="n">db</span></span><span class="o">)</span>`
 	output := diffToHTML("main.v", diffRecord, DiffLineAdd)
 
-	assertEqual(t, expected, output)
+	assertEqual(t, expected, output.Content)
 }
 
 func TestNoCrashes(t *testing.T) {

templates/repo/blame.tmpl

@@ -16,11 +16,13 @@
 				{{end}}
 				<a class="ui tiny button" href="{{.RepoLink}}/src/{{.BranchNameSubURL}}/{{.TreePath | PathEscapeSegments}}">{{.i18n.Tr "repo.normal_view"}}</a>
 				<a class="ui tiny button" href="{{.RepoLink}}/commits/{{.BranchNameSubURL}}/{{.TreePath | PathEscapeSegments}}">{{.i18n.Tr "repo.file_history"}}</a>
+				<a class="ui tiny button unescape-button">{{.i18n.Tr "repo.unescape_control_characters"}}</a>
+				<a class="ui tiny button escape-button" style="display: none;">{{.i18n.Tr "repo.escape_control_characters"}}</a>
 			</div>
 		</div>
 	</h4>
 	<div class="ui attached table unstackable segment">
-		<div class="file-view code-view">
+		<div class="file-view code-view unicode-escaped">
 			<table>
 				<tbody>
 					{{range $row := .BlameRows}}
@@ -52,6 +54,9 @@
 							<td class="lines-num">
 								<span id="L{{$row.RowNumber}}" data-line-number="{{$row.RowNumber}}"></span>
 							</td>
+							{{if $.EscapeStatus.Escaped}}
+								<td class="lines-escape">{{if $row.EscapeStatus.Escaped}}<a href="" class="toggle-escape-button" title="{{$.i18n.Tr "repo.line_unicode"}}"></a>{{end}}</td>
+							{{end}}
 							<td rel="L{{$row.RowNumber}}" rel="L{{$row.RowNumber}}" class="lines-code blame-code chroma">
 								<code class="code-inner pl-3">{{$row.Code}}</code>
 							</td>

templates/repo/diff/blob_excerpt.tmpl

@@ -19,14 +19,26 @@
 					</a>
 				{{end}}
 			</td>
-			<td colspan="5" class="lines-code lines-code-old "><code class="code-inner">{{$.section.GetComputedInlineDiffFor $line}}</code></td>
+			<td colspan="5" class="lines-code lines-code-old ">{{$inlineDiff := $.section.GetComputedInlineDiffFor $line}}{{$inlineDiff}}<code {{if $inlineDiff.EscapeStatus.Escaped}}class="code-inner has-escaped" title="{{$.i18n.Tr "repo.line_unicode"}}"{{else}}class="code-inner"{{end}}>{{$inlineDiff.Content}}</code></td>
 		{{else}}
 			<td class="lines-num lines-num-old" data-line-num="{{if $line.LeftIdx}}{{$line.LeftIdx}}{{end}}"><span rel="{{if $line.LeftIdx}}diff-{{Sha1 $.fileName}}L{{$line.LeftIdx}}{{end}}"></span></td>
 			<td class="blob-excerpt lines-type-marker lines-type-marker-old">{{if $line.LeftIdx}}<span class="mono" data-type-marker=""></span>{{end}}</td>
-			<td class="blob-excerpt lines-code lines-code-old halfwidth"><code class="code-inner">{{if $line.LeftIdx}}{{$.section.GetComputedInlineDiffFor $line}}{{end}}</code></td>
+			<td class="blob-excerpt lines-code lines-code-old halfwidth">{{/*
+				*/}}{{if $line.LeftIdx}}{{/*
+						*/}}{{$inlineDiff := $.section.GetComputedInlineDiffFor $line}}<code {{if $inlineDiff.EscapeStatus.Escaped}}class="code-inner has-escaped" title="{{$.i18n.Tr "repo.line_unicode"}}"{{else}}class="code-inner"{{end}}>{{$inlineDiff.Content}}</code>{{/*
+					*/}}{{else}}{{/*
+						*/}}<code class="code-inner"></code>{{/*
+					*/}}{{end}}{{/*
+				*/}}</td>
 			<td class="lines-num lines-num-new" data-line-num="{{if $line.RightIdx}}{{$line.RightIdx}}{{end}}"><span rel="{{if $line.RightIdx}}diff-{{Sha1 $.fileName}}R{{$line.RightIdx}}{{end}}"></span></td>
 			<td class="blob-excerpt lines-type-marker lines-type-marker-new">{{if $line.RightIdx}}<span class="mono" data-type-marker=""></span>{{end}}</td>
-			<td class="blob-excerpt lines-code lines-code-new halfwidth"><code class="code-inner">{{if $line.RightIdx}}{{$.section.GetComputedInlineDiffFor $line}}{{end}}</code></td>
+			<td class="blob-excerpt lines-code lines-code-new halfwidth">{{/*
+				*/}}{{if $line.RightIdx}}{{/*
+					*/}}{{$inlineDiff := $.section.GetComputedInlineDiffFor $line}}<code {{if $inlineDiff.EscapeStatus.Escaped}}class="code-inner has-escaped" title="{{$.i18n.Tr "repo.line_unicode"}}"{{else}}class="code-inner"{{end}}>{{$inlineDiff.Content}}</code>{{/*
+				*/}}{{else}}{{/*
+					*/}}<code class="code-inner"></code>{{/*
+				*/}}{{end}}{{/*
+			*/}}</td>
 		{{end}}
 	</tr>
 	{{end}}

templates/repo/diff/box.tmpl

@@ -94,6 +94,10 @@
 							{{if $file.IsProtected}}
 								<span class="ui basic label">{{$.i18n.Tr "repo.diff.protected"}}</span>
 							{{end}}
+							{{if not (or $file.IsIncomplete $file.IsBin $file.IsSubmodule)}}
+								<a class="ui basic tiny button unescape-button">{{$.i18n.Tr "repo.unescape_control_characters"}}</a>
+								<a class="ui basic tiny button escape-button" style="display: none;">{{$.i18n.Tr "repo.escape_control_characters"}}</a>
+							{{end}}
 							{{if and (not $file.IsSubmodule) (not $.PageIsWiki)}}
 								{{if $file.IsDeleted}}
 									<a class="ui basic tiny button" rel="nofollow" href="{{$.BeforeSourcePath}}/{{PathEscapeSegments .Name}}">{{$.i18n.Tr "repo.diff.view_file"}}</a>
@@ -104,7 +108,7 @@
 						</div>
 					</h4>
 					<div class="diff-file-body ui attached unstackable table segment">
-						<div id="diff-source-{{$i}}" class="file-body file-code code-diff{{if $.IsSplitStyle}} code-diff-split{{else}} code-diff-unified{{end}}{{if $showFileViewToggle}} hide{{end}}">
+						<div id="diff-source-{{$i}}" class="file-body file-code unicode-escaped code-diff{{if $.IsSplitStyle}} code-diff-split{{else}} code-diff-unified{{end}}{{if $showFileViewToggle}} hide{{end}}">
 							{{if or $file.IsIncomplete $file.IsBin}}
 								<div class="diff-file-body binary" style="padding: 5px 10px;">
 									{{if $file.IsIncomplete}}

templates/repo/diff/section_split.tmpl

@@ -21,23 +21,75 @@
 								{{svg "octicon-fold"}}
 							</a>
 						{{end}}
-					</td>
+					</td>{{$inlineDiff := $section.GetComputedInlineDiffFor $line}}
-					<td colspan="5" class="lines-code lines-code-old "><code class="code-inner">{{$section.GetComputedInlineDiffFor $line}}</span></td>
+					<td class="lines-escape lines-escape-old">{{if $inlineDiff.EscapeStatus.Escaped}}<a href="" class="toggle-escape-button" title="{{$.i18n.Tr "repo.line_unicode"}}"></a>{{end}}</td>
+					<td colspan="6" class="lines-code lines-code-old "><code {{if $inlineDiff.EscapeStatus.Escaped}}class="code-inner has-escaped" title="{{$.root.i18n.Tr "repo.line_unicode"}}"{{else}}class="code-inner"{{end}}>{{$inlineDiff.Content}}</span></td>
 				{{else if and (eq .GetType 3) $hasmatch}}{{/* DEL */}}
 					{{$match := index $section.Lines $line.Match}}
+					{{- $leftDiff := ""}}{{if $line.LeftIdx}}{{$leftDiff = $section.GetComputedInlineDiffFor $line}}{{end}}
+					{{- $rightDiff := ""}}{{if $match.RightIdx}}{{$rightDiff = $section.GetComputedInlineDiffFor $match}}{{end}}
 					<td class="lines-num lines-num-old del-code" data-line-num="{{$line.LeftIdx}}"><span rel="diff-{{Sha1 $file.Name}}L{{$line.LeftIdx}}"></span></td>
+					<td class="lines-escape del-code lines-escape-old">{{if $line.LeftIdx}}{{if $leftDiff.EscapeStatus.Escaped}}<a href="" class="toggle-escape-button" title="{{$.i18n.Tr "repo.line_unicode"}}"></a>{{end}}{{end}}</td>
 					<td class="lines-type-marker lines-type-marker-old del-code"><span class="mono" data-type-marker="{{$line.GetLineTypeMarker}}"></span></td>
-					<td class="lines-code lines-code-old halfwidth del-code">{{if and $.root.SignedUserID $.root.PageIsPullFiles}}<a class="ui primary button add-code-comment add-code-comment-left{{if (not $line.CanComment)}} invisible{{end}}" data-side="left" data-idx="{{$line.LeftIdx}}">{{svg "octicon-plus"}}</a>{{end}}<code class="code-inner">{{if $line.LeftIdx}}{{$section.GetComputedInlineDiffFor $line}}{{end}}</code></td>
+					<td class="lines-code lines-code-old halfwidth del-code">{{/*
+						*/}}{{if and $.root.SignedUserID $.root.PageIsPullFiles}}{{/*
+							*/}}<a class="ui primary button add-code-comment add-code-comment-left{{if (not $line.CanComment)}} invisible{{end}}" data-side="left" data-idx="{{$line.LeftIdx}}">{{/*
+								*/}}{{svg "octicon-plus"}}{{/*
+							*/}}</a>{{/*
+						*/}}{{end}}{{/*
+						*/}}{{if $line.LeftIdx}}{{/*
+							*/}}<code {{if $leftDiff.EscapeStatus.Escaped}}class="code-inner has-escaped" title="{{$.root.i18n.Tr "repo.line_unicode"}}"{{else}}class="code-inner"{{end}}>{{$leftDiff.Content}}</code>{{/*
+						*/}}{{else}}{{/*
+						*/}}<code class="code-inner"></code>{{/*
+						*/}}{{end}}{{/*
+					*/}}</td>
 					<td class="lines-num lines-num-new add-code" data-line-num="{{if $match.RightIdx}}{{$match.RightIdx}}{{end}}"><span rel="{{if $match.RightIdx}}diff-{{Sha1 $file.Name}}R{{$match.RightIdx}}{{end}}"></span></td>
+					<td class="lines-escape add-code lines-escape-new">{{if $match.RightIdx}}{{if $rightDiff.EscapeStatus.Escaped}}<a href="" class="toggle-escape-button" title="{{$.i18n.Tr "repo.line_unicode"}}"></a>{{end}}{{end}}</td>
 					<td class="lines-type-marker lines-type-marker-new add-code">{{if $match.RightIdx}}<span class="mono" data-type-marker="{{$match.GetLineTypeMarker}}"></span>{{end}}</td>
-					<td class="lines-code lines-code-new halfwidth add-code">{{if and $.root.SignedUserID $.root.PageIsPullFiles}}<a class="ui primary button add-code-comment add-code-comment-right{{if (not $match.CanComment)}} invisible{{end}}" data-side="right" data-idx="{{$match.RightIdx}}">{{svg "octicon-plus"}}</a>{{end}}<code class="code-inner">{{if $match.RightIdx}}{{$section.GetComputedInlineDiffFor $match}}{{end}}</code></td>
+					<td class="lines-code lines-code-new halfwidth add-code">{{/*
+						*/}}{{if and $.root.SignedUserID $.root.PageIsPullFiles}}{{/*
+							*/}}<a class="ui primary button add-code-comment add-code-comment-right{{if (not $match.CanComment)}} invisible{{end}}" data-side="right" data-idx="{{$match.RightIdx}}">{{/*
+								*/}}{{svg "octicon-plus"}}{{/*
+							*/}}</a>{{/*
+						*/}}{{end}}{{/*
+						*/}}{{if $match.RightIdx}}{{/*
+							*/}}<code {{if $rightDiff.EscapeStatus.Escaped}}class="code-inner has-escaped" title="{{$.root.i18n.Tr "repo.line_unicode"}}"{{else}}class="code-inner"{{end}}>{{$rightDiff.Content}}</code>{{/*
+						*/}}{{else}}{{/*
+							*/}}<code class="code-inner"></code>{{/*
+						*/}}{{end}}{{/*
+					*/}}</td>
 				{{else}}
+					{{$inlineDiff := $section.GetComputedInlineDiffFor $line}}
 					<td class="lines-num lines-num-old" data-line-num="{{if $line.LeftIdx}}{{$line.LeftIdx}}{{end}}"><span rel="{{if $line.LeftIdx}}diff-{{Sha1 $file.Name}}L{{$line.LeftIdx}}{{end}}"></span></td>
+					<td class="lines-escape lines-escape-old">{{if $line.LeftIdx}}{{if $inlineDiff.EscapeStatus.Escaped}}<a href="" class="toggle-escape-button" title="{{$.i18n.Tr "repo.line_unicode"}}"></a>{{end}}{{end}}</td>
 					<td class="lines-type-marker lines-type-marker-old">{{if $line.LeftIdx}}<span class="mono" data-type-marker="{{$line.GetLineTypeMarker}}"></span>{{end}}</td>
-					<td class="lines-code lines-code-old halfwidth">{{if and $.root.SignedUserID $.root.PageIsPullFiles (not (eq .GetType 2))}}<a class="ui primary button add-code-comment add-code-comment-left{{if (not $line.CanComment)}} invisible{{end}}" data-side="left" data-idx="{{$line.LeftIdx}}">{{svg "octicon-plus"}}</a>{{end}}<code class="code-inner">{{if $line.LeftIdx}}{{$section.GetComputedInlineDiffFor $line}}{{end}}</code></td>
+					<td class="lines-code lines-code-old halfwidth">{{/*
+						*/}}{{if and $.root.SignedUserID $.root.PageIsPullFiles (not (eq .GetType 2))}}{{/*
+							*/}}<a class="ui primary button add-code-comment add-code-comment-left{{if (not $line.CanComment)}} invisible{{end}}" data-side="left" data-idx="{{$line.LeftIdx}}">{{/*
+								*/}}{{svg "octicon-plus"}}{{/*
+							*/}}</a>{{/*
+						*/}}{{end}}{{/*
+						*/}}{{if $line.LeftIdx}}{{/*
+							*/}}<code {{if $inlineDiff.EscapeStatus.Escaped}}class="code-inner has-escaped" title="{{$.root.i18n.Tr "repo.line_unicode"}}"{{else}}class="code-inner"{{end}}>{{$inlineDiff.Content}}</code>{{/*
+						*/}}{{else}}{{/*
+						*/}}<code class="code-inner"></code>{{/*
+						*/}}{{end}}{{/*
+					*/}}</td>
 					<td class="lines-num lines-num-new" data-line-num="{{if $line.RightIdx}}{{$line.RightIdx}}{{end}}"><span rel="{{if $line.RightIdx}}diff-{{Sha1 $file.Name}}R{{$line.RightIdx}}{{end}}"></span></td>
+					<td class="lines-escape lines-escape-new">{{if $line.RightIdx}}{{if $inlineDiff.EscapeStatus.Escaped}}<a href="" class="toggle-escape-button" title="{{$.i18n.Tr "repo.line_unicode"}}"></a>{{end}}{{end}}</td>
 					<td class="lines-type-marker lines-type-marker-new">{{if $line.RightIdx}}<span class="mono" data-type-marker="{{$line.GetLineTypeMarker}}"></span>{{end}}</td>
-					<td class="lines-code lines-code-new halfwidth">{{if and $.root.SignedUserID $.root.PageIsPullFiles (not (eq .GetType 3))}}<a class="ui primary button add-code-comment add-code-comment-right{{if (not $line.CanComment)}} invisible{{end}}" data-side="right" data-idx="{{$line.RightIdx}}">{{svg "octicon-plus"}}</a>{{end}}<code class="code-inner">{{if $line.RightIdx}}{{$section.GetComputedInlineDiffFor $line}}{{end}}</code></td>
+					<td class="lines-code lines-code-new halfwidth">{{/*
+						*/}}{{if and $.root.SignedUserID $.root.PageIsPullFiles (not (eq .GetType 3))}}{{/*
+							*/}}<a class="ui primary button add-code-comment add-code-comment-right{{if (not $line.CanComment)}} invisible{{end}}" data-side="right" data-idx="{{$line.RightIdx}}">{{/*
+								*/}}{{svg "octicon-plus"}}{{/*
+							*/}}</a>{{/*
+						*/}}{{end}}{{/*
+						*/}}{{if $line.RightIdx}}{{/*
+							*/}}<code {{if $inlineDiff.EscapeStatus.Escaped}}class="code-inner has-escaped" title="{{$.root.i18n.Tr "repo.line_unicode"}}"{{else}}class="code-inner"{{end}}>{{$inlineDiff.Content}}</code>{{/*
+						*/}}{{else}}{{/*
+						*/}}<code class="code-inner"></code>{{/*
+						*/}}{{end}}{{/*
+					*/}}</td>
 				{{end}}
 			</tr>
 			{{if and (eq .GetType 3) $hasmatch}}
@@ -45,6 +97,7 @@
 				{{if or (gt (len $line.Comments) 0) (gt (len $match.Comments) 0)}}
 					<tr class="add-comment" data-line-type="{{DiffLineTypeToStr .GetType}}">
 						<td class="lines-num"></td>
+						<td class="lines-escape"></td>
 						<td class="lines-type-marker"></td>
 						<td class="add-comment-left">
 							{{if gt (len $line.Comments) 0}}
@@ -59,6 +112,7 @@
 							{{end}}
 						</td>
 						<td class="lines-num"></td>
+						<td class="lines-escape"></td>
 						<td class="lines-type-marker"></td>
 						<td class="add-comment-right">
 							{{if eq $line.GetCommentSide "proposed"}}
@@ -75,6 +129,7 @@
 			{{else if gt (len $line.Comments) 0}}
 				<tr class="add-comment" data-line-type="{{DiffLineTypeToStr .GetType}}">
 					<td class="lines-num"></td>
+					<td class="lines-escape"></td>
 					<td class="lines-type-marker"></td>
 					<td class="add-comment-left">
 						{{if gt (len $line.Comments) 0}}
@@ -84,6 +139,7 @@
 						{{end}}
 					</td>
 					<td class="lines-num"></td>
+					<td class="lines-escape"></td>
 					<td class="lines-type-marker"></td>
 					<td class="add-comment-right">
 						{{if eq $line.GetCommentSide "proposed"}}

templates/repo/diff/section_unified.tmpl

@@ -25,16 +25,29 @@
 					<td class="lines-num lines-num-old" data-line-num="{{if $line.LeftIdx}}{{$line.LeftIdx}}{{end}}"><span rel="{{if $line.LeftIdx}}diff-{{Sha1 $file.Name}}L{{$line.LeftIdx}}{{end}}"></span></td>
 					<td class="lines-num lines-num-new" data-line-num="{{if $line.RightIdx}}{{$line.RightIdx}}{{end}}"><span rel="{{if $line.RightIdx}}diff-{{Sha1 $file.Name}}R{{$line.RightIdx}}{{end}}"></span></td>
 				{{end}}
+				{{$inlineDiff := $section.GetComputedInlineDiffFor $line -}}
+				<td class="lines-escape">{{if $inlineDiff.EscapeStatus.Escaped}}<a href="" class="toggle-escape-button" title="{{$.i18n.Tr "repo.line_unicode"}}"></a>{{end}}</td>
 				<td class="lines-type-marker"><span class="mono" data-type-marker="{{$line.GetLineTypeMarker}}"></span></td>
 				{{if eq .GetType 4}}
-					<td class="chroma lines-code blob-hunk"><code class="code-inner">{{$section.GetComputedInlineDiffFor $line}}</code></td>
+					<td class="chroma lines-code blob-hunk">{{/*
+						*/}}<code {{if $inlineDiff.EscapeStatus.Escaped}}class="code-inner has-escaped" title="{{$.root.i18n.Tr "repo.line_unicode"}}"{{else}}class="code-inner"{{end}}>{{$inlineDiff.Content}}</code>{{/*
+					*/}}
+				{{$line.Content}}
+					</td>
 				{{else}}
-					<td class="chroma lines-code{{if (not $line.RightIdx)}} lines-code-old{{end}}">{{if and $.root.SignedUserID $.root.PageIsPullFiles}}<a class="ui primary button add-code-comment add-code-comment-{{if $line.RightIdx}}right{{else}}left{{end}}{{if (not $line.CanComment)}} invisible{{end}}" data-side="{{if $line.RightIdx}}right{{else}}left{{end}}" data-idx="{{if $line.RightIdx}}{{$line.RightIdx}}{{else}}{{$line.LeftIdx}}{{end}}">{{svg "octicon-plus"}}</a>{{end}}<code class="code-inner">{{$section.GetComputedInlineDiffFor $line}}</code></td>
+					<td class="chroma lines-code{{if (not $line.RightIdx)}} lines-code-old{{end}}">{{/*
+						*/}}{{if and $.root.SignedUserID $.root.PageIsPullFiles}}{{/*
+							*/}}<a class="ui primary button add-code-comment add-code-comment-{{if $line.RightIdx}}right{{else}}left{{end}}{{if (not $line.CanComment)}} invisible{{end}}" data-side="{{if $line.RightIdx}}right{{else}}left{{end}}" data-idx="{{if $line.RightIdx}}{{$line.RightIdx}}{{else}}{{$line.LeftIdx}}{{end}}">{{/*
+								*/}}{{svg "octicon-plus"}}{{/*
+							*/}}</a>{{/*
+						*/}}{{end}}{{/*
+						*/}}<code {{if $inlineDiff.EscapeStatus.Escaped}}class="code-inner has-escaped" title="{{$.root.i18n.Tr "repo.line_unicode"}}"{{else}}class="code-inner"{{end}}>{{$inlineDiff.Content}}</code>{{/*
+					*/}}</td>
 				{{end}}
 			</tr>
 			{{if gt (len $line.Comments) 0}}
 				<tr class="add-comment" data-line-type="{{DiffLineTypeToStr .GetType}}">
-					<td colspan="2" class="lines-num"></td>
+					<td colspan="3" class="lines-num"></td>
 					<td class="add-comment-left add-comment-right" colspan="2">
 						{{template "repo/diff/conversation" mergeinto $.root "comments" $line.Comments}}
 					</td>

templates/repo/editor/diff_preview.tmpl

@@ -1,6 +1,6 @@
 <div class="diff-file-box">
 	<div class="ui attached table segment">
-		<div class="file-body file-code code-view code-diff-unified">
+		<div class="file-body file-code code-view code-diff-unified unicode-escaped">
 			<table>
 				<tbody>
 					{{template "repo/diff/section_unified" dict "file" .File "root" $}}

templates/repo/issue/view_content/comments.tmpl

@@ -512,7 +512,7 @@
 									{{$file := (index $diff.Files 0)}}
 									<div id="code-preview-{{(index $comms 0).ID}}" class="ui table segment{{if $resolved}} hide{{end}}">
 										<div class="diff-file-box diff-box file-content {{TabSizeClass $.Editorconfig $file.Name}}">
-											<div class="file-body file-code code-view code-diff code-diff-unified">
+											<div class="file-body file-code code-view code-diff code-diff-unified unicode-escaped">
 												<table>
 													<tbody>
 														{{template "repo/diff/section_unified" dict "file" $file "root" $}}

templates/repo/settings/lfs_file.tmpl

@@ -8,10 +8,15 @@
 			<h4 class="ui top attached header">
 				<a href="{{.LFSFilesLink}}">{{.i18n.Tr "repo.settings.lfs"}}</a> / <span class="truncate sha">{{.LFSFile.Oid}}</span>
 				<div class="ui right">
+					{{if .EscapeStatus.Escaped}}
+						<a class="ui mini basic button unescape-button" style="display: none;">{{.i18n.Tr "repo.unescape_control_characters"}}</a>
+						<a class="ui mini basic button escape-button">{{.i18n.Tr "repo.escape_control_characters"}}</a>
+					{{end}}
 					<a class="ui blue show-panel button" href="{{.LFSFilesLink}}/find?oid={{.LFSFile.Oid}}&size={{.LFSFile.Size}}">{{$.i18n.Tr "repo.settings.lfs_findcommits"}}</a>
 				</div>
 			</h4>
 			<div class="ui attached table unstackable segment">
+				{{template "repo/unicode_escape_prompt" dict "EscapeStatus" .EscapeStatus "root" $}}
 				<div class="file-view{{if .IsMarkup}} markup {{.MarkupType}}{{else if .IsRenderedHTML}} plain-text{{else if .IsTextFile}} code-view{{end}}">
 					{{if .IsMarkup}}
 						{{if .FileContent}}{{.FileContent | Safe}}{{end}}

templates/repo/unicode_escape_prompt.tmpl

@@ -0,0 +1,17 @@
+{{if .EscapeStatus.BadBIDI}}
+<div class="ui error message unicode-escape-prompt">
+	<span class="close icon hide-panel button" data-panel-closest=".message">{{svg "octicon-x" 16 "close inside"}}</span>
+	<div class="header">
+		{{$.root.i18n.Tr "repo.bidi_bad_header"}}
+	</div>
+	<p>{{$.root.i18n.Tr "repo.bidi_bad_description" | Str2html}}</p>
+</div>
+{{else if .EscapeStatus.Escaped}}
+<div class="ui warning message unicode-escape-prompt">
+	<span class="close icon hide-panel button" data-panel-closest=".message">{{svg "octicon-x" 16 "close inside"}}</span>
+	<div class="header">
+		{{$.root.i18n.Tr "repo.unicode_header"}}
+	</div>
+	<p>{{$.root.i18n.Tr "repo.unicode_description" | Str2html}}</p>
+</div>
+{{end}}

templates/repo/view_file.tmpl

@@ -30,7 +30,6 @@
 				</div>
 			{{end}}
 		</div>
-		{{if not .ReadmeInList}}
 		<div class="file-header-right file-actions df ac">
 			{{if .HasSourceRenderedToggle}}
 				<div class="ui compact icon buttons two-toggle-buttons">
@@ -38,6 +37,7 @@
 					<a href="{{$.Link}}" class="ui mini basic button tooltip {{if .IsDisplayingRendered}}active{{end}}" data-content="{{.i18n.Tr "repo.file_view_rendered"}}" data-position="bottom center">{{svg "octicon-file" 15}}</a>
 				</div>
 			{{end}}
+			{{if not .ReadmeInList}}
 				<div class="ui buttons mr-2">
 					<a class="ui mini basic button" href="{{$.RawFileLink}}">{{.i18n.Tr "repo.file_raw"}}</a>
 					{{if not .IsViewCommit}}
@@ -47,6 +47,10 @@
 						<a class="ui mini basic button" href="{{.RepoLink}}/blame/{{.BranchNameSubURL}}/{{PathEscapeSegments .TreePath}}">{{.i18n.Tr "repo.blame"}}</a>
 					{{end}}
 					<a class="ui mini basic button" href="{{.RepoLink}}/commits/{{.BranchNameSubURL}}/{{PathEscapeSegments .TreePath}}">{{.i18n.Tr "repo.file_history"}}</a>
+					{{if .EscapeStatus.Escaped}}
+						<a class="ui mini basic button unescape-button" style="display: none;">{{.i18n.Tr "repo.unescape_control_characters"}}</a>
+						<a class="ui mini basic button escape-button">{{.i18n.Tr "repo.escape_control_characters"}}</a>
+					{{end}}
 				</div>
 				<a download href="{{$.RawFileLink}}"><span class="btn-octicon tooltip" data-content="{{.i18n.Tr "repo.download_file"}}" data-position="bottom center">{{svg "octicon-download"}}</span></a>
 				{{if .Repository.CanEnableEditor}}
@@ -61,10 +65,14 @@
 						<span class="btn-octicon tooltip disabled" data-content="{{.DeleteFileTooltip}}" data-position="bottom center">{{svg "octicon-trash"}}</span>
 					{{end}}
 				{{end}}
-		</div>
+			{{else if .EscapeStatus.Escaped}}
+				<a class="ui mini basic button unescape-button mr-2" style="display: none;">{{.i18n.Tr "repo.unescape_control_characters"}}</a>
+				<a class="ui mini basic button escape-button mr-2">{{.i18n.Tr "repo.escape_control_characters"}}</a>
 			{{end}}
+		</div>
 	</h4>
 	<div class="ui attached table unstackable segment">
+		{{template "repo/unicode_escape_prompt" dict "EscapeStatus" .EscapeStatus "root" $}}
 		<div class="file-view{{if .IsMarkup}} markup {{.MarkupType}}{{else if .IsRenderedHTML}} plain-text{{else if .IsTextSource}} code-view{{end}}">
 			{{if .IsMarkup}}
 				{{if .FileContent}}{{.FileContent | Safe}}{{end}}
@@ -104,6 +112,9 @@
 						{{$line := Add $idx 1}}
 						<tr>
 							<td id="L{{$line}}" class="lines-num"><span id="L{{$line}}" data-line-number="{{$line}}"></span></td>
+							{{if $.EscapeStatus.Escaped}}
+								<td class="lines-escape">{{if (index $.LineEscapeStatus $idx).Escaped}}<a href="" class="toggle-escape-button" title="{{$.i18n.Tr "repo.line_unicode"}}"></a>{{end}}</td>
+							{{end}}
 							<td rel="L{{$line}}" class="lines-code chroma"><code class="code-inner">{{$code | Safe}}</code></td>
 						</tr>
 						{{end}}

templates/repo/wiki/view.tmpl

@@ -45,6 +45,10 @@
 					</div>
 				</div>
 				<div class="eight wide right aligned column">
+					{{if .EscapeStatus.Escaped}}
+						<a class="ui small button unescape-button" style="display: none;">{{.i18n.Tr "repo.unescape_control_characters"}}</a>
+						<a class="ui small button escape-button">{{.i18n.Tr "repo.escape_control_characters"}}</a>
+					{{end}}
 					{{if and .CanWriteWiki (not .Repository.IsMirror)}}
 						<div class="ui right">
 							<a class="ui small button" href="{{.RepoLink}}/wiki/{{.PageURL}}?action=_edit">{{.i18n.Tr "repo.wiki.edit_page_button"}}</a>
@@ -61,26 +65,29 @@
 			</div>
 		{{end}}
 		<div class="ui {{if .sidebarPresent}}grid equal width{{end}}" style="margin-top: 1rem;">
-			<div class="ui {{if .sidebarPresent}}eleven wide column{{end}} segment markup">
+			<div class="ui {{if .sidebarPresent}}eleven wide column{{end}} segment markup wiki-content-main">
-				{{.content | Str2html}}
+				{{template "repo/unicode_escape_prompt" dict "EscapeStatus" .EscapeStatus "root" $}}
+				{{.content | Safe}}
 			</div>
 			{{if .sidebarPresent}}
 			<div class="column" style="padding-top: 0;">
-				<div class="ui segment">
+				<div class="ui segment wiki-content-sidebar">
 					{{if and .CanWriteWiki (not .Repository.IsMirror)}}
 						<a class="ui right floated muted" href="{{.RepoLink}}/wiki/_Sidebar?action=_edit" aria-label="{{.i18n.Tr "repo.wiki.edit_page_button"}}">{{svg "octicon-pencil"}}</a>
 					{{end}}
-					{{.sidebarContent | Str2html}}
+					{{template "repo/unicode_escape_prompt" dict "EscapeStatus" .sidebarEscapeStatus "root" $}}
+					{{.sidebarContent | Safe}}
 				</div>
 			</div>
 			{{end}}
 		</div>
 		{{if .footerPresent}}
-		<div class="ui segment">
+		<div class="ui segment wiki-content-footer">
 			{{if and .CanWriteWiki (not .Repository.IsMirror)}}
 				<a class="ui right floated muted" href="{{.RepoLink}}/wiki/_Footer?action=_edit" aria-label="{{.i18n.Tr "repo.wiki.edit_page_button"}}">{{svg "octicon-pencil"}}</a>
 			{{end}}
-			{{.footerContent | Str2html}}
+			{{template "repo/unicode_escape_prompt" dict "footerEscapeStatus" .sidebarEscapeStatus "root" $}}
+			{{.footerContent | Safe}}
 		</div>
 		{{end}}
 	</div>

web_src/js/features/common-global.js

@@ -297,8 +297,20 @@ export function initGlobalButtons() {
   });
 
   $('.hide-panel.button').on('click', function (event) {
-    $($(this).data('panel')).hide();
+    // a `.hide-panel.button` can hide a panel, by `data-panel="selector"` or `data-panel-closest="selector"`
     event.preventDefault();
+    let sel = $(this).attr('data-panel');
+    if (sel) {
+      $(sel).hide();
+      return;
+    }
+    sel = $(this).attr('data-panel-closest');
+    if (sel) {
+      $(this).closest(sel).hide();
+      return;
+    }
+    // should never happen, otherwise there is a bug in code
+    alert('Nothing to hide');
   });
 
   $('.show-modal.button').on('click', function () {

web_src/js/features/repo-legacy.js

@@ -10,6 +10,7 @@ import {
   initRepoIssueWipToggle, initRepoPullRequestMerge, initRepoPullRequestUpdate,
   updateIssuesMeta,
 } from './repo-issue.js';
+import {initUnicodeEscapeButton} from './repo-unicode-escape.js';
 import {svg} from '../svg.js';
 import {htmlEscape} from 'escape-goat';
 import {initRepoBranchTagDropdown} from '../components/RepoBranchTagDropdown.js';
@@ -533,6 +534,8 @@ export function initRepository() {
       easyMDE.codemirror.refresh();
     });
   }
+
+  initUnicodeEscapeButton();
 }
 
 function initRepoIssueCommentEdit() {

web_src/js/features/repo-unicode-escape.js

@@ -0,0 +1,28 @@
+export function initUnicodeEscapeButton() {
+  $(document).on('click', 'a.escape-button', (e) => {
+    e.preventDefault();
+    $(e.target).parents('.file-content, .non-diff-file-content').find('.file-code, .file-view').addClass('unicode-escaped');
+    $(e.target).hide();
+    $(e.target).siblings('a.unescape-button').show();
+  });
+  $(document).on('click', 'a.unescape-button', (e) => {
+    e.preventDefault();
+    $(e.target).parents('.file-content, .non-diff-file-content').find('.file-code, .file-view').removeClass('unicode-escaped');
+    $(e.target).hide();
+    $(e.target).siblings('a.escape-button').show();
+  });
+  $(document).on('click', 'a.toggle-escape-button', (e) => {
+    e.preventDefault();
+    const fileContent = $(e.target).parents('.file-content, .non-diff-file-content');
+    const fileView = fileContent.find('.file-code, .file-view');
+    if (fileView.hasClass('unicode-escaped')) {
+      fileView.removeClass('unicode-escaped');
+      fileContent.find('a.unescape-button').hide();
+      fileContent.find('a.escape-button').show();
+    } else {
+      fileView.addClass('unicode-escaped');
+      fileContent.find('a.unescape-button').show();
+      fileContent.find('a.escape-button').hide();
+    }
+  });
+}

web_src/less/_base.less

@@ -668,6 +668,12 @@ a.ui.card:hover,
   color: var(--color-text-dark);
 }
 
+.ui.error.message .header,
+.ui.warning.message .header {
+  color: inherit;
+  filter: saturate(2);
+}
+
 .dont-break-out {
   overflow-wrap: break-word;
   word-wrap: break-word;
@@ -1569,6 +1575,10 @@ a.ui.label:hover {
   }
 }
 
+.lines-escape {
+  width: 0;
+}
+
 .lines-code {
   background-color: var(--color-code-bg);
   padding-left: 5px;

web_src/less/_repository.less

@@ -76,6 +76,24 @@
     }
   }
 
+  .unicode-escaped .escaped-code-point {
+    &[data-escaped]::before {
+      visibility: visible;
+      content: attr(data-escaped);
+      font-family: var(--fonts-monospace);
+      color: var(--color-red);
+    }
+
+    .char {
+      display: none;
+    }
+  }
+
+  .broken-code-point {
+    font-family: var(--fonts-monospace);
+    color: blue;
+  }
+
   .metas {
     .menu {
       overflow-x: auto;
@@ -3020,6 +3038,26 @@ td.blob-excerpt {
   padding-left: 8px;
 }
 
+.ui.message.unicode-escape-prompt {
+  margin-bottom: 0;
+  border-radius: 0;
+  display: flex;
+  flex-direction: column;
+}
+
+.wiki-content-sidebar .ui.message.unicode-escape-prompt,
+.wiki-content-footer .ui.message.unicode-escape-prompt {
+  p {
+    display: none;
+  }
+}
+
+/* fomantic's last-child selector does not work with hidden last child */
+.ui.buttons .unescape-button {
+  border-top-right-radius: .28571429rem;
+  border-bottom-right-radius: .28571429rem;
+}
+
 .webhook-info {
   padding: 7px 12px;
   margin: 10px 0;
@@ -3110,6 +3148,7 @@ td.blob-excerpt {
 .code-diff-unified .del-code,
 .code-diff-unified .del-code td,
 .code-diff-split .del-code .lines-num-old,
+.code-diff-split .del-code .lines-escape-old,
 .code-diff-split .del-code .lines-type-marker-old,
 .code-diff-split .del-code .lines-code-old {
   background: var(--color-diff-removed-row-bg);
@@ -3120,9 +3159,11 @@ td.blob-excerpt {
 .code-diff-unified .add-code td,
 .code-diff-split .add-code .lines-num-new,
 .code-diff-split .add-code .lines-type-marker-new,
+.code-diff-split .add-code .lines-escape-new,
 .code-diff-split .add-code .lines-code-new,
 .code-diff-split .del-code .add-code.lines-num-new,
 .code-diff-split .del-code .add-code.lines-type-marker-new,
+.code-diff-split .del-code .add-code.lines-escape-new,
 .code-diff-split .del-code .add-code.lines-code-new {
   background: var(--color-diff-added-row-bg);
   border-color: var(--color-diff-added-row-border);
@@ -3131,7 +3172,9 @@ td.blob-excerpt {
 .code-diff-split .del-code .lines-num-new,
 .code-diff-split .del-code .lines-type-marker-new,
 .code-diff-split .del-code .lines-code-new,
+.code-diff-split .del-code .lines-escape-new,
 .code-diff-split .add-code .lines-num-old,
+.code-diff-split .add-code .lines-escape-old,
 .code-diff-split .add-code .lines-type-marker-old,
 .code-diff-split .add-code .lines-code-old {
   background: var(--color-diff-inactive);

web_src/less/_review.less

@@ -16,6 +16,17 @@
   }
 }
 
+.lines-escape a.toggle-escape-button::before {
+  visibility: visible;
+  content: '⚠️';
+  font-family: var(--fonts-emoji);
+  color: var(--color-red);
+}
+
+.repository .diff-file-box .code-diff td.lines-escape {
+  padding-left: 0 !important;
+}
+
 .diff-file-box .lines-code:hover .ui.button.add-code-comment {
   opacity: 1;
 }