Keep languages defined in .gitattributes (#21403)

Fixes #21377

This marks all "defined" languages in the `.gitattributes` file so they
are not removed if they are not of type `programming` or `markup`.


![grafik](https://user-images.githubusercontent.com/1666336/194942021-1e641b60-bb8a-49c6-9a1c-413e7c4ba17d.png)

Co-authored-by: zeripath <art27@cantab.net>
Co-authored-by: Lauris BH <lauris@nix.lv>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>

modules/git/repo_language_stats_gogit.go

@@ -44,7 +44,15 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 	checker, deferable := repo.CheckAttributeReader(commitID)
 	defer deferable()
 
+	// sizes contains the current calculated size of all files by language
 	sizes := make(map[string]int64)
+	// by default we will only count the sizes of programming languages or markup languages
+	// unless they are explicitly set using linguist-language
+	includedLanguage := map[string]bool{}
+	// or if there's only one language in the repository
+	firstExcludedLanguage := ""
+	firstExcludedLanguageSize := int64(0)
+
 	err = tree.Files().ForEach(func(f *object.File) error {
 		if f.Size == 0 {
 			return nil
@@ -75,8 +83,8 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 						language = group
 					}
 
+					// this language will always be added to the size
 					sizes[language] += f.Size
-
 					return nil
 				} else if language, has := attrs["gitlab-language"]; has && language != "unspecified" && language != "" {
 					// strip off a ? if present
@@ -90,6 +98,7 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 							language = group
 						}
 
+						// this language will always be added to the size
 						sizes[language] += f.Size
 						return nil
 					}
@@ -124,7 +133,18 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 			language = group
 		}
 
-		sizes[language] += f.Size
+		included, checked := includedLanguage[language]
+		if !checked {
+			langtype := enry.GetLanguageType(language)
+			included = langtype == enry.Programming || langtype == enry.Markup
+			includedLanguage[language] = included
+		}
+		if included {
+			sizes[language] += f.Size
+		} else if len(sizes) == 0 && (firstExcludedLanguage == "" || firstExcludedLanguage == language) {
+			firstExcludedLanguage = language
+			firstExcludedLanguageSize += f.Size
+		}
 
 		return nil
 	})
@@ -132,14 +152,9 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 		return nil, err
 	}
 
-	// filter special languages unless they are the only language
+	// If there are no included languages add the first excluded language
-	if len(sizes) > 1 {
+	if len(sizes) == 0 && firstExcludedLanguage != "" {
-		for language := range sizes {
+		sizes[firstExcludedLanguage] = firstExcludedLanguageSize
-			langtype := enry.GetLanguageType(language)
-			if langtype != enry.Programming && langtype != enry.Markup {
-				delete(sizes, language)
-			}
-		}
 	}
 
 	return sizes, nil

modules/git/repo_language_stats_nogogit.go

@@ -67,7 +67,16 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 
 	contentBuf := bytes.Buffer{}
 	var content []byte
+
+	// sizes contains the current calculated size of all files by language
 	sizes := make(map[string]int64)
+	// by default we will only count the sizes of programming languages or markup languages
+	// unless they are explicitly set using linguist-language
+	includedLanguage := map[string]bool{}
+	// or if there's only one language in the repository
+	firstExcludedLanguage := ""
+	firstExcludedLanguageSize := int64(0)
+
 	for _, f := range entries {
 		select {
 		case <-repo.Ctx.Done():
@@ -107,6 +116,7 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 						language = group
 					}
 
+					// this language will always be added to the size
 					sizes[language] += f.Size()
 					continue
 				} else if language, has := attrs["gitlab-language"]; has && language != "unspecified" && language != "" {
@@ -121,6 +131,7 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 							language = group
 						}
 
+						// this language will always be added to the size
 						sizes[language] += f.Size()
 						continue
 					}
@@ -180,18 +191,24 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 			language = group
 		}
 
-		sizes[language] += f.Size()
+		included, checked := includedLanguage[language]
+		if !checked {
+			langtype := enry.GetLanguageType(language)
+			included = langtype == enry.Programming || langtype == enry.Markup
+			includedLanguage[language] = included
+		}
+		if included {
+			sizes[language] += f.Size()
+		} else if len(sizes) == 0 && (firstExcludedLanguage == "" || firstExcludedLanguage == language) {
+			firstExcludedLanguage = language
+			firstExcludedLanguageSize += f.Size()
+		}
 		continue
 	}
 
-	// filter special languages unless they are the only language
+	// If there are no included languages add the first excluded language
-	if len(sizes) > 1 {
+	if len(sizes) == 0 && firstExcludedLanguage != "" {
-		for language := range sizes {
+		sizes[firstExcludedLanguage] = firstExcludedLanguageSize
-			langtype := enry.GetLanguageType(language)
-			if langtype != enry.Programming && langtype != enry.Markup {
-				delete(sizes, language)
-			}
-		}
 	}
 
 	return sizes, nil