fix: always include files set to be detectable for language stats (#11685)

- The documentation has the correct behavior about `linguist-detectable`: In cases where a file should be considered for language statistics, regardless of its category, the linguist-detectable attribute can be used.
- This patch follows that behavior by not skipping the file even if some heuristic would've said to skip the file.
- Document the conditions in more natural language.
- Resolves forgejo/forgejo#11248

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/11685
Reviewed-by: Andreas Ahlenstorf <aahlenst@noreply.codeberg.org>
Co-authored-by: Gusted <postmaster@gusted.xyz>
Co-committed-by: Gusted <postmaster@gusted.xyz>

modules/git/fetch_test.go

@@ -26,7 +26,7 @@ func TestFetch(t *testing.T) {
 
 			fetchedCommitID, err := repo.Fetch(otherRepoPath, "refs/heads/master")
 			require.NoError(t, err)
-			assert.Equal(t, "95d3505f2db273e40be79f84416051ae85e9ea0d", fetchedCommitID)
+			assert.Equal(t, "5684d0c8cfdfb17fcd59101826efc9ff54b80df4", fetchedCommitID)
 
 			c, err := repo.getCommit(MustIDFromString(fetchedCommitID))
 			require.NoError(t, err)

modules/git/grep_test.go

@@ -65,7 +65,7 @@ func TestGrepSearch(t *testing.T) {
 			return
 		}
 
-		res, err = GrepSearch(t.Context(), repo, "world", GrepOptions{MatchesPerFile: 1})
+		res, err = GrepSearch(t.Context(), repo, "world", GrepOptions{RefName: "95d3505f2db273e40be79f84416051ae85e9ea0d", MatchesPerFile: 1})
 		require.NoError(t, err)
 		assert.Equal(t, []*GrepResult{
 			{

modules/git/repo_language_stats.go

@@ -168,11 +168,21 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 			}
 		}
 
-		if isFalse(isDetectable) || isTrue(isVendored) || isTrue(isDocumentation) ||
-			(!isFalse(isVendored) && analyze.IsVendor(f.Name())) ||
-			enry.IsDotFile(f.Name()) ||
-			enry.IsConfiguration(f.Name()) ||
-			(!isFalse(isDocumentation) && enry.IsDocumentation(f.Name())) {
+		// Don't skip this file if it is explicitly set to be detectable.
+		// Skip this file if one of the following conditions holds:
+		// 1. Explicitly set to not be detectable.
+		// 2. Explicitly set that it is vendored.
+		// 3. Explicitly set that it is documentation.
+		// 4. Is not explicitly set to not be vendored and is by heuristic considered to be vendored.
+		// 5. It is considered to be a dot file.
+		// 6. It is considered to be a configuration file.
+		// 7. Is not explicitly set to not be documentation and is by heuristic considered to be documentation.
+		if !isTrue(isDetectable) &&
+			(isFalse(isDetectable) || isTrue(isVendored) || isTrue(isDocumentation) ||
+				(!isFalse(isVendored) && analyze.IsVendor(f.Name())) ||
+				enry.IsDotFile(f.Name()) ||
+				enry.IsConfiguration(f.Name()) ||
+				(!isFalse(isDocumentation) && enry.IsDocumentation(f.Name()))) {
 			continue
 		}
 

modules/git/repo_language_stats_test.go

@@ -34,6 +34,16 @@ func TestRepository_GetLanguageStats(t *testing.T) {
 		"Python": 67,
 		"Java":   112,
 	}, stats)
+
+	stats, err = gitRepo.GetLanguageStats("5684d0c8cfdfb17fcd59101826efc9ff54b80df4")
+	require.NoError(t, err)
+
+	assert.Equal(t, map[string]int64{
+		"Cobra":    67,
+		"Python":   67,
+		"Markdown": 15,
+		"Java":     112,
+	}, stats)
 }
 
 func TestMergeLanguageStats(t *testing.T) {

modules/git/tests/repos/language_stats_repo/packed-refs

@@ -1,2 +1,2 @@
 # pack-refs with: peeled fully-peeled sorted 
-95d3505f2db273e40be79f84416051ae85e9ea0d refs/heads/master
+5684d0c8cfdfb17fcd59101826efc9ff54b80df4 refs/heads/master