Add a config option to block "expensive" pages for anonymous users (#34024) (#34071)

Backport #34024 since there are too many AI crawlers. The new code is
covered by tests and it does nothing if users don't set it.

custom/conf/app.example.ini

@@ -774,6 +774,9 @@ LEVEL = Info
 ;ALLOW_ONLY_EXTERNAL_REGISTRATION = false
 ;;
 ;; User must sign in to view anything.
+;; After 1.23.7, it could be set to "expensive" to block anonymous users accessing some pages which consume a lot of resources,
+;; for example: block anonymous AI crawlers from accessing repo code pages.
+;; The "expensive" mode is experimental and subject to change.
 ;REQUIRE_SIGNIN_VIEW = false
 ;;
 ;; Mail notification

modules/setting/config_provider.go

@@ -26,6 +26,7 @@ type ConfigKey interface {
 	In(defaultVal string, candidates []string) string
 	String() string
 	Strings(delim string) []string
+	Bool() (bool, error)
 
 	MustString(defaultVal string) string
 	MustBool(defaultVal ...bool) bool

modules/setting/service.go

@@ -43,7 +43,8 @@ var Service = struct {
 	ShowRegistrationButton                  bool
 	EnablePasswordSignInForm                bool
 	ShowMilestonesDashboardPage             bool
-	RequireSignInView                       bool
+	RequireSignInViewStrict                 bool
+	BlockAnonymousAccessExpensive           bool
 	EnableNotifyMail                        bool
 	EnableBasicAuth                         bool
 	EnablePasskeyAuth                       bool
@@ -159,7 +160,18 @@ func loadServiceFrom(rootCfg ConfigProvider) {
 	Service.EmailDomainBlockList = CompileEmailGlobList(sec, "EMAIL_DOMAIN_BLOCKLIST")
 	Service.ShowRegistrationButton = sec.Key("SHOW_REGISTRATION_BUTTON").MustBool(!(Service.DisableRegistration || Service.AllowOnlyExternalRegistration))
 	Service.ShowMilestonesDashboardPage = sec.Key("SHOW_MILESTONES_DASHBOARD_PAGE").MustBool(true)
-	Service.RequireSignInView = sec.Key("REQUIRE_SIGNIN_VIEW").MustBool()
+
+	// boolean values are considered as "strict"
+	var err error
+	Service.RequireSignInViewStrict, err = sec.Key("REQUIRE_SIGNIN_VIEW").Bool()
+	if s := sec.Key("REQUIRE_SIGNIN_VIEW").String(); err != nil && s != "" {
+		// non-boolean value only supports "expensive" at the moment
+		Service.BlockAnonymousAccessExpensive = s == "expensive"
+		if !Service.BlockAnonymousAccessExpensive {
+			log.Error("Invalid config option: REQUIRE_SIGNIN_VIEW = %s", s)
+		}
+	}
+
 	Service.EnableBasicAuth = sec.Key("ENABLE_BASIC_AUTHENTICATION").MustBool(true)
 	Service.EnablePasswordSignInForm = sec.Key("ENABLE_PASSWORD_SIGNIN_FORM").MustBool(true)
 	Service.EnablePasskeyAuth = sec.Key("ENABLE_PASSKEY_AUTHENTICATION").MustBool(true)

modules/setting/service_test.go

@@ -7,16 +7,14 @@ import (
 	"testing"
 
 	"code.gitea.io/gitea/modules/structs"
+	"code.gitea.io/gitea/modules/test"
 
 	"github.com/gobwas/glob"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestLoadServices(t *testing.T) {
-	oldService := Service
-	defer func() {
-		Service = oldService
-	}()
+	defer test.MockVariableValue(&Service)()
 
 	cfg, err := NewConfigProviderFromData(`
 [service]
@@ -48,10 +46,7 @@ EMAIL_DOMAIN_BLOCKLIST = d3, *.b
 }
 
 func TestLoadServiceVisibilityModes(t *testing.T) {
-	oldService := Service
-	defer func() {
-		Service = oldService
-	}()
+	defer test.MockVariableValue(&Service)()
 
 	kases := map[string]func(){
 		`
@@ -130,3 +125,33 @@ ALLOWED_USER_VISIBILITY_MODES = public, limit, privated
 		})
 	}
 }
+
+func TestLoadServiceRequireSignInView(t *testing.T) {
+	defer test.MockVariableValue(&Service)()
+
+	cfg, err := NewConfigProviderFromData(`
+[service]
+`)
+	assert.NoError(t, err)
+	loadServiceFrom(cfg)
+	assert.False(t, Service.RequireSignInViewStrict)
+	assert.False(t, Service.BlockAnonymousAccessExpensive)
+
+	cfg, err = NewConfigProviderFromData(`
+[service]
+REQUIRE_SIGNIN_VIEW = true
+`)
+	assert.NoError(t, err)
+	loadServiceFrom(cfg)
+	assert.True(t, Service.RequireSignInViewStrict)
+	assert.False(t, Service.BlockAnonymousAccessExpensive)
+
+	cfg, err = NewConfigProviderFromData(`
+[service]
+REQUIRE_SIGNIN_VIEW = expensive
+`)
+	assert.NoError(t, err)
+	loadServiceFrom(cfg)
+	assert.False(t, Service.RequireSignInViewStrict)
+	assert.True(t, Service.BlockAnonymousAccessExpensive)
+}

routers/api/packages/cargo/cargo.go

@@ -51,7 +51,7 @@ func apiError(ctx *context.Context, status int, obj any) {
 
 // https://rust-lang.github.io/rfcs/2789-sparse-index.html
 func RepositoryConfig(ctx *context.Context) {
-	ctx.JSON(http.StatusOK, cargo_service.BuildConfig(ctx.Package.Owner, setting.Service.RequireSignInView || ctx.Package.Owner.Visibility != structs.VisibleTypePublic))
+	ctx.JSON(http.StatusOK, cargo_service.BuildConfig(ctx.Package.Owner, setting.Service.RequireSignInViewStrict || ctx.Package.Owner.Visibility != structs.VisibleTypePublic))
 }
 
 func EnumeratePackageVersions(ctx *context.Context) {

routers/api/packages/container/container.go

@@ -126,7 +126,7 @@ func apiUnauthorizedError(ctx *context.Context) {
 
 // ReqContainerAccess is a middleware which checks the current user valid (real user or ghost if anonymous access is enabled)
 func ReqContainerAccess(ctx *context.Context) {
-	if ctx.Doer == nil || (setting.Service.RequireSignInView && ctx.Doer.IsGhost()) {
+	if ctx.Doer == nil || (setting.Service.RequireSignInViewStrict && ctx.Doer.IsGhost()) {
 		apiUnauthorizedError(ctx)
 	}
 }
@@ -152,7 +152,7 @@ func Authenticate(ctx *context.Context) {
 	u := ctx.Doer
 	packageScope := auth_service.GetAccessScope(ctx.Data)
 	if u == nil {
-		if setting.Service.RequireSignInView {
+		if setting.Service.RequireSignInViewStrict {
 			apiUnauthorizedError(ctx)
 			return
 		}

routers/api/v1/api.go

@@ -356,7 +356,7 @@ func reqToken() func(ctx *context.APIContext) {
 
 func reqExploreSignIn() func(ctx *context.APIContext) {
 	return func(ctx *context.APIContext) {
-		if (setting.Service.RequireSignInView || setting.Service.Explore.RequireSigninView) && !ctx.IsSigned {
+		if (setting.Service.RequireSignInViewStrict || setting.Service.Explore.RequireSigninView) && !ctx.IsSigned {
 			ctx.Error(http.StatusUnauthorized, "reqExploreSignIn", "you must be signed in to search for users")
 		}
 	}
@@ -874,7 +874,7 @@ func Routes() *web.Router {
 	m.Use(apiAuth(buildAuthGroup()))
 
 	m.Use(verifyAuthWithOptions(&common.VerifyOptions{
-		SignInRequired: setting.Service.RequireSignInView,
+		SignInRequired: setting.Service.RequireSignInViewStrict,
 	}))
 
 	addActionsRoutes := func(

routers/common/blockexpensive.go

@@ -0,0 +1,91 @@
+// Copyright 2025 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package common
+
+import (
+	"context"
+	"net/http"
+	"strings"
+
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/web/middleware"
+
+	"github.com/go-chi/chi/v5"
+)
+
+func BlockExpensive() func(next http.Handler) http.Handler {
+	if !setting.Service.BlockAnonymousAccessExpensive {
+		return nil
+	}
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+			ret := determineRequestPriority(req.Context())
+			if !ret.SignedIn {
+				if ret.Expensive || ret.LongPolling {
+					http.Redirect(w, req, setting.AppSubURL+"/user/login", http.StatusSeeOther)
+					return
+				}
+			}
+			next.ServeHTTP(w, req)
+		})
+	}
+}
+
+func isRoutePathExpensive(routePattern string) bool {
+	if strings.HasPrefix(routePattern, "/user/") || strings.HasPrefix(routePattern, "/login/") {
+		return false
+	}
+
+	expensivePaths := []string{
+		// code related
+		"/{username}/{reponame}/archive/",
+		"/{username}/{reponame}/blame/",
+		"/{username}/{reponame}/commit/",
+		"/{username}/{reponame}/commits/",
+		"/{username}/{reponame}/graph",
+		"/{username}/{reponame}/media/",
+		"/{username}/{reponame}/raw/",
+		"/{username}/{reponame}/src/",
+
+		// issue & PR related (no trailing slash)
+		"/{username}/{reponame}/issues",
+		"/{username}/{reponame}/{type:issues}",
+		"/{username}/{reponame}/pulls",
+		"/{username}/{reponame}/{type:pulls}",
+
+		// wiki
+		"/{username}/{reponame}/wiki/",
+
+		// activity
+		"/{username}/{reponame}/activity/",
+	}
+	for _, path := range expensivePaths {
+		if strings.HasPrefix(routePattern, path) {
+			return true
+		}
+	}
+	return false
+}
+
+func isRoutePathForLongPolling(routePattern string) bool {
+	return routePattern == "/user/events"
+}
+
+func determineRequestPriority(ctx context.Context) (ret struct {
+	SignedIn    bool
+	Expensive   bool
+	LongPolling bool
+},
+) {
+	dataStore := middleware.GetContextData(ctx)
+	chiRoutePath := chi.RouteContext(ctx).RoutePattern()
+	if _, ok := dataStore[middleware.ContextDataKeySignedUser].(*user_model.User); ok {
+		ret.SignedIn = true
+	} else {
+		ret.Expensive = isRoutePathExpensive(chiRoutePath)
+		ret.LongPolling = isRoutePathForLongPolling(chiRoutePath)
+	}
+	return ret
+}

routers/common/blockexpensive_test.go

@@ -0,0 +1,30 @@
+// Copyright 2025 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package common
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBlockExpensive(t *testing.T) {
+	cases := []struct {
+		expensive bool
+		routePath string
+	}{
+		{false, "/user/xxx"},
+		{false, "/login/xxx"},
+		{true, "/{username}/{reponame}/archive/xxx"},
+		{true, "/{username}/{reponame}/graph"},
+		{true, "/{username}/{reponame}/src/xxx"},
+		{true, "/{username}/{reponame}/wiki/xxx"},
+		{true, "/{username}/{reponame}/activity/xxx"},
+	}
+	for _, c := range cases {
+		assert.Equal(t, c.expensive, isRoutePathExpensive(c.routePath), "routePath: %s", c.routePath)
+	}
+
+	assert.True(t, isRoutePathForLongPolling("/user/events"))
+}

routers/install/install.go

@@ -156,7 +156,7 @@ func Install(ctx *context.Context) {
 	form.DisableRegistration = setting.Service.DisableRegistration
 	form.AllowOnlyExternalRegistration = setting.Service.AllowOnlyExternalRegistration
 	form.EnableCaptcha = setting.Service.EnableCaptcha
-	form.RequireSignInView = setting.Service.RequireSignInView
+	form.RequireSignInView = setting.Service.RequireSignInViewStrict
 	form.DefaultKeepEmailPrivate = setting.Service.DefaultKeepEmailPrivate
 	form.DefaultAllowCreateOrganization = setting.Service.DefaultAllowCreateOrganization
 	form.DefaultEnableTimetracking = setting.Service.DefaultEnableTimetracking

routers/private/serv.go

@@ -286,7 +286,7 @@ func ServCommand(ctx *context.PrivateContext) {
 			repo.IsPrivate ||
 			owner.Visibility.IsPrivate() ||
 			(user != nil && user.IsRestricted) || // user will be nil if the key is a deploykey
-			setting.Service.RequireSignInView) {
+			setting.Service.RequireSignInViewStrict) {
 		if key.Type == asymkey_model.KeyTypeDeploy {
 			if deployKey.Mode < mode {
 				ctx.JSON(http.StatusUnauthorized, private.Response{

routers/web/repo/githttp.go

@@ -127,7 +127,7 @@ func httpBase(ctx *context.Context) *serviceHandler {
 	// Only public pull don't need auth.
 	isPublicPull := repoExist && !repo.IsPrivate && isPull
 	var (
-		askAuth = !isPublicPull || setting.Service.RequireSignInView
+		askAuth = !isPublicPull || setting.Service.RequireSignInViewStrict
 		environ []string
 	)
 

routers/web/web.go

@@ -285,23 +285,23 @@ func Routes() *web.Router {
 	mid = append(mid, repo.GetActiveStopwatch)
 	mid = append(mid, goGet)
 
-	others := web.NewRouter()
-	others.Use(mid...)
-	registerRoutes(others)
-	routes.Mount("", others)
+	webRoutes := web.NewRouter()
+	webRoutes.Use(mid...)
+	webRoutes.Group("", func() { registerWebRoutes(webRoutes) }, common.BlockExpensive())
+	routes.Mount("", webRoutes)
 	return routes
 }
 
 var optSignInIgnoreCsrf = verifyAuthWithOptions(&common.VerifyOptions{DisableCSRF: true})
 
-// registerRoutes register routes
-func registerRoutes(m *web.Router) {
+// registerWebRoutes register routes
+func registerWebRoutes(m *web.Router) {
 	// required to be signed in or signed out
 	reqSignIn := verifyAuthWithOptions(&common.VerifyOptions{SignInRequired: true})
 	reqSignOut := verifyAuthWithOptions(&common.VerifyOptions{SignOutRequired: true})
 	// optional sign in (if signed in, use the user as doer, if not, no doer)
-	optSignIn := verifyAuthWithOptions(&common.VerifyOptions{SignInRequired: setting.Service.RequireSignInView})
-	optExploreSignIn := verifyAuthWithOptions(&common.VerifyOptions{SignInRequired: setting.Service.RequireSignInView || setting.Service.Explore.RequireSigninView})
+	optSignIn := verifyAuthWithOptions(&common.VerifyOptions{SignInRequired: setting.Service.RequireSignInViewStrict})
+	optExploreSignIn := verifyAuthWithOptions(&common.VerifyOptions{SignInRequired: setting.Service.RequireSignInViewStrict || setting.Service.Explore.RequireSigninView})
 
 	validation.AddBindingRules()
 

services/context/package.go

@@ -93,7 +93,7 @@ func packageAssignment(ctx *packageAssignmentCtx, errCb func(int, string, any))
 }
 
 func determineAccessMode(ctx *Base, pkg *Package, doer *user_model.User) (perm.AccessMode, error) {
-	if setting.Service.RequireSignInView && (doer == nil || doer.IsGhost()) {
+	if setting.Service.RequireSignInViewStrict && (doer == nil || doer.IsGhost()) {
 		return perm.AccessModeNone, nil
 	}
 

services/packages/cargo/index.go

@@ -248,7 +248,7 @@ func createOrUpdateConfigFile(ctx context.Context, repo *repo_model.Repository,
 		"Initialize Cargo Config",
 		func(t *files_service.TemporaryUploadRepository) error {
 			var b bytes.Buffer
-			err := json.NewEncoder(&b).Encode(BuildConfig(owner, setting.Service.RequireSignInView || owner.Visibility != structs.VisibleTypePublic || repo.IsPrivate))
+			err := json.NewEncoder(&b).Encode(BuildConfig(owner, setting.Service.RequireSignInViewStrict || owner.Visibility != structs.VisibleTypePublic || repo.IsPrivate))
 			if err != nil {
 				return err
 			}

templates/admin/config.tmpl

@@ -148,7 +148,7 @@
 				<dt>{{ctx.Locale.Tr "admin.config.enable_openid_signin"}}</dt>
 				<dd>{{svg (Iif .Service.EnableOpenIDSignIn "octicon-check" "octicon-x")}}</dd>
 				<dt>{{ctx.Locale.Tr "admin.config.require_sign_in_view"}}</dt>
-				<dd>{{svg (Iif .Service.RequireSignInView "octicon-check" "octicon-x")}}</dd>
+				<dd>{{svg (Iif .Service.RequireSignInViewStrict "octicon-check" "octicon-x")}}</dd>
 				<dt>{{ctx.Locale.Tr "admin.config.mail_notify"}}</dt>
 				<dd>{{svg (Iif .Service.EnableNotifyMail "octicon-check" "octicon-x")}}</dd>
 				<dt>{{ctx.Locale.Tr "admin.config.enable_captcha"}}</dt>

tests/integration/api_org_test.go

@@ -148,9 +148,9 @@ func TestAPIOrgEditBadVisibility(t *testing.T) {
 
 func TestAPIOrgDeny(t *testing.T) {
 	onGiteaRun(t, func(*testing.T, *url.URL) {
-		setting.Service.RequireSignInView = true
+		setting.Service.RequireSignInViewStrict = true
 		defer func() {
-			setting.Service.RequireSignInView = false
+			setting.Service.RequireSignInViewStrict = false
 		}()
 
 		orgName := "user1_org"

tests/integration/api_packages_container_test.go

@@ -111,7 +111,7 @@ func TestPackageContainer(t *testing.T) {
 				AddTokenAuth(anonymousToken)
 			MakeRequest(t, req, http.StatusOK)
 
-			defer test.MockVariableValue(&setting.Service.RequireSignInView, true)()
+			defer test.MockVariableValue(&setting.Service.RequireSignInViewStrict, true)()
 
 			req = NewRequest(t, "GET", fmt.Sprintf("%sv2", setting.AppURL))
 			MakeRequest(t, req, http.StatusUnauthorized)

tests/integration/api_packages_generic_test.go

@@ -15,6 +15,7 @@ import (
 	"code.gitea.io/gitea/models/unittest"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/test"
 	"code.gitea.io/gitea/tests"
 
 	"github.com/stretchr/testify/assert"
@@ -131,11 +132,7 @@ func TestPackageGeneric(t *testing.T) {
 
 		t.Run("RequireSignInView", func(t *testing.T) {
 			defer tests.PrintCurrentTest(t)()
-
-			setting.Service.RequireSignInView = true
-			defer func() {
-				setting.Service.RequireSignInView = false
-			}()
+			defer test.MockVariableValue(&setting.Service.RequireSignInViewStrict, true)()
 
 			req = NewRequest(t, "GET", url+"/dummy.bin")
 			MakeRequest(t, req, http.StatusUnauthorized)

tests/integration/git_smart_http_test.go

@@ -74,7 +74,7 @@ func testGitSmartHTTP(t *testing.T, u *url.URL) {
 }
 
 func testRenamedRepoRedirect(t *testing.T) {
-	defer test.MockVariableValue(&setting.Service.RequireSignInView, true)()
+	defer test.MockVariableValue(&setting.Service.RequireSignInViewStrict, true)()
 
 	// git client requires to get a 301 redirect response before 401 unauthorized response
 	req := NewRequest(t, "GET", "/user2/oldrepo1/info/refs")

tests/integration/signin_test.go

@@ -15,11 +15,13 @@ import (
 	"code.gitea.io/gitea/modules/test"
 	"code.gitea.io/gitea/modules/translation"
 	"code.gitea.io/gitea/modules/web"
+	"code.gitea.io/gitea/routers"
 	"code.gitea.io/gitea/services/context"
 	"code.gitea.io/gitea/tests"
 
 	"github.com/markbates/goth"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func testLoginFailed(t *testing.T, username, password, message string) {
@@ -158,3 +160,32 @@ func TestEnablePasswordSignInFormAndEnablePasskeyAuth(t *testing.T) {
 		NewHTMLParser(t, resp.Body).AssertElement(t, ".signin-passkey", true)
 	})
 }
+
+func TestRequireSignInView(t *testing.T) {
+	defer tests.PrepareTestEnv(t)()
+	t.Run("NoRequireSignInView", func(t *testing.T) {
+		require.False(t, setting.Service.RequireSignInViewStrict)
+		require.False(t, setting.Service.BlockAnonymousAccessExpensive)
+		req := NewRequest(t, "GET", "/user2/repo1/src/branch/master")
+		MakeRequest(t, req, http.StatusOK)
+	})
+	t.Run("RequireSignInView", func(t *testing.T) {
+		defer test.MockVariableValue(&setting.Service.RequireSignInViewStrict, true)()
+		defer test.MockVariableValue(&testWebRoutes, routers.NormalRoutes())()
+		req := NewRequest(t, "GET", "/user2/repo1/src/branch/master")
+		resp := MakeRequest(t, req, http.StatusSeeOther)
+		assert.Equal(t, "/user/login", resp.Header().Get("Location"))
+	})
+	t.Run("BlockAnonymousAccessExpensive", func(t *testing.T) {
+		defer test.MockVariableValue(&setting.Service.RequireSignInViewStrict, false)()
+		defer test.MockVariableValue(&setting.Service.BlockAnonymousAccessExpensive, true)()
+		defer test.MockVariableValue(&testWebRoutes, routers.NormalRoutes())()
+
+		req := NewRequest(t, "GET", "/user2/repo1")
+		MakeRequest(t, req, http.StatusOK)
+
+		req = NewRequest(t, "GET", "/user2/repo1/src/branch/master")
+		resp := MakeRequest(t, req, http.StatusSeeOther)
+		assert.Equal(t, "/user/login", resp.Header().Get("Location"))
+	})
+}