fix(windvane): stop paginating after the last page

pkg/passive/sources_wo_auth_test.go

@@ -24,20 +24,21 @@ func TestSourcesWithoutKeys(t *testing.T) {
 	}
 
 	ignoredSources := []string{
-		"commoncrawl",    // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ
-		"riddler",        // failing due to cloudfront protection
-		"crtsh",          // Fails in GH Action (possibly IP-based ban) causing a timeout.
-		"hackertarget",   // Fails in GH Action (possibly IP-based ban) but works locally
-		"waybackarchive", // Fails randomly
-		"alienvault",     // 503 Service Temporarily Unavailable
-		"digitorus",      // failing with "Failed to retrieve certificate"
-		"dnsdumpster",    // failing with "unexpected status code 403 received"
-		"anubis",         // previously failed on jonlu.ca (Cloudflare 403 / redirects); endpoint moved to anubisdb.com
-		"threatcrowd",    // failing with "randomly failing with unmarshal error when hit multiple times"
-		"leakix",         // now requires API key (returns 401)
-		"reconeer",       // now requires API key (returns 401)
-		"sitedossier",    // flaky - returns no results in CI
-		"submd",          // origin is unavailable (Cloudflare 522/523)
+		"commoncrawl",     // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ
+		"riddler",         // failing due to cloudfront protection
+		"crtsh",           // Fails in GH Action (possibly IP-based ban) causing a timeout.
+		"hackertarget",    // Fails in GH Action (possibly IP-based ban) but works locally
+		"waybackarchive",  // Fails randomly
+		"alienvault",      // 503 Service Temporarily Unavailable
+		"digitorus",       // failing with "Failed to retrieve certificate"
+		"dnsdumpster",     // failing with "unexpected status code 403 received"
+		"anubis",          // previously failed on jonlu.ca (Cloudflare 403 / redirects); endpoint moved to anubisdb.com
+		"threatcrowd",     // failing with "randomly failing with unmarshal error when hit multiple times"
+		"leakix",          // now requires API key (returns 401)
+		"reconeer",        // now requires API key (returns 401)
+		"sitedossier",     // flaky - returns no results in CI
+		"submd",           // origin is unavailable (Cloudflare 522/523)
+		"subdomaincenter", // anonymous API returns 503 "server busy" from shared CI addresses
 	}
 
 	domain := "hackerone.com"

pkg/subscraping/sources/windvane/windvane.go

@@ -88,8 +88,6 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 				return
 			}
 
-			defer session.DiscardHTTPResponse(resp)
-
 			var windvaneResponse response
 			err = json.NewDecoder(resp.Body).Decode(&windvaneResponse)
 			if err != nil {
@@ -102,12 +100,17 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 			for _, record := range windvaneResponse.Data.List {
 				select {
 				case <-ctx.Done():
+					session.DiscardHTTPResponse(resp)
 					return
 				case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Domain}:
 					stats.Results++
 				}
 			}
 
+			// Close this page before the next request. A defer in the loop
+			// would hold every body until the source returns.
+			session.DiscardHTTPResponse(resp)
+
 			pageInfo := windvaneResponse.Data.PageResponse
 			var totalRecords, recordsPerPage int
 
@@ -118,7 +121,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 				break
 			}
 
-			if (page-1)*recordsPerPage >= totalRecords {
+			if page*recordsPerPage >= totalRecords {
 				break
 			}
 

pkg/subscraping/sources/windvane/windvane_test.go

@@ -0,0 +1,137 @@
+package windvane
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"math"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/projectdiscovery/ratelimit"
+	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// rewriteTransport sends every request to the test server's host while
+// preserving the original path, so the source's hardcoded windvane URL
+// still resolves to our handler.
+type rewriteTransport struct {
+	target *url.URL
+}
+
+func (r *rewriteTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	req.URL.Scheme = r.target.Scheme
+	req.URL.Host = r.target.Host
+	return http.DefaultTransport.RoundTrip(req)
+}
+
+// newPagedServer serves total subdomains split in pages of the requested
+// count, with the page_response fields of the windvane API documentation,
+// and records every requested page.
+func newPagedServer(t *testing.T, total int) (*httptest.Server, func() []int) {
+	t.Helper()
+	var mu sync.Mutex
+	var pages []int
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var req struct {
+			PageRequest struct {
+				Page  int `json:"page"`
+				Count int `json:"count"`
+			} `json:"page_request"`
+		}
+		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		page, count := req.PageRequest.Page, req.PageRequest.Count
+		mu.Lock()
+		pages = append(pages, page)
+		mu.Unlock()
+
+		var resp response
+		for i := (page - 1) * count; i < min(page*count, total); i++ {
+			resp.Data.List = append(resp.Data.List, domainEntry{Domain: fmt.Sprintf("sub%d.example.com", i)})
+		}
+		resp.Data.PageResponse = pageInfo{
+			Total:     strconv.Itoa(total),
+			Count:     strconv.Itoa(count),
+			TotalPage: strconv.Itoa((total + count - 1) / count),
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(resp)
+	}))
+	t.Cleanup(server.Close)
+
+	return server, func() []int {
+		mu.Lock()
+		defer mu.Unlock()
+		return append([]int(nil), pages...)
+	}
+}
+
+func runSource(t *testing.T, server *httptest.Server) (subs []string, errs []error) {
+	t.Helper()
+	target, err := url.Parse(server.URL)
+	require.NoError(t, err)
+
+	mrl, err := ratelimit.NewMultiLimiter(context.Background(), &ratelimit.Options{
+		Key:         "windvane",
+		IsUnlimited: false,
+		MaxCount:    math.MaxInt32,
+		Duration:    time.Millisecond,
+	})
+	require.NoError(t, err)
+	session := &subscraping.Session{
+		Client:           &http.Client{Transport: &rewriteTransport{target: target}, Timeout: 5 * time.Second},
+		MultiRateLimiter: mrl,
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	ctxWithValue := context.WithValue(ctx, subscraping.CtxSourceArg, "windvane")
+
+	source := &Source{}
+	source.AddApiKeys([]string{"test-key"})
+	for r := range source.Run(ctxWithValue, "example.com", session) {
+		switch r.Type {
+		case subscraping.Subdomain:
+			subs = append(subs, r.Value)
+		case subscraping.Error:
+			errs = append(errs, r.Error)
+		}
+	}
+	return subs, errs
+}
+
+// Pagination must stop after the last page instead of requesting one more
+// empty page.
+func TestWindvaneSource_StopsAfterLastPage(t *testing.T) {
+	tests := []struct {
+		name  string
+		total int
+		pages []int
+	}{
+		{name: "single page", total: 2, pages: []int{1}},
+		{name: "full single page", total: 1000, pages: []int{1}},
+		{name: "partial last page", total: 1500, pages: []int{1, 2}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			server, requestedPages := newPagedServer(t, tt.total)
+
+			subs, errs := runSource(t, server)
+
+			assert.Empty(t, errs)
+			assert.Len(t, subs, tt.total)
+			assert.Equal(t, tt.pages, requestedPages())
+		})
+	}
+}