Skip to content

Commit 02e5dd4

Browse files
rogpepperogpeppe
authored
goproxytest: add overlay capability (#304)
It's useful to have a hybrid mode where some modules are served from the txtar contents but others come from the regular Go sources. We make goproxytest capable of doing that: adding a file named `OVERLAY` to the top level `.gomodproxy` directory triggers that behavior. Also standardize the log messages produced by goproxytest to make it clear in all cases where the logs are coming from, and remove the "no archive" log because that's more common in overlay mode and doesn't provide much useful information.
1 parent 3030644 commit 02e5dd4

6 files changed

Lines changed: 187 additions & 30 deletions

File tree

‎goproxytest/proxy.go‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ func newServer(dir, addr string, logf func(string, ...any)) (*Server, error) {
9595
srv.URL = "http://" + addr + "/mod"
9696
go func() {
9797
if err := srv.server.Serve(l); err != nil && err != http.ErrServerClosed {
98-
srv.logf("go proxy: http.Serve: %v", err)
98+
srv.logf("go proxy_test http.Serve: %v", err)
9999
}
100100
}()
101101
return &srv, nil
@@ -258,7 +258,6 @@ func (srv *Server) handler(w http.ResponseWriter, r *http.Request) {
258258
// to resolve github.com, github.com/hello and github.com/hello/world.
259259
// cmd/go expects a 404/410 response if there is nothing there. Hence we
260260
// cannot return with a 500.
261-
srv.logf("go proxy: no archive %s %s\n", path, vers)
262261
http.NotFound(w, r)
263262
return
264263
}
@@ -300,7 +299,7 @@ func (srv *Server) handler(w http.ResponseWriter, r *http.Request) {
300299
}).(cached)
301300

302301
if c.err != nil {
303-
srv.logf("go proxy: %v\n", c.err)
302+
srv.logf("go proxy_test %v\n", c.err)
304303
http.Error(w, c.err.Error(), 500)
305304
return
306305
}
@@ -331,12 +330,12 @@ func (srv *Server) findHash(m module.Version) string {
331330
func (srv *Server) readArchive(path, vers string) *txtar.Archive {
332331
enc, err := module.EscapePath(path)
333332
if err != nil {
334-
srv.logf("go proxy: %v\n", err)
333+
srv.logf("go proxy_test %v\n", err)
335334
return nil
336335
}
337336
encVers, err := module.EscapeVersion(vers)
338337
if err != nil {
339-
srv.logf("go proxy: %v\n", err)
338+
srv.logf("go proxy_test %v\n", err)
340339
return nil
341340
}
342341

@@ -378,7 +377,7 @@ func (srv *Server) readArchive(path, vers string) *txtar.Archive {
378377
}
379378
if err != nil {
380379
if !os.IsNotExist(err) {
381-
srv.logf("go proxy: %v\n", err)
380+
srv.logf("go proxy_test %v\n", err)
382381
}
383382
a = nil
384383
}

‎goproxytest/proxy_test.go‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ func TestSetup(t *testing.T) {
3838
t.Run("with_proxy", func(t *testing.T) {
3939
p := testscript.Params{
4040
Files: []string{
41-
filepath.Join("testdata", "setup", "with_proxy.txt"),
41+
"testdata/setup/with_proxy.txt",
4242
},
4343
}
4444
if err := gotooltest.Setup(&p); err != nil {
@@ -50,7 +50,7 @@ func TestSetup(t *testing.T) {
5050
t.Run("no_proxy", func(t *testing.T) {
5151
p := testscript.Params{
5252
Files: []string{
53-
filepath.Join("testdata", "setup", "no_proxy.txt"),
53+
"testdata/setup/no_proxy.txt",
5454
},
5555
Setup: func(e *testscript.Env) error {
5656
e.Vars = append(e.Vars, "GOPROXY=original")
@@ -66,7 +66,7 @@ func TestSetup(t *testing.T) {
6666
t.Run("chained", func(t *testing.T) {
6767
p := testscript.Params{
6868
Files: []string{
69-
filepath.Join("testdata", "setup", "chained.txt"),
69+
"testdata/setup/chained.txt",
7070
},
7171
Setup: func(e *testscript.Env) error {
7272
e.Vars = append(e.Vars, "CUSTOM=hello")
@@ -79,6 +79,18 @@ func TestSetup(t *testing.T) {
7979
goproxytest.Setup(&p)
8080
testscript.Run(t, p)
8181
})
82+
t.Run("overlay", func(t *testing.T) {
83+
p := testscript.Params{
84+
Files: []string{
85+
"testdata/setup/overlay.txt",
86+
},
87+
}
88+
if err := gotooltest.Setup(&p); err != nil {
89+
t.Fatal(err)
90+
}
91+
goproxytest.Setup(&p)
92+
testscript.Run(t, p)
93+
})
8294
}
8395

8496
func TestLatest(t *testing.T) {

‎goproxytest/setup.go‎

Lines changed: 85 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,13 @@ package goproxytest
66

77
import (
88
"fmt"
9+
"net/url"
910
"os"
1011
"path/filepath"
12+
"strings"
13+
"unicode"
1114

15+
"github.com/rogpeppe/go-internal/internal/goenv"
1216
"github.com/rogpeppe/go-internal/testscript"
1317
)
1418

@@ -24,6 +28,13 @@ const GoModProxyDir = ".gomodproxy"
2428
// It wraps p.Setup to start a proxy server when the directory is present,
2529
// set GOPROXY and GONOSUMDB appropriately, and shuts down
2630
// the server when the test completes.
31+
//
32+
// If a file exists in the top level of the [GoModProxyDir] directory
33+
// named "_enable_overlay", then instead of replacing the Go proxy entirely, the
34+
// existing Go proxy is used to serve modules that are not present in
35+
// the test script. In this case it uses the existing GOMODPROXY
36+
// download cache to avoid network downloads if it can, but takes care
37+
// not to pollute it with the localhost proxy server content.
2738
func Setup(p *testscript.Params) {
2839
origSetup := p.Setup
2940
p.Setup = func(env *testscript.Env) error {
@@ -33,19 +44,84 @@ func Setup(p *testscript.Params) {
3344
}
3445
}
3546
proxyDir := filepath.Join(env.WorkDir, GoModProxyDir)
36-
if info, err := os.Stat(proxyDir); err == nil && info.IsDir() {
37-
srv, err := NewServer(proxyDir, "")
38-
if err != nil {
39-
return fmt.Errorf("cannot start Go proxy: %v", err)
47+
info, err := os.Stat(proxyDir)
48+
if err != nil || !info.IsDir() {
49+
return nil
50+
}
51+
srv, err := NewServer(proxyDir, "")
52+
if err != nil {
53+
return fmt.Errorf("cannot start Go proxy: %v", err)
54+
}
55+
env.Defer(srv.Close)
56+
57+
env.Vars = append(env.Vars, "GONOSUMDB=*")
58+
if info, err := os.Stat(filepath.Join(proxyDir, "_enable_overlay")); err == nil && info.Mode().IsRegular() {
59+
// Overlay is enabled: overlay rather than replacing the proxy.
60+
// NOTE: if you change the list of names here, make sure
61+
// to keep ../internal/goenv/goenv.go:/^var goEnv
62+
// up to date.
63+
var goEnv struct {
64+
GOMODCACHE string
65+
GOPROXY string
66+
GONOPROXY string
4067
}
41-
env.Defer(srv.Close)
42-
// Add GOPROXY after calling the original setup
43-
// so that it overrides any GOPROXY set there.
68+
if err := goenv.Unmarshal(&goEnv); err != nil {
69+
return err
70+
}
71+
if goEnv.GOMODCACHE == "" || goEnv.GOPROXY == "" {
72+
// Shouldn't happen.
73+
return fmt.Errorf("missing GOMODCACHE or GOPROXY from go env output")
74+
}
75+
// Look for local test modules first, falling back to the
76+
// existing module download cache. See example under the
77+
// GOPROXY entry in https://go.dev/ref/mod#environment-variables
4478
env.Vars = append(env.Vars,
45-
"GOPROXY="+srv.URL,
46-
"GONOSUMDB=*",
79+
fmt.Sprintf("GOPROXY=%s,%s,%s",
80+
srv.URL,
81+
uriFromPath(filepath.Join(goEnv.GOMODCACHE, "/cache/download")),
82+
goEnv.GOPROXY,
83+
),
84+
// Pass through the following variables on the grounds that
85+
// they should be used when fetching external modules.
86+
// Hopefully they will be immaterial because tests shouldn't
87+
// be relying on modules that match them, but it seems safer
88+
// to do this.
89+
"GONOPROXY="+goEnv.GONOPROXY,
4790
)
91+
} else {
92+
env.Vars = append(env.Vars, "GOPROXY="+srv.URL)
4893
}
4994
return nil
5095
}
5196
}
97+
98+
// uriFromPath returns a file:// URI for the supplied file path.
99+
// This code was adapted from cuelang.org/go/internal/golangorgx/gopls/protocol.URIFromPath
100+
func uriFromPath(path string) string {
101+
if path == "" {
102+
return ""
103+
}
104+
if !isWindowsDrivePath(path) {
105+
if abs, err := filepath.Abs(path); err == nil {
106+
path = abs
107+
}
108+
}
109+
// Check the file path again, in case it became absolute.
110+
if isWindowsDrivePath(path) {
111+
path = "/" + strings.ToUpper(string(path[0])) + path[1:]
112+
}
113+
return (&url.URL{
114+
Scheme: "file",
115+
Path: filepath.ToSlash(path),
116+
}).String()
117+
}
118+
119+
// isWindowsDrivePath returns true if the file path is of the form used by
120+
// Windows. We check if the path begins with a drive letter, followed by a ":".
121+
// For example: C:/x/y/z.
122+
func isWindowsDrivePath(path string) bool {
123+
if len(path) < 3 {
124+
return false
125+
}
126+
return unicode.IsLetter(rune(path[0])) && path[1] == ':'
127+
}

‎goproxytest/testdata/setup/overlay.txt‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Verify that goproxytest.Setup sets GOPROXY when .gomodproxy is present
2+
# and that the proxy actually serves modules.
3+
go env GOPROXY
4+
stdout ^http://.*/mod,file://.*$
5+
6+
go list -m -versions fruit.com
7+
stdout 'v1.0.0'
8+
9+
go get -d fruit.com@v1.0.0
10+
11+
# The goproxytest package information should definitely be available
12+
# and cached.
13+
go get -d github.com/rogpeppe/go-internal/goproxytest
14+
15+
-- go.mod --
16+
module mod
17+
18+
-- .gomodproxy/_enable_overlay --
19+
We want to be able to use regular Go modules as well as test-related
20+
ones. The contents of this file are ignored; only
21+
its presence is necessary.
22+
-- .gomodproxy/fruit.com_v1.0.0/.mod --
23+
module fruit.com
24+
25+
-- .gomodproxy/fruit.com_v1.0.0/.info --
26+
{"Version":"v1.0.0","Time":"2018-10-22T18:45:39Z"}
27+
28+
-- .gomodproxy/fruit.com_v1.0.0/go.mod --
29+
module fruit.com
30+
31+
-- .gomodproxy/fruit.com_v1.0.0/fruit/fruit.go --
32+
package fruit
33+
34+
const Name = "Apple"

‎gotooltest/setup.go‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ package gotooltest
88

99
import (
1010
"bytes"
11-
"encoding/json"
1211
"fmt"
1312
"os"
1413
"os/exec"
@@ -18,6 +17,7 @@ import (
1817
"strings"
1918
"sync"
2019

20+
"github.com/rogpeppe/go-internal/internal/goenv"
2121
"github.com/rogpeppe/go-internal/testscript"
2222
)
2323

@@ -74,18 +74,9 @@ func initGoEnv() (err error) {
7474
tagStr = strings.Trim(tagStr, "[]")
7575
goEnv.releaseTags = strings.Split(tagStr, " ")
7676

77-
eout, stderr, err := run("go", "env", "-json",
78-
"GOROOT",
79-
"GOCACHE",
80-
"GOPROXY",
81-
)
82-
if err != nil {
83-
return fmt.Errorf("failed to determine environment from go command: %v\n%v", err, stderr)
84-
}
85-
if err := json.Unmarshal(eout.Bytes(), &goEnv); err != nil {
86-
return fmt.Errorf("failed to unmarshal GOROOT and GOCACHE tags from go command out: %v\n%v", err, eout)
77+
if err := goenv.Unmarshal(&goEnv); err != nil {
78+
return err
8779
}
88-
8980
version := goEnv.releaseTags[len(goEnv.releaseTags)-1]
9081
if !goVersionRegex.MatchString(version) {
9182
return fmt.Errorf("invalid go version %q", version)

‎internal/goenv/goenv.go‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
// goenv is used by various testscript-related package to avoid
2+
// invoking `go env` multiple times independently
3+
// when a single execution is sufficient.
4+
package goenv
5+
6+
import (
7+
"bytes"
8+
"encoding/json"
9+
"fmt"
10+
"os/exec"
11+
"sync"
12+
)
13+
14+
// Unmarshal JSON-unmarshals the result of `go env -json` into the value pointed
15+
// to by dst.
16+
func Unmarshal(dst any) error {
17+
data, err := goEnv()
18+
if err != nil {
19+
return err
20+
}
21+
if err := json.Unmarshal(data, dst); err != nil {
22+
return fmt.Errorf("failed to unmarshal environment from go command out: %v\n%s", err, data)
23+
}
24+
return nil
25+
}
26+
27+
var goEnv = sync.OnceValues(func() ([]byte, error) {
28+
var stdout, stderr bytes.Buffer
29+
// Note: we explicitly mention all the variables that
30+
// any of the callers of goenv might need to avoid
31+
// fetching any of the variables that are costly to calculate.
32+
cmd := exec.Command("go", "env", "-json",
33+
"GOCACHE",
34+
"GOMODCACHE",
35+
"GONOPROXY",
36+
"GOPROXY",
37+
"GOROOT",
38+
)
39+
cmd.Stdout = &stdout
40+
cmd.Stderr = &stderr
41+
if err := cmd.Run(); err != nil {
42+
return nil, fmt.Errorf("failed to determine environment from go command: %v\n%v", err, &stderr)
43+
}
44+
return stdout.Bytes(), nil
45+
})

0 commit comments

Comments
 (0)