teleport kubernetes doesn't support adding the "proxy" verb about teleport

[aviramha]

I tried doing workarounds, but nothing seems to work - tried using "proxy" as verb, or portforward - all fails with similar error.
I collected logs from teleport-agent, seems to get nil dereference.

2023-12-28T09:57:51Z ERRO [KUBERNETE] Unable to hijack the connection: does not implement http.Hijacker pid:6.1 forward/fwd.go:285
2023/12/28 09:57:51 http: panic serving 0.0.0.0:0: runtime error: invalid memory address or nil pointer dereference
goroutine 317868 [running]:
net/http.(*conn).serve.func1()
	net/http/server.go:1854 +0xbf
panic({0x77418c0, 0xe33ce40})
	runtime/panic.go:890 +0x263
github.com/gravitational/teleport/lib/kube/proxy.(*Forwarder).formatStatusResponseError(0xc00036c280, {0x9d19a20, 0xc00179b1a0}, {0x0, 0x0})
	github.com/gravitational/teleport/lib/kube/proxy/forwarder.go:733 +0x69
github.com/gravitational/teleport/lib/kube/proxy.(*Forwarder).formatForwardResponseError(...)
	github.com/gravitational/teleport/lib/kube/proxy/forwarder.go:715
github.com/gravitational/oxy/utils.ErrorHandlerFunc.ServeHTTP(0x8633e3e?, {0x9d19a20?, 0xc00179b1a0?}, 0x0?, {0x0?, 0x0?})
	github.com/gravitational/[email protected]/utils/handler.go:37 +0x43
github.com/gravitational/oxy/forward.(*websocketForwarder).serveHTTP(0xc00239a7a0, {0x9d19a20, 0xc00179b1a0}, 0x83504e0?, 0xc00239a7c0)
	github.com/gravitational/[email protected]/forward/fwd.go:286 +0xb9e
github.com/gravitational/oxy/forward.(*Forwarder).ServeHTTP(0xc001feaf60, {0x9d19a20, 0xc00179b1a0}, 0xc002075e60?)
	github.com/gravitational/[email protected]/forward/fwd.go:165 +0x55
github.com/gravitational/teleport/lib/kube/proxy.(*Forwarder).catchAll(0xc00036c280, 0xc0020c31e0, {0x7f800c244c40?, 0xc001ae20f0}, 0xc001df2800)
	github.com/gravitational/teleport/lib/kube/proxy/forwarder.go:2159 +0xae7
github.com/gravitational/teleport/lib/kube/proxy.(*Forwarder).withAuthStd.func1({0x7f800c244c40, 0xc001ae20f0}, 0xc001df2700)
	github.com/gravitational/teleport/lib/kube/proxy/forwarder.go:607 +0x5fa
github.com/gravitational/teleport/lib/httplib.MakeStdHandlerWithErrorWriter.func1({0x7f800c244c40, 0xc001ae20f0}, 0x0?)
	github.com/gravitational/teleport/lib/httplib/httplib.go:138 +0x78
net/http.HandlerFunc.ServeHTTP(0xc002414120?, {0x7f800c244c40?, 0xc001ae20f0?}, 0xc001ce4980?)
	net/http/server.go:2122 +0x2f
github.com/julienschmidt/httprouter.(*Router).ServeHTTP(0xc002414120, {0x7f800c244c40, 0xc001ae20f0}, 0xc001df2700)
	github.com/julienschmidt/[email protected]/router.go:460 +0x669
github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerResponseSize.func1({0x7f800c244c40?, 0xc001ae20a0?}, 0xc001df2700)
	github.com/prometheus/[email protected]/prometheus/promhttp/instrument_server.go:296 +0xce
net/http.HandlerFunc.ServeHTTP(0x9d2f710?, {0x7f800c244c40?, 0xc001ae20a0?}, 0x0?)
	net/http/server.go:2122 +0x2f
github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerCounter.func1({0x9d2f710?, 0xc001da9b60?}, 0xc001df2700)
	github.com/prometheus/[email protected]/prometheus/promhttp/instrument_server.go:147 +0xc5
net/http.HandlerFunc.ServeHTTP(0x578baa?, {0x9d2f710?, 0xc001da9b60?}, 0x30?)
	net/http/server.go:2122 +0x2f
github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerDuration.func2({0x9d2f710, 0xc001da9b60}, 0xc001df2700)
	github.com/prometheus/[email protected]/prometheus/promhttp/instrument_server.go:109 +0xc7
net/http.HandlerFunc.ServeHTTP(0x7f800c285758?, {0x9d2f710?, 0xc001da9b60?}, 0xc001df2700?)
	net/http/server.go:2122 +0x2f
github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerInFlight.func1({0x9d2f710, 0xc001da9b60}, 0x9c79001?)
	github.com/prometheus/[email protected]/prometheus/promhttp/instrument_server.go:60 +0xd4
net/http.HandlerFunc.ServeHTTP(0x9d304e8?, {0x9d2f710?, 0xc001da9b60?}, 0x9c79068?)
	net/http/server.go:2122 +0x2f
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp.(*middleware).serveHTTP(0xc002270630, {0x9d2f710?, 0xc001da97a0}, 0xc001df2600, {0x9cbb640, 0xc00227ff50})
	go.opentelemetry.io/contrib/instrumentation/net/http/[email protected]/handler.go:229 +0x122e
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp.NewMiddleware.func1.1({0x9d2f710?, 0xc001da97a0?}, 0xc001df2600?)
	go.opentelemetry.io/contrib/instrumentation/net/http/[email protected]/handler.go:81 +0x3b
net/http.HandlerFunc.ServeHTTP(0x578f87?, {0x9d2f710?, 0xc001da97a0?}, 0x9c7df01?)
	net/http/server.go:2122 +0x2f
github.com/gravitational/teleport/lib/kube/proxy.(*Forwarder).ServeHTTP(0x9d304e8?, {0x9d2f710?, 0xc001da97a0?}, 0x9c7df10?)
	github.com/gravitational/teleport/lib/kube/proxy/forwarder.go:410 +0x2c
github.com/gravitational/teleport/lib/auth.(*Middleware).ServeHTTP(0xc0024ca500, {0x9d2f710, 0xc001da97a0}, 0xc001df2500)
	github.com/gravitational/teleport/lib/auth/middleware.go:694 +0x685
github.com/gravitational/oxy/ratelimit.(*TokenLimiter).ServeHTTP(0xc002276d00, {0x9d2f710, 0xc001da97a0}, 0xc001df2500)
	github.com/gravitational/[email protected]/ratelimit/tokenlimiter.go:118 +0x1ce
github.com/gravitational/oxy/connlimit.(*ConnLimiter).ServeHTTP(0xc0024140c0, {0x9d2f710, 0xc001da97a0}, 0xb?)
	github.com/gravitational/[email protected]/connlimit/connlimit.go:75 +0x31d
github.com/gravitational/teleport/lib/httplib.MakeTracingHandler.func1({0x9d2f710, 0xc001da97a0}, 0xc001df2500)
	github.com/gravitational/teleport/lib/httplib/httplib.go:86 +0x1df
net/http.HandlerFunc.ServeHTTP(0x9d304e8?, {0x9d2f710?, 0xc001da97a0?}, 0x9c79068?)
	net/http/server.go:2122 +0x2f
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp.(*middleware).serveHTTP(0xc0022909a0, {0x9d2cf20?, 0xc001a84380}, 0xc001df2400, {0x9cbb640, 0xc0024c66c0})
	go.opentelemetry.io/contrib/instrumentation/net/http/[email protected]/handler.go:229 +0x122e
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp.NewMiddleware.func1.1({0x9d2cf20?, 0xc001a84380?}, 0x7a09220?)
	go.opentelemetry.io/contrib/instrumentation/net/http/[email protected]/handler.go:81 +0x3b
net/http.HandlerFunc.ServeHTTP(0xc001ce49cf?, {0x9d2cf20?, 0xc001a84380?}, 0x5d994e?)
	net/http/server.go:2122 +0x2f
net/http.serverHandler.ServeHTTP({0xc002377830?}, {0x9d2cf20, 0xc001a84380}, 0xc001df2400)
	net/http/server.go:2936 +0x316
net/http.(*conn).serve(0xc001a50630, {0x9d304e8, 0xc002376180})
	net/http/server.go:1995 +0x612
created by net/http.(*Server).Serve
	net/http/server.go:3089 +0x5ed

[aviramha]

Probably related to #11712

[tigrato]

@aviramha the fix is available #36079

Upgrading both Teleport proxies and Kube agents is necessary for this. Since you operate a Teleport cloud cluster, you'll need to wait through a merge, backport, and new release process before being able to test it.

I would appreciate it if you could provide me with the most concise steps to install metabear, allowing me to reproduce the issue. I've already tested it with #11712, and it functions correctly.

Ping me on slack if you prefer - tiago in Teleport community slack

[aviramha]

Thank you @tigrato ! I'll ping you on Slack with instructions. Appreciate the prompt support <3

[aviramha]

Update - @tigrato verified it works with our operator, thanks so much!