GithubHelp home page GithubHelp logo

Comments (3)

amwolff avatar amwolff commented on August 28, 2024 2

I just realized what was wrong and managed to reproduce this panic.

Steps to reproduce:

  1. Run gateway-mt or gateway-st with uploaded parts verification enabled
  2. Run programs below:
2.1
// Usage: go run main.go 

package main


import (

"context"

"crypto/rand"

"fmt"

"io"

"os"


"storj.io/common/memory"
"storj.io/uplink"



)


const bucket, key = "testbucket", "testfile"


func mustNilErr(err error) {

if err != nil {

panic(err)

}

}


func mustNilErrWithAbort(ctx context.Context, err error, part *uplink.PartUpload, project *uplink.Project, uploadID string) {

if err != nil {

if err2 := part.Abort(); err2 != nil {

err = fmt.Errorf("%v: %w", err, err2)

}

if err2 := project.AbortUpload(ctx, bucket, key, uploadID); err2 != nil {

err = fmt.Errorf("%v: %w", err, err2)

}

panic(err)

}

}


func main() {

ctx := context.Background()


access, err := uplink.ParseAccess(os.Args[1])
mustNilErr(err)

project, err := uplink.OpenProject(ctx, access)
mustNilErr(err)

defer func() { mustNilErr(project.Close()) }()

_, err = project.EnsureBucket(ctx, bucket)
mustNilErr(err)

upload, err := project.BeginUpload(ctx, bucket, key, nil)
mustNilErr(err)

for i := uint32(1337); i <= 1339; i++ {
	part, err := project.UploadPart(ctx, bucket, key, upload.UploadID, i)
	mustNilErrWithAbort(ctx, err, part, project, upload.UploadID)

	_, err = io.CopyN(part, rand.Reader, 4*memory.KB.Int64())
	mustNilErrWithAbort(ctx, err, part, project, upload.UploadID)

	err = part.Commit()
	mustNilErrWithAbort(ctx, err, part, project, upload.UploadID)
}

fmt.Println(upload.UploadID)



}

2.2
package main

import (

"fmt"

"os"


"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"



)


func exitErrorf(msg string, args ...interface{}) {

fmt.Fprintf(os.Stderr, msg+"\n", args...)

os.Exit(1)

}


func main() {

sess, err := session.NewSession(&aws.Config{

Credentials:      credentials.NewStaticCredentials("", "", ""),

Endpoint:         aws.String("http://127.0.0.1:7777"),

Region:           aws.String("eu1"),

S3ForcePathStyle: aws.Bool(true),

})

if err != nil {

exitErrorf("NewSession: %v", err)

}


svc := s3.New(sess)

_, err = svc.CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{
	Bucket: aws.String("testbucket"),
	Key:    aws.String("testfile"),
	MultipartUpload: &s3.CompletedMultipartUpload{
		Parts: []*s3.CompletedPart{
			{PartNumber: aws.Int64(0)},
			{PartNumber: aws.Int64(1)},
			{PartNumber: aws.Int64(2)},
		},
	},
	UploadId: aws.String("<Upload-ID>"),
})
if err != nil {
	exitErrorf("CompleteMultipartUpload: %v", err)
}



}

gateway's output:

2022-01-31T17:05:36.031+0100    INFO    http/server.go:3160     http: panic serving 127.0.0.1:54441: runtime error: index out of range [10] with length 3
goroutine 74 [running]:
net/http.(*conn).serve.func1(0x140008400a0)
        net/http/server.go:1802 +0xdc
panic({0x105de0b80, 0x140002682e8})
        runtime/panic.go:1052 +0x2ac
storj.io/minio/cmd.criticalErrorHandler.ServeHTTP.func1(0x14000842100, {0x105f21bb0, 0x14000858000})
        storj.io/[email protected]/cmd/generic-handlers.go:789 +0x158
panic({0x105de0b80, 0x140002682e8})
        runtime/panic.go:1038 +0x21c
github.com/spacemonkeygo/monkit/v3.newSpan.func1(0x1400033ded0)
        github.com/spacemonkeygo/monkit/[email protected]/ctx.go:147 +0x360
panic({0x105de0b80, 0x140002682e8})
        runtime/panic.go:1052 +0x2ac
storj.io/gateway/miniogw.(*gatewayLayer).CompleteMultipartUpload(0x14000042480, {0x105f31ff8, 0x14000849e60}, {0x1400000ac06, 0xa}, {0x1400000ac11, 0x8}, {0x1400000ac23, 0xc5}, {0x1400058a460, ...}, ...)
        storj.io/gateway/miniogw/multipart.go:303 +0xa24
storj.io/gateway/miniogw.(*singleTenancyLayer).CompleteMultipartUpload(0x14000848f90, {0x105f31ff8, 0x14000849920}, {0x1400000ac06, 0xa}, {0x1400000ac11, 0x8}, {0x1400000ac23, 0xc5}, {0x1400058a460, ...}, ...)
        storj.io/gateway/miniogw/single_tenant.go:215 +0x124
storj.io/minio/cmd.objectAPIHandlers.CompleteMultipartUploadHandler({0x105ee8c70, 0x105ee8c50}, {0x105f23e90, 0x140008401e0}, 0x14000842300)
        storj.io/[email protected]/cmd/object-handlers.go:2625 +0x14b4
net/http.HandlerFunc.ServeHTTP(...)
        net/http/server.go:2047
storj.io/minio/cmd.httpTraceAll.func1({0x105f23e90, 0x140008401e0}, 0x14000842300)
        storj.io/[email protected]/cmd/handler-utils.go:352 +0x124
net/http.HandlerFunc.ServeHTTP(...)
        net/http/server.go:2047
storj.io/minio/cmd.collectAPIStats.func1({0x105f23e90, 0x14000840140}, 0x14000842300)
        storj.io/[email protected]/cmd/handler-utils.go:379 +0xf4
net/http.HandlerFunc.ServeHTTP(...)
        net/http/server.go:2047
storj.io/minio/cmd.maxClients.func1({0x105f23e90, 0x14000840140}, 0x14000842300)
        storj.io/[email protected]/cmd/handler-api.go:147 +0x2b0
net/http.HandlerFunc.ServeHTTP(0x1400128e700, {0x105f23e90, 0x14000840140}, 0x14000842300)
        net/http/server.go:2047 +0x40
storj.io/minio/cmd.redirectHandler.ServeHTTP({{0x105f02f80, 0x1400128e700}}, {0x105f23e90, 0x14000840140}, 0x14000842300)
        storj.io/[email protected]/cmd/generic-handlers.go:201 +0x23c
storj.io/minio/cmd.customHeaderHandler.ServeHTTP({{0x105f03e60, 0x1400033d200}}, {0x105f23e60, 0x140005681c8}, 0x14000842300)
        storj.io/[email protected]/cmd/generic-handlers.go:759 +0x158
storj.io/minio/cmd.securityHeaderHandler.ServeHTTP({{0x105f03dc0, 0x1400033d210}}, {0x105f23e60, 0x140005681c8}, 0x14000842300)
        storj.io/[email protected]/cmd/generic-handlers.go:774 +0x1d4
storj.io/minio/cmd.bucketForwardingHandler.ServeHTTP({0x140008495c0, {0x105f03f20, 0x1400033d220}}, {0x105f23e60, 0x140005681c8}, 0x14000842300)
        storj.io/[email protected]/cmd/generic-handlers.go:632 +0xeb4
storj.io/minio/cmd.requestValidityHandler.ServeHTTP({{0x105f03d20, 0x14000568168}}, {0x105f23e60, 0x140005681c8}, 0x14000842300)
        storj.io/[email protected]/cmd/generic-handlers.go:616 +0x2f4
storj.io/minio/cmd.httpStatsHandler.ServeHTTP({{0x105f03ec0, 0x1400033d250}}, {0x105f21bb0, 0x14000858000}, 0x14000842300)
        storj.io/[email protected]/cmd/generic-handlers.go:544 +0x114
storj.io/minio/cmd.requestSizeLimitHandler.ServeHTTP({{0x105f03de0, 0x1400033d260}, 0x50004000000}, {0x105f21bb0, 0x14000858000}, 0x14000842300)
        storj.io/[email protected]/cmd/generic-handlers.go:69 +0x110
storj.io/minio/cmd.requestHeaderSizeLimitHandler.ServeHTTP({{0x105f03ea0, 0x14000568180}}, {0x105f21bb0, 0x14000858000}, 0x14000842300)
        storj.io/[email protected]/cmd/generic-handlers.go:94 +0x154
storj.io/minio/cmd.crossDomainPolicy.ServeHTTP({{0x105f03e80, 0x1400033d2a0}}, {0x105f21bb0, 0x14000858000}, 0x14000842300)
        storj.io/[email protected]/cmd/crossdomain-xml-handler.go:51 +0x110
storj.io/minio/cmd.browserRedirectHandler.ServeHTTP({{0x105f03da0, 0x1400033d2d0}}, {0x105f21bb0, 0x14000858000}, 0x14000842300)
        storj.io/[email protected]/cmd/generic-handlers.go:285 +0x84
storj.io/minio/cmd.minioReservedBucketHandler.ServeHTTP({{0x105f03d00, 0x1400033d2e0}}, {0x105f21bb0, 0x14000858000}, 0x14000842300)
        storj.io/[email protected]/cmd/generic-handlers.go:353 +0x2d8
storj.io/minio/cmd.cacheControlHandler.ServeHTTP({{0x105f03e20, 0x1400033d2f0}}, {0x105f21bb0, 0x14000858000}, 0x14000842300)
        storj.io/[email protected]/cmd/generic-handlers.go:312 +0x330
storj.io/minio/cmd.timeValidityHandler.ServeHTTP({{0x105f03d40, 0x1400033d300}}, {0x105f21bb0, 0x14000858000}, 0x14000842300)
        storj.io/[email protected]/cmd/generic-handlers.go:423 +0x338
storj.io/minio/cmd.resourceHandler.ServeHTTP({{0x105f03f60, 0x1400033d310}}, {0x105f21bb0, 0x14000858000}, 0x14000842300)
        storj.io/[email protected]/cmd/generic-handlers.go:524 +0x29c
storj.io/minio/cmd.authHandler.ServeHTTP({{0x105f03f00, 0x1400033d320}}, {0x105f21bb0, 0x14000858000}, 0x14000842300)
        storj.io/[email protected]/cmd/auth-handler.go:505 +0x23c
storj.io/minio/cmd.sseTLSHandler.ServeHTTP({{0x105f03ce0, 0x1400033d330}}, {0x105f21bb0, 0x14000858000}, 0x14000842300)
        storj.io/[email protected]/cmd/generic-handlers.go:810 +0x3ac
storj.io/minio/cmd.reservedMetadataHandler.ServeHTTP({{0x105f03f40, 0x1400033d340}}, {0x105f21bb0, 0x14000858000}, 0x14000842300)
        storj.io/[email protected]/cmd/generic-handlers.go:140 +0x154
github.com/gorilla/mux.(*Router).ServeHTTP(0x1400050bd40, {0x105f21bb0, 0x14000858000}, 0x14000842100)
        github.com/gorilla/[email protected]/mux.go:210 +0x1e4
github.com/rs/cors.(*Cors).Handler.func1({0x105f21bb0, 0x14000858000}, 0x14000842100)
        github.com/rs/[email protected]/cors.go:219 +0x21c
net/http.HandlerFunc.ServeHTTP(0x1400131c0a0, {0x105f21bb0, 0x14000858000}, 0x14000842100)
        net/http/server.go:2047 +0x40
storj.io/minio/cmd.criticalErrorHandler.ServeHTTP({{0x105f02f80, 0x1400131c0a0}}, {0x105f21bb0, 0x14000858000}, 0x14000842100)
        storj.io/[email protected]/cmd/generic-handlers.go:792 +0x8c
storj.io/minio/cmd/http.(*Server).Start.func1({0x105f21bb0, 0x14000858000}, 0x14000842100)
        storj.io/[email protected]/cmd/http/server.go:102 +0xc0
net/http.HandlerFunc.ServeHTTP(0x140002ca000, {0x105f21bb0, 0x14000858000}, 0x14000842100)
        net/http/server.go:2047 +0x40
net/http.serverHandler.ServeHTTP({0x14001264360}, {0x105f21bb0, 0x14000858000}, 0x14000842100)
        net/http/server.go:2879 +0x444
net/http.(*conn).serve(0x140008400a0, {0x105f31ff8, 0x140002901b0})
        net/http/server.go:1930 +0xb6c
created by net/http.(*Server).Serve
        net/http/server.go:3034 +0x4b8
2022-01-31T17:05:36.129+0100    INFO    http/server.go:3160     http: panic serving 127.0.0.1:54443: runtime error: index out of range [10] with length 3
goroutine 86 [running]:
net/http.(*conn).serve.func1(0x140005d0820)
        net/http/server.go:1802 +0xdc
panic({0x105de0b80, 0x14000368660})
        runtime/panic.go:1052 +0x2ac
storj.io/minio/cmd.criticalErrorHandler.ServeHTTP.func1(0x140007fc700, {0x105f21bb0, 0x140005da1c0})
        storj.io/[email protected]/cmd/generic-handlers.go:789 +0x158
panic({0x105de0b80, 0x14000368660})
        runtime/panic.go:1038 +0x21c
github.com/spacemonkeygo/monkit/v3.newSpan.func1(0x14000265080)
        github.com/spacemonkeygo/monkit/[email protected]/ctx.go:147 +0x360
panic({0x105de0b80, 0x14000368660})
        runtime/panic.go:1052 +0x2ac
storj.io/gateway/miniogw.(*gatewayLayer).CompleteMultipartUpload(0x14000042480, {0x105f31ff8, 0x140005f48d0}, {0x1400010c306, 0xa}, {0x1400010c311, 0x8}, {0x1400010c323, 0xc5}, {0x1400022efa0, ...}, ...)
        storj.io/gateway/miniogw/multipart.go:303 +0xa24
storj.io/gateway/miniogw.(*singleTenancyLayer).CompleteMultipartUpload(0x14000848f90, {0x105f31ff8, 0x140005f43f0}, {0x1400010c306, 0xa}, {0x1400010c311, 0x8}, {0x1400010c323, 0xc5}, {0x1400022efa0, ...}, ...)
        storj.io/gateway/miniogw/single_tenant.go:215 +0x124
storj.io/minio/cmd.objectAPIHandlers.CompleteMultipartUploadHandler({0x105ee8c70, 0x105ee8c50}, {0x105f23e90, 0x140005d0960}, 0x140007fc900)
        storj.io/[email protected]/cmd/object-handlers.go:2625 +0x14b4
net/http.HandlerFunc.ServeHTTP(...)
        net/http/server.go:2047
storj.io/minio/cmd.httpTraceAll.func1({0x105f23e90, 0x140005d0960}, 0x140007fc900)
        storj.io/[email protected]/cmd/handler-utils.go:352 +0x124
net/http.HandlerFunc.ServeHTTP(...)
        net/http/server.go:2047
storj.io/minio/cmd.collectAPIStats.func1({0x105f23e90, 0x140005d08c0}, 0x140007fc900)
        storj.io/[email protected]/cmd/handler-utils.go:379 +0xf4
net/http.HandlerFunc.ServeHTTP(...)
        net/http/server.go:2047
storj.io/minio/cmd.maxClients.func1({0x105f23e90, 0x140005d08c0}, 0x140007fc900)
        storj.io/[email protected]/cmd/handler-api.go:147 +0x2b0
net/http.HandlerFunc.ServeHTTP(0x1400128e700, {0x105f23e90, 0x140005d08c0}, 0x140007fc900)
        net/http/server.go:2047 +0x40
storj.io/minio/cmd.redirectHandler.ServeHTTP({{0x105f02f80, 0x1400128e700}}, {0x105f23e90, 0x140005d08c0}, 0x140007fc900)
        storj.io/[email protected]/cmd/generic-handlers.go:201 +0x23c
storj.io/minio/cmd.customHeaderHandler.ServeHTTP({{0x105f03e60, 0x14000264c10}}, {0x105f23e60, 0x140005089c0}, 0x140007fc900)
        storj.io/[email protected]/cmd/generic-handlers.go:759 +0x158
storj.io/minio/cmd.securityHeaderHandler.ServeHTTP({{0x105f03dc0, 0x14000264c20}}, {0x105f23e60, 0x140005089c0}, 0x140007fc900)
        storj.io/[email protected]/cmd/generic-handlers.go:774 +0x1d4
storj.io/minio/cmd.bucketForwardingHandler.ServeHTTP({0x140005f40c0, {0x105f03f20, 0x14000264c30}}, {0x105f23e60, 0x140005089c0}, 0x140007fc900)
        storj.io/[email protected]/cmd/generic-handlers.go:632 +0xeb4
storj.io/minio/cmd.requestValidityHandler.ServeHTTP({{0x105f03d20, 0x14000508978}}, {0x105f23e60, 0x140005089c0}, 0x140007fc900)
        storj.io/[email protected]/cmd/generic-handlers.go:616 +0x2f4
storj.io/minio/cmd.httpStatsHandler.ServeHTTP({{0x105f03ec0, 0x14000264c60}}, {0x105f21bb0, 0x140005da1c0}, 0x140007fc900)
        storj.io/[email protected]/cmd/generic-handlers.go:544 +0x114
storj.io/minio/cmd.requestSizeLimitHandler.ServeHTTP({{0x105f03de0, 0x14000264c70}, 0x50004000000}, {0x105f21bb0, 0x140005da1c0}, 0x140007fc900)
        storj.io/[email protected]/cmd/generic-handlers.go:69 +0x110
storj.io/minio/cmd.requestHeaderSizeLimitHandler.ServeHTTP({{0x105f03ea0, 0x14000508990}}, {0x105f21bb0, 0x140005da1c0}, 0x140007fc900)
        storj.io/[email protected]/cmd/generic-handlers.go:94 +0x154
storj.io/minio/cmd.crossDomainPolicy.ServeHTTP({{0x105f03e80, 0x14000264c80}}, {0x105f21bb0, 0x140005da1c0}, 0x140007fc900)
        storj.io/[email protected]/cmd/crossdomain-xml-handler.go:51 +0x110
storj.io/minio/cmd.browserRedirectHandler.ServeHTTP({{0x105f03da0, 0x14000264c90}}, {0x105f21bb0, 0x140005da1c0}, 0x140007fc900)
        storj.io/[email protected]/cmd/generic-handlers.go:285 +0x84
storj.io/minio/cmd.minioReservedBucketHandler.ServeHTTP({{0x105f03d00, 0x14000264ca0}}, {0x105f21bb0, 0x140005da1c0}, 0x140007fc900)
        storj.io/[email protected]/cmd/generic-handlers.go:353 +0x2d8
storj.io/minio/cmd.cacheControlHandler.ServeHTTP({{0x105f03e20, 0x14000264cb0}}, {0x105f21bb0, 0x140005da1c0}, 0x140007fc900)
        storj.io/[email protected]/cmd/generic-handlers.go:312 +0x330
storj.io/minio/cmd.timeValidityHandler.ServeHTTP({{0x105f03d40, 0x14000264cc0}}, {0x105f21bb0, 0x140005da1c0}, 0x140007fc900)
        storj.io/[email protected]/cmd/generic-handlers.go:423 +0x338
storj.io/minio/cmd.resourceHandler.ServeHTTP({{0x105f03f60, 0x14000264cd0}}, {0x105f21bb0, 0x140005da1c0}, 0x140007fc900)
        storj.io/[email protected]/cmd/generic-handlers.go:524 +0x29c
storj.io/minio/cmd.authHandler.ServeHTTP({{0x105f03f00, 0x14000264ce0}}, {0x105f21bb0, 0x140005da1c0}, 0x140007fc900)
        storj.io/[email protected]/cmd/auth-handler.go:505 +0x23c
storj.io/minio/cmd.sseTLSHandler.ServeHTTP({{0x105f03ce0, 0x14000264cf0}}, {0x105f21bb0, 0x140005da1c0}, 0x140007fc900)
        storj.io/[email protected]/cmd/generic-handlers.go:810 +0x3ac
storj.io/minio/cmd.reservedMetadataHandler.ServeHTTP({{0x105f03f40, 0x14000264d00}}, {0x105f21bb0, 0x140005da1c0}, 0x140007fc900)
        storj.io/[email protected]/cmd/generic-handlers.go:140 +0x154
github.com/gorilla/mux.(*Router).ServeHTTP(0x1400050bd40, {0x105f21bb0, 0x140005da1c0}, 0x140007fc700)
        github.com/gorilla/[email protected]/mux.go:210 +0x1e4
github.com/rs/cors.(*Cors).Handler.func1({0x105f21bb0, 0x140005da1c0}, 0x140007fc700)
        github.com/rs/[email protected]/cors.go:219 +0x21c
net/http.HandlerFunc.ServeHTTP(0x1400131c0a0, {0x105f21bb0, 0x140005da1c0}, 0x140007fc700)
        net/http/server.go:2047 +0x40
storj.io/minio/cmd.criticalErrorHandler.ServeHTTP({{0x105f02f80, 0x1400131c0a0}}, {0x105f21bb0, 0x140005da1c0}, 0x140007fc700)
        storj.io/[email protected]/cmd/generic-handlers.go:792 +0x8c
storj.io/minio/cmd/http.(*Server).Start.func1({0x105f21bb0, 0x140005da1c0}, 0x140007fc700)
        storj.io/[email protected]/cmd/http/server.go:102 +0xc0
net/http.HandlerFunc.ServeHTTP(0x140002ca000, {0x105f21bb0, 0x140005da1c0}, 0x140007fc700)
        net/http/server.go:2047 +0x40
net/http.serverHandler.ServeHTTP({0x14001264360}, {0x105f21bb0, 0x140005da1c0}, 0x140007fc700)
        net/http/server.go:2879 +0x444
net/http.(*conn).serve(0x140005d0820, {0x105f31ff8, 0x140002901b0})
        net/http/server.go:1930 +0xb6c
created by net/http.(*Server).Serve
        net/http/server.go:3034 +0x4b8
2022-01-31T17:05:36.279+0100    INFO    http/server.go:3160     http: panic serving 127.0.0.1:54444: runtime error: index out of range [10] with length 3
goroutine 58 [running]:
net/http.(*conn).serve.func1(0x140002f4000)
        net/http/server.go:1802 +0xdc
panic({0x105de0b80, 0x140008761c8})
        runtime/panic.go:1052 +0x2ac
storj.io/minio/cmd.criticalErrorHandler.ServeHTTP.func1(0x140002f8000, {0x105f21bb0, 0x1400087a000})
        storj.io/[email protected]/cmd/generic-handlers.go:789 +0x158
panic({0x105de0b80, 0x140008761c8})
        runtime/panic.go:1038 +0x21c
github.com/spacemonkeygo/monkit/v3.newSpan.func1(0x140002845d0)
        github.com/spacemonkeygo/monkit/[email protected]/ctx.go:147 +0x360
panic({0x105de0b80, 0x140008761c8})
        runtime/panic.go:1052 +0x2ac
storj.io/gateway/miniogw.(*gatewayLayer).CompleteMultipartUpload(0x14000042480, {0x105f31ff8, 0x14000290d20}, {0x1400060c206, 0xa}, {0x1400060c211, 0x8}, {0x1400060c223, 0xc5}, {0x140002ce280, ...}, ...)
        storj.io/gateway/miniogw/multipart.go:303 +0xa24
storj.io/gateway/miniogw.(*singleTenancyLayer).CompleteMultipartUpload(0x14000848f90, {0x105f31ff8, 0x14000290840}, {0x1400060c206, 0xa}, {0x1400060c211, 0x8}, {0x1400060c223, 0xc5}, {0x140002ce280, ...}, ...)
        storj.io/gateway/miniogw/single_tenant.go:215 +0x124
storj.io/minio/cmd.objectAPIHandlers.CompleteMultipartUploadHandler({0x105ee8c70, 0x105ee8c50}, {0x105f23e90, 0x140002f4140}, 0x140002f8200)
        storj.io/[email protected]/cmd/object-handlers.go:2625 +0x14b4
net/http.HandlerFunc.ServeHTTP(...)
        net/http/server.go:2047
storj.io/minio/cmd.httpTraceAll.func1({0x105f23e90, 0x140002f4140}, 0x140002f8200)
        storj.io/[email protected]/cmd/handler-utils.go:352 +0x124
net/http.HandlerFunc.ServeHTTP(...)
        net/http/server.go:2047
storj.io/minio/cmd.collectAPIStats.func1({0x105f23e90, 0x140002f40a0}, 0x140002f8200)
        storj.io/[email protected]/cmd/handler-utils.go:379 +0xf4
net/http.HandlerFunc.ServeHTTP(...)
        net/http/server.go:2047
storj.io/minio/cmd.maxClients.func1({0x105f23e90, 0x140002f40a0}, 0x140002f8200)
        storj.io/[email protected]/cmd/handler-api.go:147 +0x2b0
net/http.HandlerFunc.ServeHTTP(0x1400128e700, {0x105f23e90, 0x140002f40a0}, 0x140002f8200)
        net/http/server.go:2047 +0x40
storj.io/minio/cmd.redirectHandler.ServeHTTP({{0x105f02f80, 0x1400128e700}}, {0x105f23e90, 0x140002f40a0}, 0x140002f8200)
        storj.io/[email protected]/cmd/generic-handlers.go:201 +0x23c
storj.io/minio/cmd.customHeaderHandler.ServeHTTP({{0x105f03e60, 0x14000284160}}, {0x105f23e60, 0x14000288180}, 0x140002f8200)
        storj.io/[email protected]/cmd/generic-handlers.go:759 +0x158
storj.io/minio/cmd.securityHeaderHandler.ServeHTTP({{0x105f03dc0, 0x14000284170}}, {0x105f23e60, 0x14000288180}, 0x140002f8200)
        storj.io/[email protected]/cmd/generic-handlers.go:774 +0x1d4
storj.io/minio/cmd.bucketForwardingHandler.ServeHTTP({0x14000290510, {0x105f03f20, 0x14000284180}}, {0x105f23e60, 0x14000288180}, 0x140002f8200)
        storj.io/[email protected]/cmd/generic-handlers.go:632 +0xeb4
storj.io/minio/cmd.requestValidityHandler.ServeHTTP({{0x105f03d20, 0x14000288138}}, {0x105f23e60, 0x14000288180}, 0x140002f8200)
        storj.io/[email protected]/cmd/generic-handlers.go:616 +0x2f4
storj.io/minio/cmd.httpStatsHandler.ServeHTTP({{0x105f03ec0, 0x140002841b0}}, {0x105f21bb0, 0x1400087a000}, 0x140002f8200)
        storj.io/[email protected]/cmd/generic-handlers.go:544 +0x114
storj.io/minio/cmd.requestSizeLimitHandler.ServeHTTP({{0x105f03de0, 0x140002841c0}, 0x50004000000}, {0x105f21bb0, 0x1400087a000}, 0x140002f8200)
        storj.io/[email protected]/cmd/generic-handlers.go:69 +0x110
storj.io/minio/cmd.requestHeaderSizeLimitHandler.ServeHTTP({{0x105f03ea0, 0x14000288150}}, {0x105f21bb0, 0x1400087a000}, 0x140002f8200)
        storj.io/[email protected]/cmd/generic-handlers.go:94 +0x154
storj.io/minio/cmd.crossDomainPolicy.ServeHTTP({{0x105f03e80, 0x140002841d0}}, {0x105f21bb0, 0x1400087a000}, 0x140002f8200)
        storj.io/[email protected]/cmd/crossdomain-xml-handler.go:51 +0x110
storj.io/minio/cmd.browserRedirectHandler.ServeHTTP({{0x105f03da0, 0x140002841e0}}, {0x105f21bb0, 0x1400087a000}, 0x140002f8200)
        storj.io/[email protected]/cmd/generic-handlers.go:285 +0x84
storj.io/minio/cmd.minioReservedBucketHandler.ServeHTTP({{0x105f03d00, 0x140002841f0}}, {0x105f21bb0, 0x1400087a000}, 0x140002f8200)
        storj.io/[email protected]/cmd/generic-handlers.go:353 +0x2d8
storj.io/minio/cmd.cacheControlHandler.ServeHTTP({{0x105f03e20, 0x14000284200}}, {0x105f21bb0, 0x1400087a000}, 0x140002f8200)
        storj.io/[email protected]/cmd/generic-handlers.go:312 +0x330
storj.io/minio/cmd.timeValidityHandler.ServeHTTP({{0x105f03d40, 0x14000284210}}, {0x105f21bb0, 0x1400087a000}, 0x140002f8200)
        storj.io/[email protected]/cmd/generic-handlers.go:423 +0x338
storj.io/minio/cmd.resourceHandler.ServeHTTP({{0x105f03f60, 0x14000284220}}, {0x105f21bb0, 0x1400087a000}, 0x140002f8200)
        storj.io/[email protected]/cmd/generic-handlers.go:524 +0x29c
storj.io/minio/cmd.authHandler.ServeHTTP({{0x105f03f00, 0x14000284230}}, {0x105f21bb0, 0x1400087a000}, 0x140002f8200)
        storj.io/[email protected]/cmd/auth-handler.go:505 +0x23c
storj.io/minio/cmd.sseTLSHandler.ServeHTTP({{0x105f03ce0, 0x14000284240}}, {0x105f21bb0, 0x1400087a000}, 0x140002f8200)
        storj.io/[email protected]/cmd/generic-handlers.go:810 +0x3ac
storj.io/minio/cmd.reservedMetadataHandler.ServeHTTP({{0x105f03f40, 0x14000284250}}, {0x105f21bb0, 0x1400087a000}, 0x140002f8200)
        storj.io/[email protected]/cmd/generic-handlers.go:140 +0x154
github.com/gorilla/mux.(*Router).ServeHTTP(0x1400050bd40, {0x105f21bb0, 0x1400087a000}, 0x140002f8000)
        github.com/gorilla/[email protected]/mux.go:210 +0x1e4
github.com/rs/cors.(*Cors).Handler.func1({0x105f21bb0, 0x1400087a000}, 0x140002f8000)
        github.com/rs/[email protected]/cors.go:219 +0x21c
net/http.HandlerFunc.ServeHTTP(0x1400131c0a0, {0x105f21bb0, 0x1400087a000}, 0x140002f8000)
        net/http/server.go:2047 +0x40
storj.io/minio/cmd.criticalErrorHandler.ServeHTTP({{0x105f02f80, 0x1400131c0a0}}, {0x105f21bb0, 0x1400087a000}, 0x140002f8000)
        storj.io/[email protected]/cmd/generic-handlers.go:792 +0x8c
storj.io/minio/cmd/http.(*Server).Start.func1({0x105f21bb0, 0x1400087a000}, 0x140002f8000)
        storj.io/[email protected]/cmd/http/server.go:102 +0xc0
net/http.HandlerFunc.ServeHTTP(0x140002ca000, {0x105f21bb0, 0x1400087a000}, 0x140002f8000)
        net/http/server.go:2047 +0x40
net/http.serverHandler.ServeHTTP({0x14001264360}, {0x105f21bb0, 0x1400087a000}, 0x140002f8000)
        net/http/server.go:2879 +0x444
net/http.(*conn).serve(0x140002f4000, {0x105f31ff8, 0x140002901b0})
        net/http/server.go:1930 +0xb6c
created by net/http.(*Server).Serve
        net/http/server.go:3034 +0x4b8
2022-01-31T17:05:36.480+0100    INFO    http/server.go:3160     http: panic serving 127.0.0.1:54445: runtime error: index out of range [10] with length 3
goroutine 90 [running]:
net/http.(*conn).serve.func1(0x140005d1220)
        net/http/server.go:1802 +0xdc
panic({0x105de0b80, 0x14000368858})
        runtime/panic.go:1052 +0x2ac
storj.io/minio/cmd.criticalErrorHandler.ServeHTTP.func1(0x140007fca00, {0x105f21bb0, 0x140005da2a0})
        storj.io/[email protected]/cmd/generic-handlers.go:789 +0x158
panic({0x105de0b80, 0x14000368858})
        runtime/panic.go:1038 +0x21c
github.com/spacemonkeygo/monkit/v3.newSpan.func1(0x140002657f0)
        github.com/spacemonkeygo/monkit/[email protected]/ctx.go:147 +0x360
panic({0x105de0b80, 0x14000368858})
        runtime/panic.go:1052 +0x2ac
storj.io/gateway/miniogw.(*gatewayLayer).CompleteMultipartUpload(0x14000042480, {0x105f31ff8, 0x140005f55c0}, {0x1400010c406, 0xa}, {0x1400010c411, 0x8}, {0x1400010c423, 0xc5}, {0x1400022f3b0, ...}, ...)
        storj.io/gateway/miniogw/multipart.go:303 +0xa24
storj.io/gateway/miniogw.(*singleTenancyLayer).CompleteMultipartUpload(0x14000848f90, {0x105f31ff8, 0x140005f50e0}, {0x1400010c406, 0xa}, {0x1400010c411, 0x8}, {0x1400010c423, 0xc5}, {0x1400022f3b0, ...}, ...)
        storj.io/gateway/miniogw/single_tenant.go:215 +0x124
storj.io/minio/cmd.objectAPIHandlers.CompleteMultipartUploadHandler({0x105ee8c70, 0x105ee8c50}, {0x105f23e90, 0x140005d1360}, 0x140007fcc00)
        storj.io/[email protected]/cmd/object-handlers.go:2625 +0x14b4
net/http.HandlerFunc.ServeHTTP(...)
        net/http/server.go:2047
storj.io/minio/cmd.httpTraceAll.func1({0x105f23e90, 0x140005d1360}, 0x140007fcc00)
        storj.io/[email protected]/cmd/handler-utils.go:352 +0x124
net/http.HandlerFunc.ServeHTTP(...)
        net/http/server.go:2047
storj.io/minio/cmd.collectAPIStats.func1({0x105f23e90, 0x140005d12c0}, 0x140007fcc00)
        storj.io/[email protected]/cmd/handler-utils.go:379 +0xf4
net/http.HandlerFunc.ServeHTTP(...)
        net/http/server.go:2047
storj.io/minio/cmd.maxClients.func1({0x105f23e90, 0x140005d12c0}, 0x140007fcc00)
        storj.io/[email protected]/cmd/handler-api.go:147 +0x2b0
net/http.HandlerFunc.ServeHTTP(0x1400128e700, {0x105f23e90, 0x140005d12c0}, 0x140007fcc00)
        net/http/server.go:2047 +0x40
storj.io/minio/cmd.redirectHandler.ServeHTTP({{0x105f02f80, 0x1400128e700}}, {0x105f23e90, 0x140005d12c0}, 0x140007fcc00)
        storj.io/[email protected]/cmd/generic-handlers.go:201 +0x23c
storj.io/minio/cmd.customHeaderHandler.ServeHTTP({{0x105f03e60, 0x14000265380}}, {0x105f23e60, 0x14000508db0}, 0x140007fcc00)
        storj.io/[email protected]/cmd/generic-handlers.go:759 +0x158
storj.io/minio/cmd.securityHeaderHandler.ServeHTTP({{0x105f03dc0, 0x14000265390}}, {0x105f23e60, 0x14000508db0}, 0x140007fcc00)
        storj.io/[email protected]/cmd/generic-handlers.go:774 +0x1d4
storj.io/minio/cmd.bucketForwardingHandler.ServeHTTP({0x140005f4db0, {0x105f03f20, 0x140002653a0}}, {0x105f23e60, 0x14000508db0}, 0x140007fcc00)
        storj.io/[email protected]/cmd/generic-handlers.go:632 +0xeb4
storj.io/minio/cmd.requestValidityHandler.ServeHTTP({{0x105f03d20, 0x14000508d68}}, {0x105f23e60, 0x14000508db0}, 0x140007fcc00)
        storj.io/[email protected]/cmd/generic-handlers.go:616 +0x2f4
storj.io/minio/cmd.httpStatsHandler.ServeHTTP({{0x105f03ec0, 0x140002653d0}}, {0x105f21bb0, 0x140005da2a0}, 0x140007fcc00)
        storj.io/[email protected]/cmd/generic-handlers.go:544 +0x114
storj.io/minio/cmd.requestSizeLimitHandler.ServeHTTP({{0x105f03de0, 0x140002653e0}, 0x50004000000}, {0x105f21bb0, 0x140005da2a0}, 0x140007fcc00)
        storj.io/[email protected]/cmd/generic-handlers.go:69 +0x110
storj.io/minio/cmd.requestHeaderSizeLimitHandler.ServeHTTP({{0x105f03ea0, 0x14000508d80}}, {0x105f21bb0, 0x140005da2a0}, 0x140007fcc00)
        storj.io/[email protected]/cmd/generic-handlers.go:94 +0x154
storj.io/minio/cmd.crossDomainPolicy.ServeHTTP({{0x105f03e80, 0x140002653f0}}, {0x105f21bb0, 0x140005da2a0}, 0x140007fcc00)
        storj.io/[email protected]/cmd/crossdomain-xml-handler.go:51 +0x110
storj.io/minio/cmd.browserRedirectHandler.ServeHTTP({{0x105f03da0, 0x14000265400}}, {0x105f21bb0, 0x140005da2a0}, 0x140007fcc00)
        storj.io/[email protected]/cmd/generic-handlers.go:285 +0x84
storj.io/minio/cmd.minioReservedBucketHandler.ServeHTTP({{0x105f03d00, 0x14000265410}}, {0x105f21bb0, 0x140005da2a0}, 0x140007fcc00)
        storj.io/[email protected]/cmd/generic-handlers.go:353 +0x2d8
storj.io/minio/cmd.cacheControlHandler.ServeHTTP({{0x105f03e20, 0x14000265420}}, {0x105f21bb0, 0x140005da2a0}, 0x140007fcc00)
        storj.io/[email protected]/cmd/generic-handlers.go:312 +0x330
storj.io/minio/cmd.timeValidityHandler.ServeHTTP({{0x105f03d40, 0x14000265430}}, {0x105f21bb0, 0x140005da2a0}, 0x140007fcc00)
        storj.io/[email protected]/cmd/generic-handlers.go:423 +0x338
storj.io/minio/cmd.resourceHandler.ServeHTTP({{0x105f03f60, 0x14000265440}}, {0x105f21bb0, 0x140005da2a0}, 0x140007fcc00)
        storj.io/[email protected]/cmd/generic-handlers.go:524 +0x29c
storj.io/minio/cmd.authHandler.ServeHTTP({{0x105f03f00, 0x14000265450}}, {0x105f21bb0, 0x140005da2a0}, 0x140007fcc00)
        storj.io/[email protected]/cmd/auth-handler.go:505 +0x23c
storj.io/minio/cmd.sseTLSHandler.ServeHTTP({{0x105f03ce0, 0x14000265460}}, {0x105f21bb0, 0x140005da2a0}, 0x140007fcc00)
        storj.io/[email protected]/cmd/generic-handlers.go:810 +0x3ac
storj.io/minio/cmd.reservedMetadataHandler.ServeHTTP({{0x105f03f40, 0x14000265470}}, {0x105f21bb0, 0x140005da2a0}, 0x140007fcc00)
        storj.io/[email protected]/cmd/generic-handlers.go:140 +0x154
github.com/gorilla/mux.(*Router).ServeHTTP(0x1400050bd40, {0x105f21bb0, 0x140005da2a0}, 0x140007fca00)
        github.com/gorilla/[email protected]/mux.go:210 +0x1e4
github.com/rs/cors.(*Cors).Handler.func1({0x105f21bb0, 0x140005da2a0}, 0x140007fca00)
        github.com/rs/[email protected]/cors.go:219 +0x21c
net/http.HandlerFunc.ServeHTTP(0x1400131c0a0, {0x105f21bb0, 0x140005da2a0}, 0x140007fca00)
        net/http/server.go:2047 +0x40
storj.io/minio/cmd.criticalErrorHandler.ServeHTTP({{0x105f02f80, 0x1400131c0a0}}, {0x105f21bb0, 0x140005da2a0}, 0x140007fca00)
        storj.io/[email protected]/cmd/generic-handlers.go:792 +0x8c
storj.io/minio/cmd/http.(*Server).Start.func1({0x105f21bb0, 0x140005da2a0}, 0x140007fca00)
        storj.io/[email protected]/cmd/http/server.go:102 +0xc0
net/http.HandlerFunc.ServeHTTP(0x140002ca000, {0x105f21bb0, 0x140005da2a0}, 0x140007fca00)
        net/http/server.go:2047 +0x40
net/http.serverHandler.ServeHTTP({0x14001264360}, {0x105f21bb0, 0x140005da2a0}, 0x140007fca00)
        net/http/server.go:2879 +0x444
net/http.(*conn).serve(0x140005d1220, {0x105f31ff8, 0x140002901b0})
        net/http/server.go:1930 +0xb6c
created by net/http.(*Server).Serve
        net/http/server.go:3034 +0x4b8
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x2 addr=0x18 pc=0x104714f28]

goroutine 76 [running]:

bufio.(*Writer).Available(...)

bufio/bufio.go:624

bufio.(*Writer).WriteString(0x0, {0x1057c2f27, 0x9})

bufio/bufio.go:706 +0x78

net/http.writeStatusLine(0x0, 0x1, 0xc8, {0x140008580cb, 0x3, 0x3})

net/http/server.go:1492 +0x44

net/http.(*chunkWriter).writeHeader(0x14000858040, {0x1400085c000, 0x28, 0x800})

net/http/server.go:1462 +0x1144

net/http.(*chunkWriter).Write(0x14000858040, {0x1400085c000, 0x28, 0x800})

net/http/server.go:370 +0x48

bufio.(*Writer).Flush(0x14000847e00)

bufio/bufio.go:607 +0x64

net/http.(*response).Flush(0x14000858000)

net/http/server.go:1659 +0x48

storj.io/minio/cmd/http/stats.(*OutgoingTrafficMeter).Flush(0x140005681c8)

storj.io/[email protected]/cmd/http/stats/http-traffic-recorder.go:58 +0x54

storj.io/minio/cmd/logger.(*ResponseWriter).Flush(0x14000840140)

storj.io/[email protected]/cmd/logger/audit.go:120 +0x54

storj.io/minio/cmd/logger.(*ResponseWriter).Flush(0x140008401e0)

storj.io/[email protected]/cmd/logger/audit.go:120 +0x54

storj.io/minio/cmd.sendWhiteSpace.func1({0x105f23e00, 0x14000849e30}, 0x1400083e1e0)

storj.io/[email protected]/cmd/object-handlers.go:2469 +0xb4

created by storj.io/minio/cmd.sendWhiteSpace

storj.io/[email protected]/cmd/object-handlers.go:2453 +0x64

What happens?

It's really interesting; it's not obvious from the stack trace.

  1. CompleteMultipartUpload handlers sends spaces for picky clients that would otherwise terminate while we verify
  2. During verification, we did out of bounds access for the uploaded parts slice
    2.1 Someone would send us parts 0, 1, 2... and we would list 1337, 1338, 1339...
    2.2 Who's right? We assumed client. Wrong.
  3. This caused panic, and net/http catches this panic. This only resets the connection from the client's perspective
    3.1 Gateway should be intact

The problem is that after a few seconds, the concurrent space-sending goroutine will try to write to the client as it didn't hear from CompleteMultipartUpload yet. But the resources associated with that connection are long gone now. It terminates the gateway since this panic isn't in the handler by now.

This change should solve all related problems: https://review.dev.storj.io/c/storj/gateway/+/6764.

from edge.

halkyon avatar halkyon commented on August 28, 2024 1

relevant changes that fixed this in production:

https://review.dev.storj.io/c/storj/gateway/+/5920
https://review.dev.storj.io/c/storj/gateway-mt/+/5921

it would be interesting if we could reproduce this on ST as well, or if it's just MT in very specific circumstances.

from edge.

storjBuildBot avatar storjBuildBot commented on August 28, 2024

Change https://review.dev.storj.io/c/storj/gateway/+/6764 mentions this issue.

from edge.

Related Issues (20)

Recommend Projects

  • React photo React

    A declarative, efficient, and flexible JavaScript library for building user interfaces.

  • Vue.js photo Vue.js

    🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.

  • Typescript photo Typescript

    TypeScript is a superset of JavaScript that compiles to clean JavaScript output.

  • TensorFlow photo TensorFlow

    An Open Source Machine Learning Framework for Everyone

  • Django photo Django

    The Web framework for perfectionists with deadlines.

  • D3 photo D3

    Bring data to life with SVG, Canvas and HTML. 📊📈🎉

Recommend Topics

  • javascript

    JavaScript (JS) is a lightweight interpreted programming language with first-class functions.

  • web

    Some thing interesting about web. New door for the world.

  • server

    A server is a program made to process requests and deliver data to clients.

  • Machine learning

    Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.

  • Game

    Some thing interesting about game, make everyone happy.

Recommend Org

  • Facebook photo Facebook

    We are working to build community through open source technology. NB: members must have two-factor auth.

  • Microsoft photo Microsoft

    Open source projects and samples from Microsoft.

  • Google photo Google

    Google ❤️ Open Source for everyone.

  • D3 photo D3

    Data-Driven Documents codes.