Interceptionid is repeat about cdp

[jokerbo1]

Thank you ,now running is normal ,only occasionally pause,Thank you very much。

[mafredri]

How are you trying to use interception?

[jokerbo1]

the code:

	// event clients before enabling events so that we don't miss any.
	if err = c.Page.Enable(ctx); err != nil {
		return err
	}
	sbul := network.NewSetBlockedURLsArgs([]string{"*.jpg", "*.gif", "*.css", "*.png", "*.jpeg"})
	c.Network.SetBlockedURLs(ctx, sbul)
	url := "*"
	rii := network.NewSetRequestInterceptionArgs([]network.RequestPattern{network.RequestPattern{URLPattern: &url}})
	err = c.Network.SetRequestInterception(ctx, rii)
	if err != nil {
		log.Fatal("intece error:", err.Error())
	}

	go func() {
		ric, err := c.Network.RequestIntercepted(ctx)
		if err != nil {
			log.Fatalln(err.Error())
		}
		for {
			select {
			case <-ric.Ready():
				ri, er := ric.Recv()
				if er != nil {
					log.Fatalln("rec err:", er.Error())
				}
				d, _ := json.Marshal(ri)
				log.Println(string(d))
				cira := network.NewContinueInterceptedRequestArgs(ri.InterceptionID)
				cira.SetURL(ri.Request.URL).SetMethod(ri.Request.Method).
				//SetPostData(*(ri.Request.PostData)).
					SetHeaders(ri.Request.Headers).
					SetErrorReason(ri.ResponseErrorReason)

				//time.Sleep(3 * time.Second)
				e := c.Network.ContinueInterceptedRequest(ctx, cira)
				if e != nil {
					log.Fatalln(e.Error())
				}
				// default:
				// 	log.Println("没有拦截住")
				// 	time.Sleep(1 * time.Second)
			}
		}
	}()

[jokerbo1]

now i meet another problem is the url request's time equals ctx's timeout time ,this why ?can you solution this problem? ,eg ,this is request finish ,but the willbesent functon RECV() is blocking ,until context is going to timeout.

	select {
			case <-rc.Ready():
				log.Println("3:", time.Now().Format(time.RFC3339))
				r1, r1e := rc.Recv()
				if r1e != nil {
					signals = true
					log.Println("r1e:", r1e.Error())

					break

				}
				res := Response{r1.Request.URL, r1.Request.Method}
				urls = append(urls, res)
				//log.Println(r1.Request.URL)
			}

[jokerbo1]

hello ,now set cookie exists problem,please help me.
cdp.Network: SetCookies: websocket: close 1006 (abnormal closure): unexpected EOF
this is cookie:

[
{
   "name": "PRD",
  "value": "05WFTkwa0rJzrqcZdklX4t_u:S"
 
},
{
    "name": "_RDG",
    "value": "281c45bd33bc6c23222159f54b10b673ef"
    
},
{
  "name": "_RF1",
 "value": "121.79.134.50"
   
},
{
    "domain": ".elong.com",
    "expirationDate": 4042022400,
    "hostOnly": false,
    "httpOnly": false,
    "name": "_RGUID",
    "path": "/",
    "sameSite": "no_restriction",
    "secure": false,
    "session": false,
    "storeId": "0",
    "value": "730508e6-48a9-4907-b125-15fc0951bdc8",
    "id": 4
},
{
    "domain": "elp.corp.elong.com",
    "hostOnly": true,
    "httpOnly": true,
    "name": "JSESSIONID",
    "path": "/elp",
    "sameSite": "no_restriction",
    "secure": false,
    "session": true,
    "storeId": "0",
    "value": "22752664E8D949FBDB7A238800DD42B9",
    "id": 22
}
]

[mafredri]

now i meet another problem is the url request's time equals ctx's timeout time ,this why ?can you solution this problem? ,eg ,this is request finish ,but the willbesent functon RECV() is blocking ,until context is going to timeout.

Recv() will not block since you are checking the Ready()-channel before calling Recv(). So I don't think there should be a problem here?

hello ,now set cookie exists problem,please help me.
cdp.Network: SetCookies: websocket: close 1006 (abnormal closure): unexpected EOF

Are you using the latest version of cdp? This message means the websocket connection has terminated unexpectedly, usually this is due to the write buffer size not being large enough. You can try to increase it when connecting (via rpcc) rpcc#WithWriteBufferSize.

[mafredri]

Also, I don't think you should be setting all those values on ContinueInterceptedRequestArgs (since you are not modifying them). This should be enough:

c.Network.ContinueInterceptedRequest(ctx, network.NewContinueInterceptedRequestArgs(ri.InterceptionID))

I recommend looking at other resources for learning how to best work with the Chrome DevTools Protocol, for example, the mailing list can be a good place to start.

[jokerbo1]

HI,sir ,thank you,follow your way the code is running normal . now ,i want ask a problem ,i how to abort a request and set request timeout ,because i want fetch one page's url include js'request and page request,now i can running a fetch url ,but the time is too long,request to response time is 4s,beyond the context time(=4000ms),this is my code:

func (cp *ChromeProtocol) FetchUrl(method, cookie, headers, url string) ([]Response, error) {
	var urls = []Response{}
	ctx, cancel := context.WithTimeout(context.Background(), cp.Timeout)
	defer cancel()
	devt := devtool.New(cp.Remotehost)
	pt, err := devt.Get(ctx, devtool.Page)
	if err != nil {
		pt, err = devt.Create(ctx)
		if err != nil {
			return nil, err
		}
	}
	cp.ctx = ctx
	cp.t = pt
	conn, err := rpcc.DialContext(cp.ctx, cp.t.WebSocketDebuggerURL,rpcc.WithWriteBufferSize(1048562))
	if err != nil {
		return nil, err
	}
	defer conn.Close() // Leaving connections open will leak memory.
	c := cdp.NewClient(conn)
	domContent, err := c.Page.DOMContentEventFired(cp.ctx)
	if err != nil {
		return nil, err
	}
	defer domContent.Close()
	if err = c.Page.Enable(cp.ctx); err != nil {
		return nil, err
	}

	sbul := network.NewSetBlockedURLsArgs([]string{"*.jpg", "*.gif", "*.css", "*.png", "*.jpeg"})
	c.Network.SetBlockedURLs(cp.ctx, sbul)

	if cookie != "" {
		var cookies []network.CookieParam
		e := json.Unmarshal([]byte(cookie), &cookies)
		if e != nil {
			return nil, errors.New("cookie 格式传递错误,原因:" + e.Error())
		}
		sca := network.NewSetCookiesArgs(cookies)
		err := c.Network.SetCookies(cp.ctx, sca)
		if err != nil {
			return nil, errors.New("设置cookie错误,原因:" + err.Error())
		}
	}

	scd := new(network.SetCacheDisabledArgs)
	enable := new(network.EnableArgs)
	enable.SetMaxResourceBufferSize(10 * 1024 * 1024)
	enable.SetMaxTotalBufferSize(10 * 1024 * 1024)
	scd.CacheDisabled = true
	c.Network.Enable(cp.ctx, enable)
	err = c.Network.SetCacheDisabled(cp.ctx, scd)
	if err != nil {
		return nil, errors.New("禁用缓存失败,原因:" + err.Error())
	}
		rii := network.NewSetRequestInterceptionArgs([]network.RequestPattern{network.RequestPattern{URLPattern: &url}})
		err = c.Network.SetRequestInterception(cp.ctx, rii)
		if err != nil {
			return nil, errors.New("拦截url失败,原因:" + err.Error())
		}
		ric, err := c.Network.RequestIntercepted(cp.ctx)
		if err != nil {
			return nil, errors.New("设置拦截url失败,原因:" + err.Error())
		}
		//为了拦截错误的标签
		go func(iurl string) {
			var sig = false
			for {
				if sig {
					return
				}
				select {
				case <-ric.Ready():
					ri, er := ric.Recv()
					if er != nil {
						log.Println(er.Error())
						return
					}
					cira := network.NewContinueInterceptedRequestArgs(ri.InterceptionID)

					if ri.Request.URL == iurl {
						if strings.ToLower(method) == "post" {
							cira.SetURL(ri.Request.URL).SetMethod(http.MethodPost)
						}
						//SetPostData(*(ri.Request.PostData)).
						if strings.TrimSpace(headers) != "" {
							h := new(network.Headers)
							e := h.UnmarshalJSON([]byte(headers))
							if e != nil {
								cp.intere <- errors.New("header 格式不正确,原因:" + e.Error())
								sig = true
								break
							}
							h1, e := h.MarshalJSON()
							if e != nil {
								cp.intere <- errors.New("header 生成data失败,原因:" + e.Error())
								sig = true
								break
							}
							ri.Request.Headers = append(ri.Request.Headers, h1...)
						}
						cira.SetHeaders(ri.Request.Headers).
							SetErrorReason(ri.ResponseErrorReason)
						e := c.Network.ContinueInterceptedRequest(cp.ctx, cira)
						if e != nil {
							cp.intere <- errors.New("post 请求失败:" + e.Error())
							sig = true
							break
						}
					}else{
						if matchUrl(ri.Request.URL){
							//直接终止
							cira.ErrorReason=network.ErrorReasonAborted
							_= c.Network.ContinueInterceptedRequest(cp.ctx, cira)
						} else if !matchjs(ri.Request.URL){
							cira.ErrorReason=network.ErrorReasonAborted
							_= c.Network.ContinueInterceptedRequest(cp.ctx, cira)
						}else{
							_= c.Network.ContinueInterceptedRequest(cp.ctx, cira)
						}
					}


				}
			}
		}(url)
	//}

	if strings.ToLower(method) == "get" && strings.TrimSpace(headers) != "" {
		h := new(network.Headers)
		e := h.UnmarshalJSON([]byte(headers))
		if e != nil {
			return nil, errors.New("header 格式不正确,原因:" + e.Error())
		}
		e = c.Network.SetExtraHTTPHeaders(cp.ctx, network.NewSetExtraHTTPHeadersArgs(*h))
		if e != nil {
			return nil, errors.New("设置header 错误,原因:" + e.Error())
		}
	}
	navArgs := page.NewNavigateArgs(url).
		SetReferrer("http://devtester.elong.com")
	_, err = c.Page.Navigate(cp.ctx, navArgs)
	if err != nil {
		return nil, errors.New("navigate error:" + err.Error())
	}

	rc, err := c.Network.RequestWillBeSent(cp.ctx)
	if err != nil {
		return nil, errors.New("fetch url fail:" + err.Error())
	}
	log.Println(time.Now().Format(time.RFC3339))
	//异步接收响应结果
		var signals = false
		for {
			log.Println("进 ",time.Now().Format(time.RFC3339))
			if signals {
				break
			}
			//log.Println("1:", time.Now().Format(time.RFC3339))
			select {
			case <-rc.Ready():
				log.Println("进1 ",time.Now().Format(time.RFC3339))
				//log.Println("3:", time.Now().Format(time.RFC3339))
				r1, r1e := rc.Recv()
				log.Println("进2 ",time.Now().Format(time.RFC3339))
				if r1e != nil {
					signals = true
					log.Println("r1e:", r1e.Error())

					break

				}
				res := Response{r1.Request.URL, r1.Request.Method}
				urls = append(urls, res)
				//log.Println(r1.Request.URL)
			}
			//log.Println("2:", time.Now().Format(time.RFC3339))
			log.Println("出",time.Now().Format(time.RFC3339))
		}
	log.Println(time.Now().Format(time.RFC3339))
		cp.complete <- true
	//发送完成
	//log.Println(time.Now().Format(time.RFC3339))

	return urls, nil

}

[jokerbo1]

i found in RequestWillBeSent func , at some point the chan<-Ready() will block 2s ,this is runing time log ,please help me see the problem:

2017/12/14 15:54:40 进  2017-12-14T15:54:40+08:00
2017/12/14 15:54:40 进1  2017-12-14T15:54:40+08:00
2017/12/14 15:54:40 进2  2017-12-14T15:54:40+08:00
2017/12/14 15:54:40 出 2017-12-14T15:54:40+08:00
**2017/12/14 15:54:40 进  2017-12-14T15:54:40+08:00**
// <-Ready() :wait ready signal diff 2s
**2017/12/14 15:54:42 进1  2017-12-14T15:54:42+08:00**
2017/12/14 15:54:42 进2  2017-12-14T15:54:42+08:00
2017/12/14 15:54:42 出 2017-12-14T15:54:42+08:00

[mafredri]

Regarding your first problem, why not increase the timeout of the context? Or fashion some other kind of keep-alive / timeout system.

As a quick example:

ctx, cancel := context.WithCancel(context.TODO())
keepAlive := make(chan struct{}, 1)
go func() {
	for {
		select {
		case <-time.After(4*time.Second):
			cancel()
			return
		case <-keepAlive:
		}
	}
}

To prevent timeout, e.g. on every incoming network request, or response, send a message on the keepAlive channel keepAlive <- struct{}{}.

For the problem with RequestWillBeSent the Ready() channel will block until there is a message. Please enable protocol (rpcc) logging (see logging example in documentation) to see when the messages are actually coming in.