Golang short tips & tricks

This list of short golang code tips & tricks will help keep collected knowledge in one place.

Tips list

41 - Telegram bot
40 - Distributed consensus
39 - Public private struct fields and funcs
38 - Marshal Unmarshal to byte slice using gob
37 - Time series on leveldb
36 - Custom type marshal unmarshal json
35 - Test specific functions
34 - File path exists
33 - Table driven tests
32 - Work with consul
31 - Do not need any web framework
30 - Heart beat and lead election with etcd
29 - Partial json read
28 - Interact with etcd with http.Request
27 - Go-style concurrency in C
26 - Go channels are slow
25 - Avoid conversions with hidden alloc copy
24 - Slice shuffle
23 - Defer is slow
22 - Any number of args
21 - Test bench http request handler
20 - Use Atomics or GOMAXPROCS=1
19 - Chunked HTTP response with flusher
18 - Use for range for channels
17 - Use context API
16 - Go routines syncronization
15 - Time interval measurement
14 - Benchmark switch vs else if
13 - Use ASM in Go Code
12 - JSON with unknown structure
11 - Websocket over HTTP2
10 - HTTP2
9 - Error handling
8 - Memory management with pools of objects
7 - Sort slice of time periods
6 - Fast http server
5 - Close channel to notify many
4 - Is channel closed?
3 - Http request/response with close notify and timeout
2 - Import packages
1 - Map
0 - Slices

#41 - Telegram bot

2016-06-04 by @beyondns

Simple echo Telegram bot no external dependencies

package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"time"
)

// https://core.telegram.org/bots/api

const (
	API   = "https://api.telegram.org/bot"
	TOKEN = <YOUR TOKEN HERE>
)

type Chat struct {
	Id int `json:"id"`
}

type From struct {
	Id int `json:"id"`
}

type Message struct {
	Id   int    `json:"message_id"`
	From From   `json:"from"`
	Chat Chat   `json:"chat"`
	Text string `json:"text"`
	Date uint32 `json:"date"`
}

type Update struct {
	Id  int     `json:"update_id"`
	Msg Message `json:"message"`
}

type Result struct {
	Ok  bool     `json:"ok"`
	Res []Update `json:"result"`
}

/*
{
     "ok": true,
     "result": [{
         "update_id": 215439784,
         "message": {
             "message_id": 21,
             "from": {
                 "id": 209816615,
                 "first_name": "Black",
                 "last_name": "Ninja",
                 "username": "blackninja3k"
             },
             "chat": {
                 "id": 209816615,
                 "first_name": "Black",
                 "last_name": "Ninja",
                 "username": "blackninja3k",
                 "type": "private"
             },
             "date": 1459938757,
             "text": "hello"
         }
     }, {
         "update_id": 215439785,
         "message": {
             "message_id": 22,
             "from": {
                 "id": 209816615,
                 "first_name": "Black",
                 "last_name": "Ninja",
                 "username": "blackninja3k"
             },
             "chat": {
                 "id": 209816615,
                 "first_name": "Black",
                 "last_name": "Ninja",
                 "username": "blackninja3k",
                 "type": "private"
             },
             "date": 1459939461,
             "text": "pooooop"
         }
     }]
 }

*/

func HandleUpdate(u *Update) {
	log.Printf("upd:%v", *u)
	msgDate := time.Unix(int64(u.Msg.Date), 0)
	if time.Since(msgDate).Minutes() < 1 {
		SendMessage(u.Msg.Chat.Id, "u:"+u.Msg.Text)
	}
}

func main() {
	log.Printf("Me:%s", Me())
	var lastID = 0
	for {
		res := Updates()
		if res.Ok {
			for _, u := range res.Res {
				if u.Id > lastID {
					lastID = u.Id
					HandleUpdate(&u)
				}
			}
		}
		time.Sleep(time.Second * 2)
	}
}

func Me() string {
	st, d, err := httpRequest("GET", API+TOKEN+"/getMe", nil, time.Second*15)
	if err != nil {
		log.Fatal(err)
	}
	if st != http.StatusOK {
		log.Fatalf("status %d", st)
	}
	return string(d)
}

func Updates() Result {
	st, d, err := httpRequest("GET", API+TOKEN+"/getUpdates", nil, time.Second*15)
	if err != nil {
		log.Fatal(err)
	}
	if st != http.StatusOK {
		log.Fatalf("status %d", st)
	}

	res := Result{}
	if len(d) > 0 {
		//log.Printf(string(d))
		err := json.Unmarshal(d, &res)
		if err != nil {
			log.Fatal(err)
		}
	}
	return res
}

func SendMessage(chat_id int, text string) string {
	v := url.Values{}
	v.Add("chat_id", strconv.Itoa(chat_id))
	v.Add("text", text)
	st, d, err := httpRequest("GET", API+TOKEN+"/sendMessage?"+v.Encode(),
		nil, time.Second*15)
	if err != nil {
		log.Fatal(err)
	}
	if st != http.StatusOK {
		log.Fatalf("status %d", st)
	}
	return string(d)
}

func httpRequest(meth, u string, data []byte,
	timeLimit time.Duration) (int, []byte, error) {

	tr := &http.Transport{}
	client := &http.Client{Transport: tr}
	c := make(chan error, 1)

	var respStatus int
	var respBody []byte
	req, err := http.NewRequest(meth, u, bytes.NewBuffer(data))
	if err != nil {
		return 0, nil, err
	}

	go func() {
		resp, err := client.Do(req)

		if err != nil {
			goto E
		}

		respStatus = resp.StatusCode

		respBody, err = ioutil.ReadAll(resp.Body)
	E:
		c <- err
		if resp != nil && resp.Body != nil {
			resp.Body.Close()
		}
	}()

	select {
	case <-time.After(timeLimit):
		tr.CancelRequest(req)
		log.Printf("Request timeout")
		<-c // Wait for goroutine to return.
		return 0, nil, errors.New("request time out")
	case err := <-c:
		if err != nil {
			log.Printf("Error in request goroutine %v", err)
			return 0, nil, err
		}
		return respStatus, respBody, nil
	}

}

#40 - Distributed consensus

2016-01-04 by @beyondns

Distributed consensus is a common shared state or logic support equal across multiple nodes.
1-stage (semi or one-many) consensus: send data to nodes, get responses, find consensus on one node.
2-stage (full or many-many) consensus: send data to nodes, nodes send all-to-all, find consensus on each node.

package main

import (
	"flag"
	"log"
	"net/http"
	"fmt"
	"strings"
	"time"
	"bytes"
	"errors"
	"io/ioutil"
	"sync"
)

var (
	Nodes []string
)

func sysHandler1(w http.ResponseWriter, r *http.Request) {
	d, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, fmt.Sprintf("bad reguest, error %v", err), http.StatusBadRequest)
		return
	}
	log.Printf("sysHandler1: %s",string(d))
	w.WriteHeader(http.StatusOK)
}

func sysHandler2(w http.ResponseWriter, r *http.Request) {
	d, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, fmt.Sprintf("bad reguest, error %v", err), http.StatusBadRequest)
		return
	}
	log.Printf("sysHandler2: %s",string(d))
	if localConsensus(d,"1",w,r){
		w.WriteHeader(http.StatusOK)
	}

	w.WriteHeader(http.StatusOK)
}

func apiHandler(w http.ResponseWriter, r *http.Request) {
	d, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, fmt.Sprintf("bad reguest, error %v", err), http.StatusBadRequest)
		return
	}

	log.Printf("apiHandler: %s",string(d))

	if localConsensus(d,"2",w,r){
		w.WriteHeader(http.StatusOK)
	}

}

func localConsensus(d []byte, st string, 
	w http.ResponseWriter, r *http.Request) bool{
	if len(d)>0{
		ok,err:=SendDataToWorld(d,st)
		if err!=nil {
			http.Error(w, fmt.Sprintf("SendDataToWorld error %v", err), http.StatusBadRequest)
			return false
		}
		if !ok {
			http.Error(w, "no consensus", http.StatusBadRequest)
			return false
		}
	}
	return true
}

func main() {
	nodes := flag.String("nodes", "127.0.0.1:12379,127.0.0.1:22379,127.0.0.1:32379", "nodes host0:port0,host1:port1,..")
	api := flag.String("api", "127.0.0.1:12379", "api host:port")
	flag.Parse()

	Nodes=strings.Split(*nodes,",")

	log.Printf("Nodes %v",Nodes)

	http.HandleFunc("/api", apiHandler)
	http.HandleFunc("/sys1", sysHandler1)
	http.HandleFunc("/sys2", sysHandler2)
	log.Fatal(http.ListenAndServe(*api, nil))
}

func SendDataToWorld(data []byte, st string) (bool,error){

	var l = len(Nodes)
	var wg sync.WaitGroup
	wg.Add(l)

	var rd [][]byte = make([][]byte, l)
	var rs []int = make([]int, l)
	var re []bool = make([]bool, l)
	for i, u := range Nodes {
		go func(i int, u string) {
			var e error
			rs[i],rd[i],e=httpRequest("POST",
				"http://"+u+"/sys"+st,data,time.Second*15)
			re[i]=(e==nil)
			wg.Done()
		}(i, u)
	}
	wg.Wait()

	log.Printf("%v %v %v",rs,rd,re)

	return findConsensus(rs,rd,re)

}

func httpRequest(meth, u string, data []byte, 
	timeLimit time.Duration) (int, []byte, error) {

    tr := &http.Transport{}
    client := &http.Client{Transport: tr}
    c := make(chan error, 1)

    var respStatus int
    var respBody []byte
    req, err := http.NewRequest(meth, u, bytes.NewBuffer(data))
    if err != nil {
        return 0, nil, err
    }


    go func() {
        resp, err := client.Do(req)

        if err != nil {
            goto E
        }

        respStatus = resp.StatusCode

        respBody, err = ioutil.ReadAll(resp.Body)
    E:
        c <- err
        if resp != nil && resp.Body != nil {
            resp.Body.Close()
        }
    }()

    select {
    case <-time.After(timeLimit):
        tr.CancelRequest(req)
        log.Printf("Request timeout")
        <-c // Wait for goroutine to return.
        return 0, nil, errors.New("request time out")
    case err := <-c:
        if err != nil {
            log.Printf("Error in request goroutine %v", err)
            return 0, nil, err
        }
        return respStatus, respBody, nil
    }

}

func findConsensus(rs []int,rd [][]byte,re []bool)(bool,error){
	if len(rs) != len(rd) || len(rs) == 0 || len(rd) == 0 {
		panic(fmt.Sprintf("FindConsensus error papams: %v,%v", rs, rd))
	}
	ok:=0
	n:=0
	for i, _ := range rs{
		if !re[i]{
			continue	
		}
		n++
		if rs[i] == http.StatusOK{
			ok++
		}
	}
	log.Printf("find consensus %d : %d",ok,n)
	// 51% ok
	if float32(ok)>float32(n)/float32(2){
		return true,nil
	}
	return false,nil
}

#39 - Public private struct fields and funcs

2016-26-03 by @beyondns

Go has specific approach of definition private/public package fields and funcs. All declarations within package is visible (different os files).
But from other packages only names with 1st capital letter is visible (public)

lib/l.go

package mylib

import (
	"fmt"
)

const (
	Const1 = "c1"
	const2 = "c2"
)

var (
	Var1 = 1
	var2 = 2
)

type Mydata struct{
	X int // Public
	y int // private
}

func (md *Mydata) DoAPI(){
	fmt.Println("DoAPI")
}

func (md *Mydata) doAPI(){
	fmt.Println("doAPI")
}

main.go

package main

import(
	"./lib"
	"fmt"
)

func main(){
	x:=mylib.Mydata{
		X:1,
		//y:2, // error private
	}

	x.DoAPI()

//	x.doAPI() // error private

	fmt.Println(x.X)

//	fmt.Print(x.y) // error private

	fmt.Println(mylib.Const1)
//	fmt.Println(mylib.const2) // error private

	fmt.Println(mylib.Var1)
//	fmt.Println(mylib.var2) // error private

}

#38 - Marshal Unmarshal to byte slice usng gob

2016-25-03 by @beyondns

import (
	"bytes"
	"encoding/gob"
)

func gobMarshal(v interface{}) ([]byte, error) {
	var buf bytes.Buffer
	if err := gob.NewEncoder(&buf).Encode(v); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func gobUnmarshal(data []byte, v interface{}) error {
	return gob.NewDecoder(bytes.NewBuffer(data)).Decode(v)
}

#37 - Time series on leveldb

2016-22-03 by @beyondns

Use timestamp as a key in keyvalue storage, leveldb or any other. Just iterate over specific period of time.

import (
	"github.com/syndtr/goleveldb/leveldb"
)
var (
	bcdb *leveldb.DB
)

func init(){
	var err error
	bcdb, err = leveldb.OpenFile("./db/bc.db", nil)
	if err!=nil{
		xlog.Error(err)
		return
	}
	xlog.Notice("db init ok")
}

Test

func TestDbTS(t *testing.T) {
	err := bcdb.Put([]byte("t0"),[]byte("boom"), nil)
	if err!=nil{
		t.Error(err)
	}
	
	var timelabel int64
	batch := new(leveldb.Batch)
	for i:=0;i<16;i++{
		if i==8{
			timelabel=time.Now().UnixNano()
		}
		batch.Put([]byte(fmt.Sprintf("t%d",time.Now().UnixNano())), 
			[]byte(fmt.Sprintf("v%d",i)))
	}
	err = bcdb.Write(batch, nil)
	if err!=nil{
		t.Error(err)
	}

    iter := bcdb.NewIterator(nil, nil)
    for ok := iter.Seek([]byte(fmt.Sprintf("t%d",timelabel))); ok; ok = iter.Next() {
    	key,value := iter.Key(),iter.Value()
    	ks:=string(key)
    	if ks[0] == 't'{
			ks=ks[1:]    		
    	}
    	ki, err := strconv.ParseInt(ks, 10, 64)
    	if err!=nil{
    		xlog.Error(err)
    	}
    	xlog.Debugf("key time : %s | value: %s\n", time.Unix(0,ki), value)
	}
	iter.Release()
	err = iter.Error()
	if err!=nil{
		t.Error(err)
	}
}

#36 - Custom type marshal unmarshal json

2016-21-03 by @beyondns

Json marshalling can be defined for custom type by implementing MarshalJSON/UnmarshalJSON methods.

type ByteSlice []byte

func (s *ByteSlice) MarshalJSON() ([]byte, error) {
	return []byte(`"`+hex.EncodeToString(*s)+`"`), nil
}

func (s *ByteSlice) UnmarshalJSON(data []byte) error {
	d:=string(data)
	if d[0]=='"'{d=d[1:]}
	l:=len(d)
 	if d[l-1]=='"'{d=d[:l-1]}
	var err error
	*s,err=hex.DecodeString(d)
	return err
}

Testing

type ByteSliceHolder struct {
	Bin ByteSlice `json:"bin"`
}

func TestByteSliceJSON(t *testing.T) {
	bsh := &ByteSliceHolder{}
	_, err := fmt.Sscanf("082372739127341723ab", "%x", &bsh.Bin)
	if err != nil {
		t.Fatal("fmt.Sscanf failed")
	}
	bin, err := json.Marshal(bsh)

	if err != nil {
		t.Errorf("json.Marshal failed ", err)
	}
	//xlog.Debugf("%s", bin)

	bsh2 := &ByteSliceHolder{}
	err = json.Unmarshal(bin, &bsh2)

	if err != nil {
		t.Errorf("json.Marshal failed ", err)
	}

	if !bytes.Equal(bsh.Bin,bsh2.Bin){
		t.Errorf("json.Marshal failed ", err)
	
	}

}

#35 - Test specific functions

2016-21-03 by @beyondns

To test TestFunc1

go test -run Func1

run argument is a regex, "Func$" test all funcs with "Func" at beginning.

Run only specific test cases in golang go test

#34 - File path exists

2016-15-03 by @beyondns

if _, err := os.Stat(filepath); os.IsNotExist(err) {
	// doesn't exist
}

if _, err := os.Stat(filepath); err == nil {
	// exists
}

#33 - Table driven tests

2016-15-03 by @beyondns

Table driven tests very simple but efficient. An example shamelessly stolen from advanced-testing-with-go

package main

import (
	"testing"
)

func TestAdd(t *testing.T) {
	cases := []struct{ A, B, Sum int }{
		{1, 1, 2},
		{1, -1, 0},
		{1, 0, 1},
		{0, 0, 0},
		{3, 2, 1}, // error!!!
	}

	for _, c := range cases {
		a := c.A + c.B
		e := c.Sum
		if a != e {
			t.Errorf("%d + %d = %d, expected %d", c.A, c.B, a, e)
		}
	}

}

--- FAIL: TestAdd (0.00s)
	tdt_test.go:20: 3 + 2 = 5, expected 1

advanced-testing-with-go

#32 - Work with consul

2016-14-03 by @beyondns

Consul has its own client but just use net/http

consul agent -dev -advertise=127.0.0.1

package main

import (
	"errors"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"time"
	"encoding/json"
)

// https://www.consul.io/docs/agent/http/kv.html

var (
	consulKV = "http://127.0.0.1:8500/v1/kv/"
)

func httpRequest(meth, u, val string, timeLimit time.Duration) (int, []byte, error) {

	tr := &http.Transport{}
	client := &http.Client{Transport: tr}
	c := make(chan error, 1)

	var respStatus int
	var respBody []byte

	req, err := http.NewRequest(meth, u, strings.NewReader(val))
	if err != nil {
		return 0, nil, err
	}


	go func() {
		resp, err := client.Do(req)

		if err != nil {
			goto E
		}

		respStatus = resp.StatusCode

		respBody, err = ioutil.ReadAll(resp.Body)
	E:
		c <- err
		if resp != nil && resp.Body != nil {
			resp.Body.Close()
		}
	}()

	select {
	case <-time.After(timeLimit):
		tr.CancelRequest(req)
		log.Printf("Request timeout")
		<-c // Wait for goroutine to return.
		return 0, nil, errors.New("request time out")
	case err := <-c:
		if err != nil {
			log.Printf("Error in request goroutine %v", err)
			return 0, nil, err
		}
		return respStatus, respBody, nil
	}

}

//curl -v -XPUT http://127.0.0.1:8500/v1/kv/boom -d foo

func consulSet(k, v string) (int, []byte, error) {
	return httpRequest("PUT", consulKV+k, v, time.Second)
}

//curl -v http://127.0.0.1:8500/v1/kv/boom

func consulGet(k string) (int, []byte, error) {
	return httpRequest("GET", consulKV+k, "", time.Second)
}

type KVPair struct {
	Key         string
	CreateIndex uint64
	ModifyIndex uint64
	LockIndex   uint64
	Flags       uint64
	Value       []byte
	Session     string
}

func main() {

	s, d, err := consulSet("foo", "hero")
	if err != nil {
		log.Fatalf("Set error %v", err)
	}
	log.Printf("set %d %s", s, string(d))

	s, d, err = consulGet("foo")
	if err != nil {
		log.Fatalf("Get error %v", err)
	}

	if s!=200{
		log.Fatalf("Get status %d", s)
	}

    var kv []KVPair	
	json.Unmarshal(d,&kv)

	if len(kv)==0{
		log.Fatalf("len(kv)==0")
	}

	log.Printf("get %d %s %s", s, string(d), kv[0].Value)

}

2016/03/15 20:49:54 set 200 true
2016/03/15 20:49:54 get 200 [{"LockIndex":0,"Key":"foo","Flags":0,"Value":"aGVybw==","CreateIndex":7,"ModifyIndex":252}] hero

gotips #0..#31

General Golang links

50 Shades of Go: Traps, Gotchas, and Common Mistakes for New Golang Devs

Inspired by

jstips

hushi55 / gotips Goto Github PK

gotips's Introduction

Tips list

#41 - Telegram bot

#40 - Distributed consensus

#39 - Public private struct fields and funcs

#38 - Marshal Unmarshal to byte slice usng gob

#37 - Time series on leveldb

#36 - Custom type marshal unmarshal json

#35 - Test specific functions

#34 - File path exists

#33 - Table driven tests

#32 - Work with consul

General Golang links

Inspired by

License

gotips's People

Contributors

Watchers

Recommend Projects

Recommend Topics

Recommend Org

Jobs