This weekend i’ve been playing with my i3 desktop settings. Here are the result

Made with:

• Window Manager: i3
• Background: Ori
• Color Scheme: Nord
• Polybar: adi1090x/polybar-themes-8
• Conky: conky

If you are in the need of converting your custom intellij live tempate, you can use this method to transform it.

Locate the template file

The first thing you need to is is to locate the template file. You can find it inside the intellij settings directory which is vary per operating system. See this documentation to locate it.

For example for my golang templte in linux, I find it under

/home/username/.config/JetBrains/IntelliJIdea2020.1/jba_config/templates/Golang.xml

Convert it into vscode snippet

You can use tihs gocode

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143

package main

import (
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"strings"
)

type TemplateSet struct {
	XMLName  xml.Name       `xml:"templateSet"`
	Text     string         `xml:",chardata"`
	Group    string         `xml:"group,attr"`
	Template []ideaTemplate `xml:"template"`
}

type ideaTemplate struct {
	Text             string `xml:",chardata"`
	Name             string `xml:"name,attr"`
	Value            string `xml:"value,attr"`
	Description      string `xml:"description,attr"`
	ToReformat       string `xml:"toReformat,attr"`
	ToShortenFQNames string `xml:"toShortenFQNames,attr"`
	Variable         []struct {
		Text         string `xml:",chardata"`
		Name         string `xml:"name,attr"`
		Expression   string `xml:"expression,attr"`
		DefaultValue string `xml:"defaultValue,attr"`
		AlwaysStopAt string `xml:"alwaysStopAt,attr"`
	} `xml:"variable"`
	Context struct {
		Text   string `xml:",chardata"`
		Option struct {
			Text  string `xml:",chardata"`
			Name  string `xml:"name,attr"`
			Value string `xml:"value,attr"`
		} `xml:"option"`
	} `xml:"context"`
}

// "Print to console": {
// 	"prefix": "log",
// 	"body": [
// 		"console.log('$1');",
// 		"$2"
// 	],
// 	"description": "Log output to console"
// }

type snippet struct {
	Prefix      string   `json:"prefix,omitempty"`
	Body        []string `json:"body,omitempty"`
	Description string   `json:"description,omitempty"`
}

func main() {
	f, err := os.Open("Golang.xml")
	if err != nil {
		log.Fatal(err)
	}

	defer f.Close()

	raw, err := ioutil.ReadAll(f)
	if err != nil {
		log.Fatal(err)
	}

	var ts TemplateSet
	xml.Unmarshal(raw, &ts)

	snippets := map[string]snippet{}
	for i, t := range ts.Template {
		s := snippet{
			Prefix:      t.Name,
			Body:        getBody(t),
			Description: t.Description,
		}
		if _, ok := snippets[t.Description]; ok {
			log.Fatalf("duplicate key %d:%s %s", i, t.Description, t.Name)
		}
		snippets[t.Description] = s
	}

	raw, err = json.MarshalIndent(snippets, "", "\t")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s", raw)
}

var (
	rx = regexp.MustCompile(`\$\w+\$`)
)

func getBody(t ideaTemplate) []string {
	s := t.Value

	params := map[string]string{}
	match := rx.FindAllString(s, -1)
	varIdx := 1

MATCH:
	for _, v := range match {
		if _, ok := params[v]; ok {
			// already process the parameter
			continue
		}
		varname := strings.ReplaceAll(v, "$", "")

		for _, ideaVar := range t.Variable {
			if ideaVar.Name == varname {
				defaultValue := ideaVar.DefaultValue
				if defaultValue != "" {
					// process default
					replacement := fmt.Sprintf("${%d:another}", varIdx)
					varIdx++
					params[v] = replacement
					continue MATCH
				}

				// no variable found
				replacement := fmt.Sprintf("$%d", varIdx)
				varIdx++
				params[v] = replacement
			}
		}

	}

	// replace all parameters
	for varname, p := range params {
		s = strings.ReplaceAll(s, varname, p)
	}
	s = strings.ReplaceAll(s, "$END$", "")

	return strings.Split(s, "\n")
}

Copy the output of the file. On vscode, Ctrl+p to bring the command and search for Configure User Snippet. Chose the language and then paste it.

It’s all started when my colleague asked this question.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

package main

import "testing"

const size = 1000000

type SomeStruct struct {
	ID0 int64
	ID1 int64
	ID2 int64
	ID3 int64
	ID4 int64
	ID5 int64
	ID6 int64
	ID7 int64
	ID8 int64
}

func BenchmarkForVar(b *testing.B) {
    slice := make([]SomeStruct, size)
    b.ReportAllocs()
    b.ResetTimer()
    for i := 0; i < b.N; i++ {
        for _, s := range slice { // index and value
            _ = s
        }
    }
}
func BenchmarkForCounter(b *testing.B) {
    slice := make([]SomeStruct, size)
    b.ReportAllocs()
    b.ResetTimer()
    for i := 0; i < b.N; i++ {
        for i := range slice { // only use the index
            s := slice[i]
            _ = s
        }
    }
}

Which one do you think is faster?

$ go test -bench .
goos: linux
goarch: amd64
BenchmarkForVar-4       	    4363	    269711 ns/op	       0 B/op	       0 allocs/op
BenchmarkForCounter-4   	    4195	    285952 ns/op	       0 B/op	       0 allocs/op
PASS
ok  	_/test1	2.685s

It’s pretty much the same. However, if we increase the size of the struct by adding another field ID9 int64 The result becomes significantly different.

$ go test -bench .
goos: linux
goarch: amd64
BenchmarkForVar-4       	     282	   4264872 ns/op	       0 B/op	       0 allocs/op
BenchmarkForCounter-4   	    4363	    269761 ns/op	       0 B/op	       0 allocs/op
PASS
ok  	_/test1	3.255s

This problem become intriguing, so I dig a little deeper. I would like to point out that the code is a contrived example. It would probably not applied in a production code. I am not going to focus on the benchmark or which for-range works better but rather exploring how Go compiles and demonstrating useful Go tools in the process.

The code below also contains some assembly code. I am not going into too much detail, so I would not worry if you are not familiar with it. My intention is to show how the generated code differs and what could be the reason for it.

I am going to focus on for _, s = range slice loop (ForVar). To make it simpler I create a main.go and struct.go

main.go

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

package main

func main() {
	const size = 1000000

	slice := make([]SomeStruct, size)
	for _, s := range slice {
		_ = s
	}
}

I’ll skip the struct.go here since it’s pretty trivial.

Generating Assembly Code

To look into what instructions are generated, we can use go tool compile -S. This will print out the generated assembly code to stdout. I also Skipped some lines that are not directly related to our code.

according to asm package doc

The FUNCDATA and PCDATA directives contain information for use by the garbage collector; they are introduced by the compiler.

1

$  go tool compile -S main.go type.go | grep -v FUNCDATA | grep -v PCDATA

Here are the generated assembly code for both version.

version with 9 int64

This struct contains 9 int64 and has the size 72 bytes

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

"".main STEXT size=93 args=0x0 locals=0x28
    ...
	0x0024 00036 (main_var.go:6)	MOVQ	AX, (SP)
	0x0028 00040 (main_var.go:6)	MOVQ	$1000000, 8(SP)
	0x0031 00049 (main_var.go:6)	MOVQ	$1000000, 16(SP)
	0x003a 00058 (main_var.go:6)	CALL	runtime.makeslice(SB)
	0x003f 00063 (main_var.go:6)	XORL	AX, AX       # set AX = 0
	0x0041 00065 (main_var.go:7)	INCQ	AX           # AX++
	0x0044 00068 (main_var.go:7)	CMPQ	AX, $1000000 # AX < 1000000
	0x004a 00074 (main_var.go:7)	JLT	65               # JUMP to 00065 (next iteration)
    ...

Here you can see that at line 00065, the loop does nothing except increasing the counter. Let’s compare it with the larger struct.

The struct now contains 10 int64 and has the size 80 bytes

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

0x0000 00000 (main_var.go:3)	TEXT	"".main(SB), ABIInternal, $120-0
    ...
	0x0044 00068 (main_var.go:6)	XORL	CX, CX # CX = 0
	0x0046 00070 (main_var.go:7)	JMP	76
	0x0048 00072 (main_var.go:7)	ADDQ	$80, AX
	0x004c 00076 (main_var.go:7)	PCDATA	$0, $2 # setup temporary variable autotmp_7
	0x004c 00076 (main_var.go:7)	LEAQ	""..autotmp_7+32(SP), DI
	0x0051 00081 (main_var.go:7)	PCDATA	$0, $3
	0x0051 00081 (main_var.go:7)	MOVQ	AX, SI
	0x0054 00084 (main_var.go:7)	PCDATA	$0, $1
	0x0054 00084 (main_var.go:7)	DUFFCOPY	$826 # copy content of the struct
	0x0067 00103 (main_var.go:7)	INCQ	CX           # CX++
	0x006a 00106 (main_var.go:7)	CMPQ	CX, $1000000 # CX < 1000000
	0x0071 00113 (main_var.go:7)	JLT	72               # JUMP to 0072 (next iteration)
    ...

The conclusion is that with 80 bytes struct, every iteration copies the value of the element.

SSA Optimization

To understand how this code is generated, we first need to understand how GO compile a source code.

Stream wrote a very good introduction to the topic on How a Go Program Compiles down to Machine Code.
The short version is the compiler does:

1. break up the source code content into tokens (Scanning)
2. Construct Abstract Syntax Tree using the tokens (Parsing)
3. Generate Intermediate Representation (IR) with Static Single Assignment (SSA) form
4. Iteratively optimize the IR with multiple pass

1

$ GOSSAFUNC=main go tool compile -S main_var.go type_small.go

Each column represents the optimization pass, and the result of the IR code. When we click on a block, variable, or line. It will colorize the associated element, so we can track changes easily.

After comparing each step, I found out that the generated code was identical until the writebarrier pass. let’s focus on block b6

writebarrier (identical on both versions)

b6: ← b3 b4
    v22 (7) = Phi <*SomeStruct> v14 v45
    v28 (7) = Phi <int> v16 v37
    v23 (7) = Phi <mem> v12 v27
    v37 (+7) = Add64 <int> v28 v36
    v39 (7) = Less64 <bool> v37 v8
    v25 (7) = VarDef <mem> {.autotmp_7} v23
    v26 (7) = LocalAddr <*SomeStruct> {.autotmp_7} v2 v25
    v27 (+7) = Move <mem> {SomeStruct} [72] v26 v22 v25

If you see on v26 & v27, it does Move (or Copy) the content of the struct to local variable autotmp_7.

The lower pass basically convert the IR code into specific architecture low level code. Let’s have a look at the generated output. Don’t be afraid if you don’t really understand the assembly. What I wanted to show is the different code that was generated during the lower pass.

lower with 9 int64

b6: ← b3 b4

    v22 (7) = Phi <*SomeStruct> v14 v45
    v28 (7) = Phi <int> v16 v37
    v23 (7) = Phi <mem> v12 v27
    v37 (+7) = ADDQconst <int> [1] v28
    v25 (7) = VarDef <mem> {.autotmp_7} v23
    v26 (7) = LEAQ <*SomeStruct> {.autotmp_7} v2
    v44 (7) = CMPQconst <flags> [1000000] v37 
    v32 (+7) = LEAQ <*SomeStruct> {.autotmp_7} [8] v2
    v31 (+7) = ADDQconst <*SomeStruct> [8] v22
    v29 (+7) = MOVQload <uint64> v22 v25
    v24 (+7) = LEAQ <*SomeStruct> {.autotmp_7} [40] v2
    v15 (+7) = ADDQconst <*SomeStruct> [40] v22
    v46 (+7) = LEAQ <*SomeStruct> {.autotmp_7} [56] v2
    v35 (+7) = ADDQconst <*SomeStruct> [56] v22
    v21 (+7) = LEAQ <*SomeStruct> {.autotmp_7} [24] v2
    v17 (+7) = ADDQconst <*SomeStruct> [24] v22
    v39 (7) = SETL <bool> v44
    v42 (7) = TESTB <flags> v39 v39
    v30 (+7) = MOVQstore <mem> {.autotmp_7} v2 v29 v25  # <-- start translated Move instruction
    v41 (+7) = MOVOload <int128> [8] v22 v30
    v20 (+7) = MOVOstore <mem> {.autotmp_7} [8] v2 v41 v30
    v34 (+7) = MOVOload <int128> [24] v22 v20
    v19 (+7) = MOVOstore <mem> {.autotmp_7} [24] v2 v34 v20
    v33 (+7) = MOVOload <int128> [40] v22 v19
    v38 (+7) = MOVOstore <mem> {.autotmp_7} [40] v2 v33 v19
    v47 (+7) = MOVOload <int128> [56] v22 v38
    v27 (+7) = MOVOstore <mem> {.autotmp_7} [56] v2 v47 v38

b6: ← b3 b4

    v22 (7) = Phi <*SomeStruct> v14 v45
    v28 (7) = Phi <int> v16 v37
    v23 (7) = Phi <mem> v12 v27
    v37 (+7) = ADDQconst <int> [1] v28
    v25 (7) = VarDef <mem> {.autotmp_7} v23
    v26 (7) = LEAQ <*SomeStruct> {.autotmp_7} v2
    v44 (7) = CMPQconst <flags> [1000000] v37
    v32 (+7) = LEAQ <*SomeStruct> {.autotmp_7} [8] v2
    v31 (+7) = ADDQconst <*SomeStruct> [8] v22
    v29 (+7) = MOVQload <uint64> v22 v25
    v39 (7) = SETL <bool> v44
    v42 (7) = TESTB <flags> v39 v39
    v30 (+7) = MOVQstore <mem> {.autotmp_7} v2 v29 v25 # <-- start translated Move instruction
    v27 (+7) = DUFFCOPY <mem> [826] v32 v31 v30

LT v44 → b4 b2 (likely) (7)

The later version uses DUFFCOPY to perform the Move operation. This logic I believe is due to a rewrite rule rewriteAMD64.go. It’s an optimization to Move a large byte on memory.

// match: (Move [s] dst src mem)
// cond: s > 64 && s <= 16*64 && s%16 == 0 && !config.noDuffDevice
// result: (DUFFCOPY [14*(64-s/16)] dst src mem)

At a later SSA pass (elim unread autos) the compiler can detect that there are unused temporary variable for the first version (9 int64 struct). Thus, the Move instruction can be removed. This is not the case with for the `DUFFCOPY' version. That’s why the generated machine code is less optimized than the previous.

Note: A Duff Device is a loop optimization by splitting the task and reduce the number of loop.

Conclusion

for-range behaves differently depending on the struct size is due to Compiler SSA optimization. The compiler generated a different machine code for the larger struct where at a later pass it did not detect unused variable. The opposite happen for the smaller struct. At a later pass, it detected that some variables are un used. It removes the copy of element instruction on each iteration.

I’m so excited with the new release of golang. One particular feature is now very easy to build for multiple architecture. If you seen my other posts, I also like to tinker with my raspberry-pi. On my previous project I use either ruby or python for building some stuff. One annoying thing is dependency, setup and compilation is usually quite slow. Would be cool if I could just create some stuff in desktop and just scp the binary to pi and everything should work!

There is this nice article that explain the process. Let’s start there and create simple application

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

package main

import (
	"fmt"
	"runtime"
)

func main() {
	fmt.Printf("Hello. I'm running %s on %s architecture\n", runtime.GOOS, runtime.GOARCH)
}

Let’s try to build it

1
2
3
4
5

# create binary called 'main' for raspberry-pi (1gen)
$ env GOOS=linux GOARCH=arm GOARM=6 go build main.go

# transfer the binary to pi
$ scp main pi:

Now, crossing my finger, and run it on my pi

1
2
3
4
5

$ ssh pi

pi@raspbmc:~$ ./main 
Hello. I'm running linux on arm architecture
pi@raspbmc:~$ 

Holy crap, it’s that easy. I’m excited. Let’s see if go routine works

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

package main

import (
	"fmt"
	"sync"
)

func main() {
	var wg sync.WaitGroup
	nWorker := 2
	c := make(chan string)

	// write 10 message to channel c
	go func() {
		defer close(c)
		for i := 0; i < 10; i++ {
			c <- fmt.Sprintf("item sequence %d", i)
		}
	}()

	// create n worker to read from channel c
	for i := 0; i < nWorker; i++ {
		wg.Add(1)
		go func(worker int) {
			for msg := range c {
				fmt.Printf("worker %d: msg %s\n", worker, msg)
			}
			wg.Done()
		}(i)
	}

	wg.Wait()
	fmt.Println("That's super awesome!!, cee ya!")
}

Again, build and scp, and cross some more fingers

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

@raspbmc:~$ ls -alh main
-rwxr-xr-- 1 pi pi 1.9M Aug 29 18:08 main
pi@raspbmc:~$ ./main 
worker 1: msg item sequence 0
worker 0: msg item sequence 1
worker 1: msg item sequence 2
worker 0: msg item sequence 3
worker 0: msg item sequence 4
worker 0: msg item sequence 5
worker 0: msg item sequence 6
worker 0: msg item sequence 7
worker 0: msg item sequence 8
worker 1: msg item sequence 9
That's super awesome!!, cee ya!

Super awesome indeed, it just works!. Oh and the binary is not that big 1.9M considering it statically include the library.

Hmm let’s try something fun that fiddle with the gpio pin. Let’s create something that what raspberry-pi made for.. blinking led :p

There is already library. Let’s starts there.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

package main

import (
	"fmt"
	"os"
	"os/signal"
	"time"

	"github.com/davecheney/gpio"
	"github.com/davecheney/gpio/rpi"
)

func main() {
	// set GPIO25 to output mode
	pin, err := gpio.OpenPin(rpi.GPIO25, gpio.ModeOutput)
	if err != nil {
		fmt.Printf("Error opening pin! %s\n", err)
		return
	}

	// turn the led off on exit
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		for _ = range c {
			fmt.Printf("\nClearing and unexporting the pin.\n")
			pin.Clear()
			pin.Close()
			os.Exit(0)
		}
	}()

	dance(pin)
}

func dance(pin gpio.Pin) {
	for {
		pin.Set()
		time.Sleep(500 * time.Millisecond)
		pin.Clear()
		time.Sleep(500 * time.Millisecond)
	}
}

Dayuumm.. It’s that easy. No more pip install or bundle install on pi. Just scp the binary!

Of course we’ve got this far, we must try to control it remotely via HTTP.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

package main

import (
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"time"

	"github.com/davecheney/gpio"
	"github.com/davecheney/gpio/rpi"
)

// channel to control start blinking or not
var ctrlChan = make(chan bool)

func main() {
	// set GPIO25 to output mode
	pin, err := gpio.OpenPin(rpi.GPIO25, gpio.ModeOutput)
	if err != nil {
		fmt.Printf("Error opening pin! %s\n", err)
		return
	}

	// turn the led off on exit
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		for _ = range c {
			fmt.Printf("\nClearing and unexporting the pin.\n")
			pin.Clear()
			pin.Close()
			os.Exit(0)
		}
	}()

	go dance(pin, ctrlChan)
	ctrlChan <- true

	// http listen
	http.HandleFunc("/dance", danceHandler)
	http.ListenAndServe(":8080", nil)
}

func dance(pin gpio.Pin, ctrlChan chan bool) {
	enabled := false
	for {
		select {
		case val := <-ctrlChan:
      // got value from danceHandler via the channel
			fmt.Printf("dancing? %+v\n", val)
			enabled = val
		default:
			if enabled {
				pin.Set()
				time.Sleep(500 * time.Millisecond)
				pin.Clear()
				time.Sleep(500 * time.Millisecond)
			}
		}
	}
}

func danceHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Println("Received request")
	s := r.URL.Query().Get("s")
	ctrlChan <- s == "1" // tell dance to enable or not
}

Look at that. All it takes to do that is just 67 lines of code. The main logic just about 31 lines. That’s including the fancy dancing blinking. No framework, only gpio dependency. Everything else is go stdlib.

Ok, let’s do one other thing. The reverse! Trigger something when button is pressed. For this let’s just use websocket, and see how hard it is to implement this. What this mean that if you have websocket client (web browser) you could listen to event and stream it directly from your raspberry-pi to your computer.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

package main

import (
	"bufio"
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"time"

	"github.com/davecheney/gpio"
	"github.com/davecheney/gpio/rpi"
	"golang.org/x/net/websocket"
)

// channel to control start blinking or not
var ctrlChan = make(chan bool)

func main() {
	// set GPIO25 to output mode
	pin, err := gpio.OpenPin(rpi.GPIO25, gpio.ModeOutput)
	if err != nil {
		fmt.Printf("Error opening pin! %s\n", err)
		return
	}

	// turn the led off on exit
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		for _ = range c {
			fmt.Printf("\nClearing and unexporting the pin.\n")
			pin.Clear()
			pin.Close()
			os.Exit(0)
		}
	}()

	go buttonHandler(pin)

	// http listen
	http.Handle("/", websocket.Handler(EchoServer))
	http.ListenAndServe(":8080", nil)
}

// handle websocket connection
func EchoServer(ws *websocket.Conn) {
	w := bufio.NewWriter(ws)
	w.WriteString("Hello, i will tell you if button is pressed\n\n")
	w.Flush()

	for {
		<-ctrlChan
		w.WriteString("Kachhinggg... somebody pressed the button\n")
		w.Flush()
	}
}

// check if button is pressed
func buttonHandler(p gpio.Pin) {
	for {
		if p.Get() {
			ctrlChan <- true
		}
		time.Sleep(150 * time.Millisecond)
	}
}

You see there I connect 2 client. One browser and the other one is cli app that I created. Notice that they got their messages alternately between each other. This is because they are sharing the same channel. I will leave it to you my kind reader as an exercise to make it broadcast the message instead of distributing them :)

all source are available at https://github.com/yulrizka/go-pi-experiments

Push To Talk app for OSX

PushToTalk App installer

As a part of scrum teams, every day I need to give updates to my team via Google Hangout. We have a team here in the Netherlands and also in Indonesia. Some times I am in the same room as a colleague of mine. This sometimes quite annoying because I can hear my self (with a delay) from his mic. This somehow messed up my brain. Google Hangout already has a ‘auto adjust mic volume’ that is really great which cancel the noise. But we still have a problem and end-up muting each other when we want to talk.

There are currently an App that does this already on Apple store, and pretty cheap too. Nevertheless I could not find the one that is suitable for my needs. And I would like to develop an OSX app since I never done it before. Even though i have a hate-love relationship with XCode, I have to give credits to XCode. It’s quite easy to create an app like this.

Status bar indicator

The app sits on status-bar (tray icon ?). By default it muted the microphone and show translucent mic icon. If you hold down the Right Options key, it will un-mute the microphone as long as you pressed it. when you release the key, it will mute the microphone until you press the key again or quit the application.

Everything is at github.com/yulrizka/osx-push-to-talk:

• DMG Installer
• Source

I’ve only tested this app on my MacBook running Yosemite.

If you are on Linux, there is also a phython GTK app that does this really nicely.

If you are on windows, well Godspeed my friend :)

veel succes!

Disect the rocket, image by ProudloveNathan Proudlove

I’ve been involved with a iOS project this past week. I’m adding functionalities to the CommonSense iOS library. One of the most annoying thing is that it took about 2 minute to load the project. This is only happened in the unstable branch. The master branch seems to be working fine. So I knew that somewhere there is a commit when this starts happening.

Now this is a good example where git bisect is very useful. It will perform a binary search through commit history until the first bad commit found. So you start with a commit and you marked is as a ‘good’ or ‘bad’. Then every time you mark a commit as good / bad, it will then checkout another commit half way in the middle point between previous history. In each checkout then you test the code and see whether the bugs exist or not.

Here is an example

1
2
3

# let's start  bisecting, Im in unstable branch,
➜ sense-ios-library git:(unstable) $ git bisect start

the prompt sense-ios-library shows the current folder, and git:(unstable) show current commit. it’s part of oh-my-zsh plugin

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

# I open the xcode, and it took quite a long time. so let's marks current revision as bad
➜ sense-ios-library git:(unstable) $ git bisect bad

# I know the master branch don't have this problem, so let's mark it as good
➜ sense-ios-library git:(unstable) $ git bisect good master 

Bisecting: 9 revisions left to test after this (roughly 3 steps)
[c1841c0f99a4fa788c14c7437fac7c9547c768c9] Merge pull request #45 from senseobservationsystems/feature-KeepDataLocally

# now I'm in c1841 commit, I tested again with xcode, and found no problem, mark it as good

➜  sense2 git:(c1841c0) $ git bisect good

Bisecting: 4 revisions left to test after this (roughly 2 steps)
[06bbd23c4af8f7008abfb814e1375b271e0b23e9] Update interface and write tests lsit( APPEND CMAKE_CXX_FLAGS "-std=c++0x")

# tested again, it's good
➜  sense2 git:(06bbd23) $ git bisect good

Bisecting: 2 revisions left to test after this (roughly 1 step)
[71517b7923143a8cf1ce205341dd6942c9468198] Added threshold for checking to remove old data from local storage to save battery life

# Tested again, Now I'm beginning to see the problem. Let's mark this bad
➜  sense2 git:(71517b7) $ git bisect bad

Bisecting: 0 revisions left to test after this (roughly 0 steps)
[e5c40d450193ac14fc2faa8c1f4dba2c7c2646f1] Updated functionality and tests of getting local data

# .... 
# and we keep doing this until we find the first commit when it was introduced.

# Now let's check our progress using log

➜  sense-ios-library git:(71517b7) $ git bisect log

git bisect start
# bad: [af42213324297e1234767f9224ec1af326514292] Merge pull request #48 from senseobservationsystems/feature-LocalStorageInterface
git bisect bad af42213324297e1234767f9224ec1af326514292
# good: [19f751627d152b89d3f2ecadb144507fcf9293fc] Merge pull request #43 from senseobservationsystems/unstable
git bisect good 19f751627d152b89d3f2ecadb144507fcf9293fc
# good: [c1841c0f99a4fa788c14c7437fac7c9547c768c9] Merge pull request #45 from senseobservationsystems/feature-KeepDataLocally
git bisect good c1841c0f99a4fa788c14c7437fac7c9547c768c9
# good: [06bbd23c4af8f7008abfb814e1375b271e0b23e9] Update interface and write tests lsit( APPEND CMAKE_CXX_FLAGS "-std=c++0x")
git bisect good 06bbd23c4af8f7008abfb814e1375b271e0b23e9

# when you reach the final commit, you will be shown the commit log and modified file

➜  sense2 git:(e5c40d4) $ git bisect bad

e5c40d450193ac14fc2faa8c1f4dba2c7c2646f1 is the first bad commit
commit e5c40d450193ac14fc2faa8c1f4dba2c7c2646f1
Author: Jhon Doe <jhon-doe@sense-os.nl>
Date:   Thu Jan 8 14:38:07 2015 +0100

    Updated functionality and tests of getting local data

:040000 040000 0a82ee52c68bae4dacc1fec603ce0747b1df426c cd55743c27d2ba7554dfc3d021616b62957c4bba M  Sense Library Tests
:040000 040000 8a01944a75e80534124a7a657254e38002518f62 5d14fb11caa352acc7352c85e6a1c08794ae9e65 M  SensePlatform.xcodeproj
:040000 040000 a5e3eb78c4c352e7c010b8a4e90c12248f0ec9b0 2174d7ad531a7d989d313c826445434a073b433c M  SensePlatformTestAppTests
:040000 040000 4a40764a31981c172a089cd834eb9666b081f3f5 0713beb3c64f1b6f1d2a7811f59be4a82ec30d66 M  sense platform

Now that you know when was the bugs introduced. you can now start looking at the problem.

I found out there is something that doesn’t feel right here github:commit/e5c40. There is a reference to Xcode.app in one of the folder. So every time I open the project, or switch branch, it will try to look into things inside Xcode.app. So removing the reference indeed solve the problem.

The bisect function is very versatile tool to track down when a bug was introduced. You can even automate the test so you don’t have to check each bisect commit your self. The bisect documentation provide a good explanation about the command and also an example on how to automate the test.

Now go on catch and squash those annoying bug!

Sometimes it’s really hard to test functionality that involve with system time. Especially when we want to test the function with a specific time. For example testing whether today is end of month or test 2 different behavior at a different time

Below we look into different ways we can mock or stub the time. Each with it’s own advantages and disadvantages.

Passing the time instance

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

func CheckEndOfMonth(now time.Time) {
    // business process
}

func main() {
	CheckEndOfMonth(time.Now())
}

// for test
func TestCheckEndOfMonth(t *testing.T) {
	CheckEndOfMonth(time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC))
}

This way you can either pass time.Now() in your main pacakge or call it with something like time.Date(t.Year(), t.Month(), 1, 0, 0, 0, 0, time.UTC) in your test.

Advantages:

• Simple, no extra generator necessary
• Easy to test, no need to wire mock function or struct

Disadvantages:

• Caller need to pass instance of everywhere

Passing a function that generate time

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

func CheckEndOfMonth(now func() time.Time) {
	// ...
	x := now() // runs at this moment, not at the caller time
}

func main() {
	CheckEndOfMonth(time.Now) // instead of time.Now()
}

// for test
func TestCheckEndOfMonth(t *testing.T) {
	loc := time.UTC // closure can be used if necessary
	timeFn := func() time.Time {
		return time.Date(2019, 1, 1, 0, 0, 0, 0, loc)
	}
	CheckEndOfMonth(timeFn)
}

Advantages:

• This method allows you to have more control over how and when the time is generated. For example the time is generated inside the CheckEndOfMonth not at the caller.
• Allows the mock function to use closure inside the function

Disadvantages:

• Caller needs to pass function

Abstract the time generator as an interface

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

type Clock interface {
	Now() time.Time
}

type realClock struct {}
func (realClock) Now() time.Time { return time.Now() }

func CheckEndOfMonth(clock Clock) {
	// ...
	x := clock.Now()
}

func main() {
	CheckEndOfMonth(realClock{})
}

on the test code

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

type mockClock struct {
    t time.Time
}
func NewMockClock(t time.Time) mockClock {
    return mocktime{t}
}

func (m mockClock) Now() time.Time {
	return m.t
}

func TestCheckEndOfMonth(t *testing.T) {
	mc := NewMockClock(time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC))
	CheckEndOfMonth(mc)
}

Advantages:

• Control the generator method
• Interface can be defined small with only needed functions
• Easily switch to different implementation

Disadvantages:

• Caller need to pass instance that implements the interface

Package level time generator

Previous examples require you to pass in either concrete instance or a function on the caller.

Another approach you can use is to create a package level function to generate current time. You can change the implementation during test.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

package main

import "time"

type nowFuncT func() time.Time

var nowFunc nowFuncT

func init() {
    resetClockImplementation()
}

func resetClockImplementation() {
    nowFunc = func() time.Time {
        return time.Now()
    }
}

// function to return current time stamp in UTC
func now() time.Time {
    return nowFunc().UTC()
}

func CheckEndOfMonth() {
	// ...
	x := now()
}

now in your test you could do something like this:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

func TestCheckEndOfMonth(t *testing.T) {
    // change implementation of clock in the beginning of the test
    nowFunc = func() time.Time {
        return time.Date(2000, 12, 15, 17, 8, 00, 0, time.UTC)
    }

    // after finish with the test, reset the time implementation
    defer resetClockImplementation()
    
    CheckEndOfMonth()
    //...
}

Advantages:

• No need to pass instance of time or generator

Disadvantages:

• Caller need to pass instance
• Global variable
• Not thread safe
• Need to remember to call resetClockImplementation

Embed time generator in struct

If you are lucky enough to work in a struct, you can do this

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

// TimeValidator contains business logic to CheckEndOfMonth
type TimeValidator struct {
	// .. your fields
	clock func() time.Time
}

func (t TimeValidator) CheckEndOfMonth()  {
	x := t.now()
	// ...
}

// now is time generator that by default fall back to standard library
func (t TimeValidator) now() time.Time  {
	if t.clock == nil {
		return time.Now() // default implementation which fall back to standard library
	}

	return t.clock()
}

func main() {
	tv := TimeValidator{}
	tv.CheckEndOfMonth()
}

And the test can be

1
2
3
4
5
6
7

func TestCheckEndOfMonth(t *testing.T) {
	tv := TimeValidator{
		clock: func() time.Time {
			return time.Date(2000, 12, 15, 17, 8, 00, 0, time.UTC)
	}}
	tv.CheckEndOfMonth()
}

I like this method because you don’t have to pass instance of time or function all over the place but still have it easily create an arbitrary time mock/stub

Make this pattern reusable
If you do this more often, you can also easily reuse the functionality into other struct by creating embeddable

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

// TimeMock embeddable structure that provides ability to inject time to the parent struct
type TimeMock struct {
	NowFn func() time.Time

	// can be extended to mock other time function
	AfterFn func() <-chan time.Time
}

func (t TimeMock) Now() time.Time  {
	if t.NowFn == nil {
		return time.Now() // default implementation
	}

	return t.NowFn()
}

// Use in the business logic

// TimeValidator contains business logic that uses time.Now
type TimeValidator struct {
	clock TimeMock 
}

func (t TimeValidator) CheckEndOfMonth()  {
	x := t.clock.Now()
	// ...
}

func main () {
    tv := TimeValidator{} // empty clock field gives implementation with standard lib
    tv.CheckEndOfMonth()
}

in test

1
2
3
4
5
6
7
8
9

func TestCheckEndOfMonth(t *testing.T) {
	tv := TimeValidator{
		clock: TimeMock{
			NowFn: func() time.Time {
				return time.Date(2000, 12, 15, 17, 8, 00, 0, time.UTC)	
		},
	}}
	tv.CheckEndOfMonth()
}

Advantages:

• No need to pass instance of time or generator
• Fall back to default standard library time implementation
• Easily extend TimeMock by adding more function that is needed

Disadvantages:

• The caller needs to be a struct

Better way to write test function with time

Function with time is usually hard to test because it dependency with on time package.

A different way to test is to separate the function that generate the time with the function that process the time value. For example

1
2
3
4
5
6
7
8

func CheckEndOfMonth()  {
	now :=  time.Now()
	
	d := endOfMonth(now)
	if now == d {
		// do some business logic
	} 
}

Instead of testing that function, extract the logic of processing time

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

func CheckEndOfMonth()  {
	now :=  time.Now()

	processEndOfMonth(now)
}

func processEndOfMonth(t time.Time) {
	d := endOfMonth(t)
	if t == d {
		// do some business logic
	}
}

this way you don’t test the CheckEndOfMonth() but processEndOfMonth. This way you can easily mock time without the need to do some wiring.

You are probably familiar with above images. Some random friend sent you an email which you can instantly recognize as a spam because it only contains one link. Couple days ago I receive this email which is not the first time for me. But this time it was different. It actually came from my own Yahoo! account which I never use since more that one year ago.

My first instinct is to check whether the mail really got sent from my account or it just spoofing my email address. I kinda guess already that it is using my main account because the list of recipient is looks like it came out of my address book. So I went to my account and check the sent mail folder and there it is. One email message containing a spam that I actually sent to my friends and family.

Did some digging to my authentication history and found out that somebody from Romania has access my account. How could this be. I’ve never use the account since one or more year ago, I’m an IT guy so I know a little bit about security. I don’t click some random suspicious link. I don’t install any annoying-spyware-browser-toolbar. The password is 12 character long with apla-numeric and random symbol. Thank god that I don’t use the same password for all of my account.

I did again some digging to the given IP address and found out that the IP address came from a location in Rumania. But again this would not be his/her real IP. If i was to send a spam, I will route my email through bunch of proxy all over the world to cover my track. Or probably they don’t even bother because they just using it for spam.

Interesting part is I stumble upon this article saying about Romania has became a Global Hub for hackers and online crooks. According to the article that it’s became a commonplace for some hacker to harness people personal information and use it for illegal activity.

Luckily I don’t store any sensitive information on my email. It would have some serious impact if I store password or bank account information.

Also something that I notice is most of this spam email came from my friends account which also uses Yahoo! mail. Or probably I just never notice.

What can you do when this happened to you

If you ever received / sent this email. I would suggest you to:

1. Tell the person to reset their account password
2. Don’t store or send sensitive information unencrypted with email like: password, keys, bank account info
3. If you need to send sensitive information, try encrypting it first. I go into details of doing that in Safely sharing credentials with PGP. I’m start doing this from now on.
4. Use some secure password, and don’t use the same password for all of your account.
5. If you can, try to change the password on a regular basis. or use service like LastPass
6. When this is your work email that sometimes contain private information. Use service that offer two factor authentication like gmail does you need to enable it.
7. Stay safe friends, internet is a dangerous place

Berksfhel is cookbook dependency for chef. If you are familiar with ruby / python, think of it as a Bundler or virtual environment for chef

I faced this core dump error while doing berks upload. That command will actualy push some cookbook to a chef server.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

$ berks upload
/home/user/.rbenv/versions/1.9.3-p362/lib/ruby/gems/1.9.1/gems/celluloid-0.14.1/lib/celluloid/tasks.rb:47: [BUG] Segmentation fault
ruby 1.9.3p362 (2012-12-25 revision 38607) [x86_64-linux]

-- Control frame information -----------------------------------------------
c:0004 p:0112 s:0009 b:0007 l:002040 d:000006 BLOCK  /home/user/.rbenv/versions/1.9.3-p362/lib/ruby/gems/1.9.1/gems/celluloid-0.14.1/lib/celluloid/tasks.rb:47
c:0003 p:0031 s:0005 b:0005 l:002108 d:000004 BLOCK  /home/user/.rbenv/versions/1.9.3-p362/lib/ruby/gems/1.9.1/gems/celluloid-0.14.1/lib/celluloid/tasks/task_fiber.rb:11
c:0002 p:---- s:0003 b:0003 l:000002 d:000002 FINISH
c:0001 p:---- s:0001 b:-001 l:000000 d:000000 ------

-- Ruby level backtrace information ----------------------------------------
/home/user/.rbenv/versions/1.9.3-p362/lib/ruby/gems/1.9.1/gems/celluloid-0.14.1/lib/celluloid/tasks/task_fiber.rb:11:in `block in create'
/home/user/.rbenv/versions/1.9.3-p362/lib/ruby/gems/1.9.1/gems/celluloid-0.14.1/lib/celluloid/tasks.rb:47:in `block in initialize'

....

upgrading the ruby verison to 2.0 seems to resolve my issue.

When working in teams, we are sometimes required to share some password / keys with our team. The most common way for me is probably through email or some chat client. But even though its convenience it’s not actually a secure and a good practice. Especially if you are providing a service that deal with sensitive information.

Some simple approach would we communicating the password directly with a person through secure medium. One way to do it is both party ssh through a server and use talk client like write. But for some cases it’s quite impractical.

PGP

Enter PGP. It’s basically a software that do a Public-key cryptography. Public-key cryptography is basically encryption process which require 2 keys, one for encrypting and the other one for decrypting. Usually the public key is used for encrypting and private key is use for decrypting. I would not dive into the details about it since I’ve only have basic understanding about it. But for those people that is interested, you would read the nice article on wikipedia

There is a open source project called GPG (GNU Privacy Guard) and in this article I would like to show you how we could share some password / key file with it.

Instalation

If you don’t already have it on your system, you could installed it with:

1

  $ sudo apt-get install gpg

After installing we would start by creating a pair of keys

1

  $ gpg --gen-key

You will be asked with bunch of questions. Most of the answer you could leave it as a default but most important is fill in your email address and also provide a Passphrase to protect your key with password. When it finish gathering information, it will start creating a key pair by using system entropy. You can help the system to generate the entropy by clicking or moving your mouse randomly or doing some random IO disk by triggering for example find /

Listing keys

After installing, you can see list of keys by using this command

1
2
3
4
5
6
7

$  gpg --list-keys
/home/user/.gnupg/pubring.gpg
--------------------------------
pub   2048R/70280895 2013-07-09
uid                  Ahmy Yulrizka (ahmy135@mail.com) <ahmy135@mail.com>
sub   2048R/F7B2D44C 2013-07-09

In above output you could see that we have created public key with id of 70280895. Note this one because we are going to use it later when submitting the key to a key server

Exporting keys

To share your public key, so other people could send you encrypted message.

Note that further in this article I will discus ways to easily distribute your public key.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

$ gpg --armor --export 'ahmy135@mail.com'
-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: GnuPG v1.4.11 (GNU/Linux)

mQENBFHcRTEBCAC056qG97iJAtb604x5Hr+3lIi3UXVOnGauoHSo5S8S3bSCD0Ib
DzgSjWj8a6Xd1BY+5+HV0amp+i1sTknnd/C2WR7O1h9DIasPlWktPr2T+j4IGnYF
...
-----END PGP PUBLIC KEY BLOCK-----

$ gpg --armor --export 'ahmy135@mail.com' --output pubkey.txt # to output it to a file

Encrypting and decrypting

With those generated keys, we could now do a personal encryption. That is if you want to encrypt a file and you are the only one who are able to decrypt it.

1
2

  $ echo "this message is secret" > message.txt
  $ gpg --encrypt --recipient 'ahmy135@gmail.com' message.txt

Those code will create a file name message.txt.gpg which is encrypted message of message.txt

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

$ gpg --decrypt message.txt.gpg

You need a passphrase to unlock the secret key for
user: "Ahmy Yulrizka (ahmy135@mail.com) <ahmy135@mail.com>"
2048-bit RSA key, ID F7B2D44C, created 2013-07-09 (main key ID 70280895)

gpg: gpg-agent is not available in this session
gpg: encrypted with 2048-bit RSA key, ID F7B2D44C, created 2013-07-09
      "Ahmy Yulrizka (ahmy135@mail.com) <ahmy135@mail.com>"
this message is secret

As you can see that we are successfully decrypted the message. This example you encrypt the message using your own public key. So this method only work if you want to archive or backup the file securely. In order to send someone else an encrypted message, you need to encrypt the message using the other person public key

Distributing the key

In order for any body to send you a encrypted message, you need to give your public key. Since public key only used for encryption, It’s OK to publicly share your public key. But never share your private key. Once the other party have your public key they could start send you an encrypted message using the command above.

You could share your public key manually to some one (through usb / email etc) by exporting it first just like I mention before. But there are an easy way to distribute the key. There are some public GPG server that store your public key so that other people could easily find it and import it into their local machine. There are http://pgp.mit.edu and also ubuntu key server http://keyserver.ubuntu.com that we can use.

To send our key to MIT server we could do

1
2

$ gpg --send-key --keyserver pgp.mit.edu 70280895
gpg: sending key 70280895 to hkp server pgp.mit.edu

the last number 70280895 was the key id of the public file. You could find it with the output of gpg --list-keys command. Now we have successfully send our public key any body could get your public key through that keyserver. You could test this by searching a name or email or a person in the key server web interface. for example try searching my name on http://pgp.mit.edu/

Importing Keys

Now to import other people public key, we could also do that in two way.

if the person give you a file which contain their public key (say ahmy-pub.key). you could import it with

1

$ gpg --import ahmy-pub.key

Or if the person already publish his public key to a keyserver, we can search it with

1
2
3
4
5

$ gpg --search-keys 'Ahmy Yulrizka'

# or
$ gpg --search-keys 'ahmy135@mail.com'

It will generate a list of keys that found on the keyserver. Enter the number of the keys and it will be imported to your local machine.

after importing you can send an encrypted message to the person for example

1

$ echo "This is also a secret" | gpg --encrypt --armor --recipient 'Ahmy Yulrizka' > output.txt.gpg

You could provide a name or an email address as a recipient. THis command will encrypt the message using public key of a person name Ahmy Yulrizka

Conclusion

At this point you are able to generate, export, distribute and import keys. More over you can already encrypt and decrypt file / message to a designated recipient. The the part two of this article I will share some idea how we could share some password / password key to other member of the team.