fdtrack: temporary hack for tracking file descriptor usage

Package fdtrack logs statistics about open file descriptors.
This should help identify the source of #1549.

fdtrack/fdtrack.go

@@ -0,0 +1,112 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package fdtrack logs statistics about open file descriptors.
+package fdtrack
+
+import (
+	"fmt"
+	"net"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/logger/glog"
+)
+
+var (
+	mutex sync.Mutex
+	all   = make(map[string]int)
+)
+
+func Open(desc string) {
+	mutex.Lock()
+	all[desc] += 1
+	mutex.Unlock()
+}
+
+func Close(desc string) {
+	mutex.Lock()
+	defer mutex.Unlock()
+	if c, ok := all[desc]; ok {
+		if c == 1 {
+			delete(all, desc)
+		} else {
+			all[desc]--
+		}
+	}
+}
+
+func WrapListener(desc string, l net.Listener) net.Listener {
+	Open(desc)
+	return &wrappedListener{l, desc}
+}
+
+type wrappedListener struct {
+	net.Listener
+	desc string
+}
+
+func (w *wrappedListener) Accept() (net.Conn, error) {
+	c, err := w.Listener.Accept()
+	if err == nil {
+		c = WrapConn(w.desc, c)
+	}
+	return c, err
+}
+
+func (w *wrappedListener) Close() error {
+	err := w.Listener.Close()
+	if err == nil {
+		Close(w.desc)
+	}
+	return err
+}
+
+func WrapConn(desc string, conn net.Conn) net.Conn {
+	Open(desc)
+	return &wrappedConn{conn, desc}
+}
+
+type wrappedConn struct {
+	net.Conn
+	desc string
+}
+
+func (w *wrappedConn) Close() error {
+	err := w.Conn.Close()
+	if err == nil {
+		Close(w.desc)
+	}
+	return err
+}
+
+func Start() {
+	go func() {
+		for range time.Tick(15 * time.Second) {
+			mutex.Lock()
+			var sum, tracked = 0, []string{}
+			for what, n := range all {
+				sum += n
+				tracked = append(tracked, fmt.Sprintf("%s:%d", what, n))
+			}
+			mutex.Unlock()
+			used, _ := fdusage()
+			sort.Strings(tracked)
+			glog.Infof("fd usage %d/%d, tracked %d %v", used, fdlimit(), sum, tracked)
+		}
+	}()
+}

fdtrack/fdusage.go

@@ -0,0 +1,29 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// +build !linux,!darwin
+
+package fdtrack
+
+import "errors"
+
+func fdlimit() int {
+	return 0
+}
+
+func fdusage() (int, error) {
+	return 0, errors.New("not implemented")
+}

fdtrack/fdusage_darwin.go

@@ -0,0 +1,72 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// +build darwin
+
+package fdtrack
+
+import (
+	"os"
+	"syscall"
+	"unsafe"
+)
+
+// #cgo CFLAGS: -lproc
+// #include <libproc.h>
+// #include <stdlib.h>
+import "C"
+
+func fdlimit() int {
+	var nofile syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &nofile); err != nil {
+		return 0
+	}
+	return int(nofile.Cur)
+}
+
+func fdusage() (int, error) {
+	pid := C.int(os.Getpid())
+	// Query for a rough estimate on the amout of data that
+	// proc_pidinfo will return.
+	rlen, err := C.proc_pidinfo(pid, C.PROC_PIDLISTFDS, 0, nil, 0)
+	if rlen <= 0 {
+		return 0, err
+	}
+	// Load the list of file descriptors. We don't actually care about
+	// the content, only about the size. Since the number of fds can
+	// change while we're reading them, the loop enlarges the buffer
+	// until proc_pidinfo says the result fitted.
+	var buf unsafe.Pointer
+	defer func() {
+		if buf != nil {
+			C.free(buf)
+		}
+	}()
+	for buflen := rlen; ; buflen *= 2 {
+		buf, err = C.reallocf(buf, C.size_t(buflen))
+		if buf == nil {
+			return 0, err
+		}
+		rlen, err = C.proc_pidinfo(pid, C.PROC_PIDLISTFDS, 0, buf, buflen)
+		if rlen <= 0 {
+			return 0, err
+		} else if rlen == buflen {
+			continue
+		}
+		return int(rlen / C.PROC_PIDLISTFD_SIZE), nil
+	}
+	panic("unreachable")
+}

fdtrack/fdusage_linux.go

@@ -0,0 +1,53 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// +build linux
+
+package fdtrack
+
+import (
+	"io"
+	"os"
+	"syscall"
+)
+
+func fdlimit() int {
+	var nofile syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &nofile); err != nil {
+		return 0
+	}
+	return int(nofile.Cur)
+}
+
+func fdusage() (int, error) {
+	f, err := os.Open("/proc/self/fd")
+	if err != nil {
+		return 0, err
+	}
+	defer f.Close()
+	const batchSize = 100
+	n := 0
+	for {
+		list, err := f.Readdirnames(batchSize)
+		n += len(list)
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return 0, err
+		}
+	}
+	return n, nil
+}