diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..ebd4f0c --- /dev/null +++ b/go.mod @@ -0,0 +1 @@ +module github.com/pyke369/golang-support/uconfig diff --git a/httpd.go b/httpd.go index 7fd654d..318b99c 100644 --- a/httpd.go +++ b/httpd.go @@ -1,15 +1,24 @@ package main import ( + "bufio" + "bytes" + "crypto/tls" + "crypto/x509" "encoding/json" "fmt" + "io" "log" "net/http" + "os" ) type HttpServer struct { - Port string - ovpn *OpenVpnMgt + Port string + ovpn *OpenVpnMgt + key string + cert string + certPool *x509.CertPool } func (h *HttpServer) handler(w http.ResponseWriter, r *http.Request) { @@ -33,19 +42,51 @@ func (h *HttpServer) helpHandler(w http.ResponseWriter, r *http.Request) { err, message := h.ovpn.Help() if err != nil { fmt.Fprintf(w, "Error : %s", err) - } else { - fmt.Fprintf(w, "%s", message) } - + jsonStr, err := json.Marshal(message) + if err != nil { + fmt.Fprintf(w, "Error : %s", err) + } + fmt.Fprintf(w, "%s", jsonStr) } -func NewHTTPServer(port string, s *OpenVpnMgt) { +func NewHTTPServer(port, key, cert, ca string, s *OpenVpnMgt) { h := &HttpServer{ Port: port, ovpn: s, + key: key, + cert: cert, } + http.HandleFunc("/help", h.helpHandler) http.HandleFunc("/version", h.versionHandler) http.HandleFunc("/", h.handler) - log.Fatal(http.ListenAndServe(port, nil)) + + switch { + case key == "" || cert == "": + log.Fatal(http.ListenAndServe(port, nil)) + case ca != "": + h.certPool = x509.NewCertPool() + fi, err := os.Open(ca) + if err != nil { + log.Fatal(err) + } + defer fi.Close() + buf := new(bytes.Buffer) + reader := bufio.NewReader(fi) + io.Copy(buf, reader) + if ok := h.certPool.AppendCertsFromPEM(buf.Bytes()); !ok { + log.Fatal("Failed to append PEM.") + } + server := &http.Server{ + Addr: port, + TLSConfig: &tls.Config{ + ClientAuth: tls.RequestClientCert, + ClientCAs: h.certPool, + }, + } + log.Fatal(server.ListenAndServeTLS(cert, key)) + default: + log.Fatal(http.ListenAndServeTLS(port, cert, key, nil)) + } } diff --git a/main.go b/main.go index 1315a50..6ed16f2 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,57 @@ package main +import ( + "flag" + "log" + "log/syslog" + "os" + + "github.com/pyke369/golang-support/uconfig" +) + +var config *uconfig.UConfig + func main() { - server := NewVPNServer(":4000") + var err error + // default configuration file is ./openvpn-dm-mgt-server.conf + configFile := flag.String("config", "openvpn-dm-mgt-server.conf", "configuration file") + logToSyslog := flag.Bool("syslog", false, "Log to syslog") + flag.Parse() + + // parseconfig + if config, err = uconfig.New(*configFile); err != nil { + log.Println(err) + os.Exit(1) + } + + server := NewVPNServer(config.GetString("config.openvpnPort", "127.0.0.01:5000")) + server.vpnlogUrl = config.GetString("config.vpnLogUrl", "") + server.mailRelay = config.GetString("config.mailRelay", "") + server.MailFrom = config.GetString("config.mailFrom", "") + server.CcPwnPassword = config.GetString("config.ccPwnPassword", "") + server.pwnTemplate = config.GetString("config.pwnTemplate", "") + server.newAsTemplate = config.GetString("config.newAsTemplate", "") + server.slackTemplate = config.GetString("config.slackTemplate", "") + server.slackTemplate2 = config.GetString("config.slackTemplate2", "") + server.cacheDir = config.GetString("config.cacheDir", "") + server.authCa = config.GetString("config.authCa", "") + + server.syslog = false + if *logToSyslog { + log.SetFlags(0) + server.syslog = true + logWriter, e := syslog.New(syslog.LOG_NOTICE, "") + if e == nil { + log.SetOutput(logWriter) + defer logWriter.Close() + } + } + go server.Run() - NewHTTPServer(":8080", server) + NewHTTPServer( + config.GetString("config.httpPort", "127.0.0.01:8080"), + config.GetString("config.httpKey", ""), + config.GetString("config.httpCert", ""), + config.GetString("config.httpCa", ""), + server) } diff --git a/tcpserver.go b/openvpn.go similarity index 89% rename from tcpserver.go rename to openvpn.go index 29b2234..f05eac0 100644 --- a/tcpserver.go +++ b/openvpn.go @@ -3,28 +3,32 @@ package main import ( "bufio" "errors" - "fmt" "io" "log" "net" + "os" "strings" "sync" ) // Server represents the server type OpenVpnMgt struct { - Port string - buf *bufio.ReadWriter - connected bool - m sync.RWMutex - ret chan []string - vpnlogUrl string - mailRelay string - MailFrom string - CcPwnPassword string - pwnTemplate string - newAsTemplate string - syslog bool + Port string + buf *bufio.ReadWriter + connected bool + m sync.RWMutex + ret chan []string + authCa string + vpnlogUrl string + mailRelay string + MailFrom string + CcPwnPassword string + pwnTemplate string + newAsTemplate string + slackTemplate string + slackTemplate2 string + cacheDir string + syslog bool } // NewServer returns a pointer to a new server @@ -40,12 +44,14 @@ func (s *OpenVpnMgt) Run() { // Resolve the passed port into an address addrs, err := net.ResolveTCPAddr("tcp", s.Port) if err != nil { - return + log.Println(err) + os.Exit(1) } // start listening to client connections listener, err := net.ListenTCP("tcp", addrs) if err != nil { - fmt.Println(err) + log.Println(err) + os.Exit(1) } // Infinite loop since we dont want the server to shut down for { diff --git a/vendor/github.com/pyke369/golang-support/README.md b/vendor/github.com/pyke369/golang-support/README.md new file mode 100644 index 0000000..e69de29 diff --git a/vendor/github.com/pyke369/golang-support/bslab/bslab.go b/vendor/github.com/pyke369/golang-support/bslab/bslab.go new file mode 100644 index 0000000..4b59742 --- /dev/null +++ b/vendor/github.com/pyke369/golang-support/bslab/bslab.go @@ -0,0 +1,89 @@ +package bslab + +import "sync/atomic" + +type slab struct { + queue chan []byte + get, put, alloc, lost int64 +} + +var ( + slabs = map[int]*slab{} +) + +func init() { + slabs[0] = &slab{} + for size := uint(9); size <= 26; size++ { + slabs[1<= size { + return item[:0] + } + Put(item) + } + bits, power := uint(0), uint(0) + if size&(size-1) == 0 { + power = 1 + } + for size != 0 { + size >>= 1 + bits++ + } + size = 1 << (bits - power) + if slab, ok := slabs[size]; ok { + atomic.AddInt64(&(slab.get), 1) + select { + case item := <-slab.queue: + return item[:0] + default: + atomic.AddInt64(&(slab.alloc), 1) + return make([]byte, 0, size) + } + } + atomic.AddInt64(&(slabs[0].get), 1) + atomic.AddInt64(&(slabs[0].alloc), int64(size)) + return make([]byte, 0, size) +} + +func Put(item []byte) { + if item == nil || cap(item) <= 0 { + return + } + size, bits := cap(item), uint(0) + for size != 0 { + size >>= 1 + bits++ + } + size = 1 << (bits - 1) + if size > 0 && float64(cap(item))/float64(size) <= 1.2 { + if slab, ok := slabs[size]; ok { + atomic.AddInt64(&(slab.put), 1) + select { + case slab.queue <- item: + default: + atomic.AddInt64(&(slab.lost), 1) + } + } else { + atomic.AddInt64(&(slabs[0].put), 1) + atomic.AddInt64(&(slabs[0].lost), int64(cap(item))) + } + } else { + atomic.AddInt64(&(slabs[0].put), 1) + atomic.AddInt64(&(slabs[0].lost), int64(cap(item))) + } +} diff --git a/vendor/github.com/pyke369/golang-support/chash/README.md b/vendor/github.com/pyke369/golang-support/chash/README.md new file mode 100644 index 0000000..e69de29 diff --git a/vendor/github.com/pyke369/golang-support/chash/chash.go b/vendor/github.com/pyke369/golang-support/chash/chash.go new file mode 100644 index 0000000..604fc65 --- /dev/null +++ b/vendor/github.com/pyke369/golang-support/chash/chash.go @@ -0,0 +1,356 @@ +package chash + +import ( + "math/rand" + "os" + "runtime" + "sort" + "strconv" + "sync" + "time" +) + +const ( + CHASH_MAGIC uint32 = 0x48414843 + CHASH_REPLICAS = 128 +) + +type item struct { + hash uint32 + target uint16 +} + +type CHash struct { + targets map[string]uint8 + names []string + ring []item + ringSize uint32 + replicas uint8 + frozen bool + sync.RWMutex +} + +var cores int + +func init() { + rand.Seed(time.Now().UnixNano() + int64(os.Getpid())) +} + +func mmhash2(key []byte, keySize int) uint32 { + var magic, hash, current, value uint32 = 0x5bd1e995, uint32(0x4d4d4832 ^ keySize), 0, 0 + + if keySize < 0 { + keySize = len(key) + } + for keySize >= 4 { + value = uint32(key[current]) | uint32(key[current+1])<<8 | + uint32(key[current+2])<<16 | uint32(key[current+3])<<24 + value *= magic + value ^= value >> 24 + value *= magic + hash *= magic + hash ^= value + current += 4 + keySize -= 4 + } + if keySize >= 3 { + hash ^= uint32(key[current+2]) << 16 + } + if keySize >= 2 { + hash ^= uint32(key[current+1]) << 8 + } + if keySize >= 1 { + hash ^= uint32(key[current]) + } + if keySize != 0 { + hash *= magic + } + hash ^= hash >> 13 + hash *= magic + hash ^= hash >> 15 + return hash +} + +type ByHash []item + +func (a ByHash) Len() int { return len(a) } +func (a ByHash) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a ByHash) Less(i, j int) bool { return a[i].hash < a[j].hash } + +func (this *CHash) freeze() { + if cores > 1 { + this.Lock() + defer this.Unlock() + } + if this.frozen { + return + } + this.ringSize = 0 + for _, tweight := range this.targets { + this.ringSize += uint32(tweight) * uint32(this.replicas) + } + if this.ringSize == 0 { + this.frozen = true + return + } + + var ( + target uint16 = 0 + offset uint32 = 0 + key []byte = make([]byte, 128) + ) + this.names = make([]string, len(this.targets)) + this.ring = make([]item, this.ringSize) + for tname, tweight := range this.targets { + this.names[target] = tname + for weight := uint8(0); weight < tweight; weight++ { + for replica := uint8(0); replica < this.replicas; replica++ { + key = append(key[:0], tname...) + key = strconv.AppendInt(key, int64(weight), 10) + key = strconv.AppendInt(key, int64(replica), 10) + this.ring[offset] = item{mmhash2(key, -1), target} + offset++ + } + } + target++ + } + sort.Sort(ByHash(this.ring)) + this.frozen = true +} + +func New(replicas ...uint8) *CHash { + if cores == 0 { + cores = runtime.NumCPU() + } + chash := &CHash{ + targets: make(map[string]uint8), + names: nil, + ring: nil, + ringSize: 0, + replicas: CHASH_REPLICAS, + frozen: false, + } + if len(replicas) > 0 { + chash.replicas = replicas[0] + } + if chash.replicas < 1 { + chash.replicas = 1 + } + if chash.replicas > CHASH_REPLICAS { + chash.replicas = CHASH_REPLICAS + } + return chash +} + +func (this *CHash) AddTarget(name string, weight uint8) bool { + if weight > 0 && weight <= 100 && len(name) <= 128 && this.targets[name] != weight { + if cores > 1 { + this.Lock() + defer this.Unlock() + } + this.targets[name] = weight + this.frozen = false + return true + } + return false +} +func (this *CHash) RemoveTarget(name string) bool { + if cores > 1 { + this.Lock() + defer this.Unlock() + } + delete(this.targets, name) + this.frozen = false + return true +} +func (this *CHash) ClearTargets() bool { + if cores > 1 { + this.Lock() + defer this.Unlock() + } + this.targets = make(map[string]uint8) + this.frozen = false + return true +} + +func (this *CHash) Serialize() []byte { + this.freeze() + if cores > 1 { + this.RLock() + defer this.RUnlock() + } + size := uint32(4) + 4 + 1 + 2 + 4 + for _, name := range this.names { + size += 1 + 1 + uint32(len(name)) + } + size += (this.ringSize * 6) + serialized := make([]byte, size) + offset := uint32(0) + serialized[offset] = byte(CHASH_MAGIC & 0xff) + serialized[offset+1] = byte((CHASH_MAGIC >> 8) & 0xff) + serialized[offset+2] = byte((CHASH_MAGIC >> 16) & 0xff) + serialized[offset+3] = byte((CHASH_MAGIC >> 24) & 0xff) + serialized[offset+4] = byte(size & 0xff) + serialized[offset+5] = byte((size >> 8) & 0xff) + serialized[offset+6] = byte((size >> 16) & 0xff) + serialized[offset+7] = byte((size >> 24) & 0xff) + serialized[offset+8] = this.replicas + serialized[offset+9] = byte(uint16(len(this.names)) & 0xff) + serialized[offset+10] = byte(((uint16(len(this.names))) >> 8) & 0xff) + serialized[offset+11] = byte(this.ringSize & 0xff) + serialized[offset+12] = byte((this.ringSize >> 8) & 0xff) + serialized[offset+13] = byte((this.ringSize >> 16) & 0xff) + serialized[offset+14] = byte((this.ringSize >> 24) & 0xff) + offset += 15 + for _, name := range this.names { + serialized[offset] = this.targets[name] + serialized[offset+1] = byte(len(name) & 0xff) + copy(serialized[offset+2:offset+2+uint32(serialized[offset+1])], []byte(name)) + offset += 2 + uint32(serialized[offset+1]) + } + for _, item := range this.ring { + serialized[offset] = byte(item.hash & 0xff) + serialized[offset+1] = byte((item.hash >> 8) & 0xff) + serialized[offset+2] = byte((item.hash >> 16) & 0xff) + serialized[offset+3] = byte((item.hash >> 24) & 0xff) + serialized[offset+4] = byte(item.target & 0xff) + serialized[offset+5] = byte((item.target >> 8) & 0xff) + offset += 6 + } + return serialized +} +func (this *CHash) FileSerialize(path string) bool { + handle, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + return false + } + defer handle.Close() + if _, err := handle.Write(this.Serialize()); err != nil { + return false + } + return true +} + +func (this *CHash) Unserialize(serialized []byte) bool { + if cores > 1 { + this.Lock() + defer this.Unlock() + } + if len(serialized) < 15 { + return false + } + magic := uint32(serialized[0]) + (uint32(serialized[1]) << 8) + (uint32(serialized[2]) << 16) + (uint32(serialized[3]) << 24) + size := uint32(serialized[4]) + (uint32(serialized[5]) << 8) + (uint32(serialized[6]) << 16) + (uint32(serialized[7]) << 24) + replicas := serialized[8] + names := uint16(serialized[9]) + (uint16(serialized[10]) << 8) + ringSize := uint32(serialized[11]) + (uint32(serialized[12]) << 8) + (uint32(serialized[13]) << 16) + (uint32(serialized[14]) << 24) + if magic != CHASH_MAGIC || size != uint32(len(serialized)) { + return false + } + this.targets = make(map[string]uint8) + this.names = make([]string, names) + this.ring = make([]item, ringSize) + this.ringSize = ringSize + this.replicas = replicas + offset := uint32(15) + for index := uint16(0); index < names && offset < size; index++ { + len := uint32(serialized[offset+1]) + this.names[index] = string(serialized[offset+2 : offset+2+len]) + this.targets[this.names[index]] = serialized[offset] + offset += 2 + len + } + if offset > size { + return false + } + for item := uint32(0); item < ringSize && offset < size; item++ { + this.ring[item].hash = uint32(serialized[offset]) + (uint32(serialized[offset+1]) << 8) + (uint32(serialized[offset+2]) << 16) + (uint32(serialized[offset+3]) << 24) + this.ring[item].target = uint16(serialized[offset+4]) + (uint16(serialized[offset+5]) << 8) + offset += 6 + } + if offset != size { + return false + } + this.frozen = true + return true +} +func (this *CHash) FileUnserialize(path string) bool { + handle, err := os.OpenFile(path, os.O_RDONLY, 0) + if err != nil { + return false + } + defer handle.Close() + info, err := handle.Stat() + if err != nil { + return false + } + if info.Size() > 128*1024*1024 { + return false + } + serialized := make([]byte, info.Size()) + read, err := handle.Read(serialized) + if int64(read) != info.Size() || err != nil { + return false + } + return this.Unserialize(serialized) +} + +func (this *CHash) Lookup(candidate string, count int) []string { + var start uint32 = 0 + + this.freeze() + if cores > 1 { + this.RLock() + defer this.RUnlock() + } + if count > len(this.targets) { + count = len(this.targets) + } + if this.ringSize == 0 || count < 1 { + return []string{} + } + hash := mmhash2([]byte(candidate), -1) + if hash > this.ring[0].hash && hash <= this.ring[this.ringSize-1].hash { + start = this.ringSize / 2 + span := start / 2 + for { + if hash > this.ring[start].hash && hash <= this.ring[start+1].hash { + break + } + if hash > this.ring[start].hash { + start += span + } else { + start -= span + } + span /= 2 + if span < 1 { + span = 1 + } + } + } + result := make([]string, count) + rank := 0 + for rank < count { + index := 0 + for index = 0; index < rank; index++ { + if result[index] == this.names[this.ring[start].target] { + break + } + } + if index >= rank { + result[rank] = this.names[this.ring[start].target] + rank++ + } + start++ + if start >= this.ringSize { + start = 0 + } + } + return result +} +func (this *CHash) LookupBalance(candidate string, count int) string { + result := this.Lookup(candidate, count) + if len(result) > 0 { + return result[rand.Intn(len(result))] + } + return "" +} diff --git a/vendor/github.com/pyke369/golang-support/dynacert/dynacert.go b/vendor/github.com/pyke369/golang-support/dynacert/dynacert.go new file mode 100644 index 0000000..801dca2 --- /dev/null +++ b/vendor/github.com/pyke369/golang-support/dynacert/dynacert.go @@ -0,0 +1,54 @@ +package dynacert + +import ( + "crypto/tls" + "os" + "runtime" + "sync" + "time" +) + +type DYNACERT struct { + Public, Key string + Certificate *tls.Certificate + Last, Modified time.Time + sync.RWMutex +} + +var cores int + +func (this *DYNACERT) GetCertificate(*tls.ClientHelloInfo) (cert *tls.Certificate, err error) { + var info os.FileInfo + + if cores == 0 { + cores = runtime.NumCPU() + } + if this.Certificate == nil || time.Now().Sub(this.Last) >= 10*time.Second { + this.Last = time.Now() + if info, err = os.Stat(this.Public); err != nil { + return nil, err + } + if _, err = os.Stat(this.Key); err != nil { + return nil, err + } + if this.Certificate == nil || info.ModTime().Sub(this.Modified) != 0 { + if certificate, err := tls.LoadX509KeyPair(this.Public, this.Key); err != nil { + return nil, err + } else { + if cores > 1 { + this.Lock() + } + this.Modified = info.ModTime() + this.Certificate = &certificate + if cores > 1 { + this.Unlock() + } + } + } + } + if cores > 1 { + this.RLock() + defer this.RUnlock() + } + return this.Certificate, nil +} diff --git a/vendor/github.com/pyke369/golang-support/fqdn/README.md b/vendor/github.com/pyke369/golang-support/fqdn/README.md new file mode 100644 index 0000000..e69de29 diff --git a/vendor/github.com/pyke369/golang-support/fqdn/fqdn.go b/vendor/github.com/pyke369/golang-support/fqdn/fqdn.go new file mode 100644 index 0000000..c761d66 --- /dev/null +++ b/vendor/github.com/pyke369/golang-support/fqdn/fqdn.go @@ -0,0 +1,28 @@ +package fqdn + +import ( + "net" + "os" + "strings" +) + +func FQDN() (string, string) { + if hostname, err := os.Hostname(); err == nil { + if addresses, err := net.LookupHost(hostname); err != nil { + return hostname, "*" + } else { + for _, address := range addresses { + if hostnames, err := net.LookupAddr(address); err == nil && len(hostnames) > 0 { + for _, hostname := range hostnames { + if strings.Count(hostname, ".") > 1 { + hostname = strings.TrimSuffix(hostname, ".") + addresses, _ = net.LookupHost(hostname) + return hostname, addresses[0] + } + } + } + } + } + } + return "unknown", "*" +} diff --git a/vendor/github.com/pyke369/golang-support/listener/listener.go b/vendor/github.com/pyke369/golang-support/listener/listener.go new file mode 100644 index 0000000..343257f --- /dev/null +++ b/vendor/github.com/pyke369/golang-support/listener/listener.go @@ -0,0 +1,60 @@ +package listener + +import ( + "context" + "net" + "syscall" + + "golang.org/x/sys/unix" +) + +type TCPListener struct { + *net.TCPListener + ReadBufferSize int + WriteBufferSize int +} + +func (this *TCPListener) Accept() (net.Conn, error) { + connection, err := this.AcceptTCP() + if err != nil { + return nil, err + } + if rconnection, err := connection.SyscallConn(); err == nil { + rconnection.Control( + func(handle uintptr) { + syscall.SetsockoptInt(int(handle), syscall.SOL_SOCKET, syscall.SO_KEEPALIVE, 1) + syscall.SetsockoptInt(int(handle), syscall.IPPROTO_TCP, syscall.TCP_KEEPIDLE, 60) + syscall.SetsockoptInt(int(handle), syscall.IPPROTO_TCP, syscall.TCP_KEEPINTVL, 10) + syscall.SetsockoptInt(int(handle), syscall.IPPROTO_TCP, syscall.TCP_KEEPCNT, 3) + }) + } + if this.ReadBufferSize > 0 { + connection.SetReadBuffer(this.ReadBufferSize) + } + if this.WriteBufferSize > 0 { + connection.SetWriteBuffer(this.WriteBufferSize) + } + return connection, nil +} + +func NewTCPListener(network, address string, reuseport bool, read, write int) (listener *TCPListener, err error) { + config := net.ListenConfig{ + Control: func(network, address string, connection syscall.RawConn) error { + connection.Control(func(handle uintptr) { + if err := syscall.SetsockoptInt(int(handle), syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1); err != nil { + return + } + if reuseport { + if err := syscall.SetsockoptInt(int(handle), syscall.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil { + return + } + } + }) + return nil + }} + if clistener, err := config.Listen(context.Background(), network, address); err != nil { + return nil, err + } else { + return &TCPListener{clistener.(*net.TCPListener), read, write}, nil + } +} diff --git a/vendor/github.com/pyke369/golang-support/prefixdb/README.md b/vendor/github.com/pyke369/golang-support/prefixdb/README.md new file mode 100644 index 0000000..d29a76b --- /dev/null +++ b/vendor/github.com/pyke369/golang-support/prefixdb/README.md @@ -0,0 +1,34 @@ +PFDB binary layout +------------------ +"PFDB" +VVVV (4) +HHHHHHHHHHHHHHHH (16) +"DESC" +"DDDDDDDDDDDDDDDDDDDD" (20) +"STRS" +W (1) +SSSS (4) +CCCC (4) +... +"NUMS" +SSSS (4) +CCCC (4) +... +"PAIR" +SSSS (4) +CCCC (4) +... +"CLUS" +W (1) +SSSS (4) +CCCC (4) +... +"MAPS" +SSSS (4) +CCCC (4) +... +"NODE" +W (1) +SSSS (4) +CCCC (4) +... diff --git a/vendor/github.com/pyke369/golang-support/prefixdb/cmd/Makefile b/vendor/github.com/pyke369/golang-support/prefixdb/cmd/Makefile new file mode 100644 index 0000000..b4d8d5c --- /dev/null +++ b/vendor/github.com/pyke369/golang-support/prefixdb/cmd/Makefile @@ -0,0 +1,11 @@ +prefixdb: prefixdb.go + @go build prefixdb.go && strip prefixdb + +run: prefixdb + @#./prefixdb city city.pfdb@'MMCITY 20190402' GeoIP2-City-Locations-en.csv GeoIP2-City-Blocks-IPv4.csv GeoIP2-City-Blocks-IPv6.csv + @#./prefixdb asn asn.pfdb@'MMASN 20190402' GeoLite2-ASN-Blocks-IPv4.csv GeoLite2-ASN-Blocks-IPv6.csv + @./prefixdb lookup city.pfdb asn.pfdb 78.193.67.63 188.65.124.26 + @#./prefixdb server *:8000 city.pfdb asn.pfdb + +clean: + @rm -f prefixdb *.pfdb diff --git a/vendor/github.com/pyke369/golang-support/prefixdb/cmd/go.mod b/vendor/github.com/pyke369/golang-support/prefixdb/cmd/go.mod new file mode 100644 index 0000000..7dfafbe --- /dev/null +++ b/vendor/github.com/pyke369/golang-support/prefixdb/cmd/go.mod @@ -0,0 +1,5 @@ +module . + +go 1.12 + +require github.com/pyke369/golang-support v0.0.0-20190428173758-fae1fcd33c43 // indirect diff --git a/vendor/github.com/pyke369/golang-support/prefixdb/cmd/go.sum b/vendor/github.com/pyke369/golang-support/prefixdb/cmd/go.sum new file mode 100644 index 0000000..b89598d --- /dev/null +++ b/vendor/github.com/pyke369/golang-support/prefixdb/cmd/go.sum @@ -0,0 +1,2 @@ +github.com/pyke369/golang-support v0.0.0-20190428173758-fae1fcd33c43 h1:638A4GSCbTc/Z8N1TyymmC8iWQOE3BWnWJv9fzZeHJc= +github.com/pyke369/golang-support v0.0.0-20190428173758-fae1fcd33c43/go.mod h1:0XGrzgrEp0fa/+JSV8XZePUwyjnU6C3bMc7Xz2bHHKI= diff --git a/vendor/github.com/pyke369/golang-support/prefixdb/cmd/prefixdb.go b/vendor/github.com/pyke369/golang-support/prefixdb/cmd/prefixdb.go new file mode 100644 index 0000000..57bc683 --- /dev/null +++ b/vendor/github.com/pyke369/golang-support/prefixdb/cmd/prefixdb.go @@ -0,0 +1,367 @@ +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "net" + "net/http" + "os" + "regexp" + "strconv" + "strings" + "time" + + "github.com/pyke369/golang-support/prefixdb" +) + +type LOCATION struct { + ContinentCode string + ContinentName string + CountryCode string + CountryName string + RegionCode string + RegionName string + StateCode string + StateName string + CityName string + TimeZone string + InEurope bool +} + +var ( + csvMatcher = regexp.MustCompile(`(?:,|\n|^)("(?:(?:"")*[^"]*)*"|[^",\n]*|(?:\n|$))`) + jsonMatcher = regexp.MustCompile(`^(\S+)(?:\s(\{.+?\}))?$`) + pfdb = prefixdb.New() +) + +func size(input int) string { + if input < 1024*1024 { + return fmt.Sprintf("%.1fkB", float64(input)/1024) + } + return fmt.Sprintf("%.1fMB", float64(input)/(1024*1024)) +} + +func mkjson() { + for index := 3; index < len(os.Args); index++ { + count := 0 + if handle, err := os.Open(os.Args[index]); err == nil { + reader := bufio.NewReader(handle) + last := time.Now() + start := last + for { + if line, err := reader.ReadString('\n'); err != nil { + break + } else { + if fields := jsonMatcher.FindStringSubmatch(strings.TrimSpace(line)); fields != nil { + if _, prefix, err := net.ParseCIDR(fields[1]); err == nil { + data := map[string]interface{}{} + json.Unmarshal([]byte(fields[2]), &data) + pfdb.Add(*prefix, data, [][]string{[]string{"key1", "key2"}}) + count++ + } + } + } + if now := time.Now(); now.Sub(last) >= 250*time.Millisecond { + last = now + fmt.Fprintf(os.Stderr, "\r- adding prefixes [%s] %d", os.Args[index], count) + } + } + handle.Close() + fmt.Fprintf(os.Stderr, "\r- added prefixes [%s] (%.3fs - %d entries)\n", os.Args[index], float64(time.Now().Sub(start))/float64(time.Second), count) + } + } + start := time.Now() + description := "" + if index := strings.Index(os.Args[2], "@"); index > 0 { + description = os.Args[2][index+1:] + os.Args[2] = os.Args[2][:index] + } + fmt.Fprintf(os.Stderr, "\r- saving database [%s]... ", os.Args[2]) + if _, err := pfdb.Save(os.Args[2], description); err != nil { + fmt.Fprintf(os.Stderr, "\r- saving database [%s] failed (%v)\n", os.Args[2], err) + } else { + fmt.Fprintf(os.Stderr, "\r- saved database [%s] (%.3fs - total[%s] strings[%s] numbers[%s] pairs[%s] clusters[%s] maps[%s] nodes[%s])\n", + os.Args[2], float64(time.Now().Sub(start))/float64(time.Second), size(pfdb.Total), size(pfdb.Strings[0]), + size(pfdb.Numbers[0]), size(pfdb.Pairs[0]), size(pfdb.Clusters[0]), size(pfdb.Maps[0]), size(pfdb.Nodes[0]), + ) + } +} + +func mkcity() { + locations := map[int]*LOCATION{} + if handle, err := os.Open(os.Args[3]); err == nil { + reader := bufio.NewReader(handle) + last := time.Now() + start := last + for { + if line, err := reader.ReadString('\n'); err != nil { + break + } else { + if fields := csvMatcher.FindAllStringSubmatch(strings.TrimSpace(line), -1); len(fields) == 14 { + for index := 0; index < len(fields); index++ { + fields[index][1] = strings.Trim(fields[index][1], `"`) + } + if id, err := strconv.Atoi(fields[0][1]); err == nil { + locations[id] = &LOCATION{ + ContinentCode: fields[2][1], + ContinentName: fields[3][1], + CountryCode: fields[4][1], + CountryName: fields[5][1], + RegionCode: fields[6][1], + RegionName: fields[7][1], + StateCode: fields[8][1], + StateName: fields[9][1], + CityName: fields[10][1], + TimeZone: fields[12][1], + InEurope: fields[13][1] == "1", + } + } + } + } + if now := time.Now(); now.Sub(last) >= 250*time.Millisecond { + last = now + fmt.Fprintf(os.Stderr, "\r- loading locations [%s] %d", os.Args[3], len(locations)) + } + } + handle.Close() + fmt.Fprintf(os.Stderr, "\r- loaded locations [%s] (%.3fs - %d entries)\n", os.Args[3], float64(time.Now().Sub(start))/float64(time.Second), len(locations)) + } + + clusters := [][]string{ + []string{"continent_code", "continent_name", "country_code", "country_name", "region_code", "region_name", "state_code", "state_name", "timezone", "in_europe"}, + []string{"city_name", "postal_code", "latitude", "longitude"}, + } + for index := 4; index < len(os.Args); index++ { + count := 0 + if handle, err := os.Open(os.Args[index]); err == nil { + reader := bufio.NewReader(handle) + last := time.Now() + start := last + for { + if line, err := reader.ReadString('\n'); err != nil { + break + } else { + if fields := strings.Split(strings.TrimSpace(line), ","); len(fields) == 10 { + id := 0 + if id, _ = strconv.Atoi(fields[1]); id == 0 { + id, _ = strconv.Atoi(fields[2]) + } + if id != 0 && locations[id] != nil { + if _, prefix, err := net.ParseCIDR(fields[0]); err == nil { + latitude, _ := strconv.ParseFloat(fields[7], 64) + longitude, _ := strconv.ParseFloat(fields[8], 64) + pfdb.Add(*prefix, map[string]interface{}{ + "continent_code": locations[id].ContinentCode, + "continent_name": locations[id].ContinentName, + "country_code": locations[id].CountryCode, + "country_name": locations[id].CountryName, + "region_code": locations[id].RegionCode, + "region_name": locations[id].RegionName, + "state_code": locations[id].StateCode, + "state_name": locations[id].StateName, + "city_name": locations[id].CityName, + "in_europe": locations[id].InEurope, + "timezone": locations[id].TimeZone, + "postal_code": fields[6], + "latitude": latitude, + "longitude": longitude, + }, clusters) + count++ + } + } + } + } + if now := time.Now(); now.Sub(last) >= 250*time.Millisecond { + last = now + fmt.Fprintf(os.Stderr, "\r- adding prefixes [%s] %d", os.Args[index], count) + } + } + handle.Close() + fmt.Fprintf(os.Stderr, "\r- added prefixes [%s] (%.3fs - %d entries)\n", os.Args[index], float64(time.Now().Sub(start))/float64(time.Second), count) + } + } + + start := time.Now() + description := "" + if index := strings.Index(os.Args[2], "@"); index > 0 { + description = os.Args[2][index+1:] + os.Args[2] = os.Args[2][:index] + } + fmt.Fprintf(os.Stderr, "\r- saving database [%s]... ", os.Args[2]) + if _, err := pfdb.Save(os.Args[2], description); err != nil { + fmt.Fprintf(os.Stderr, "\r- saving database [%s] failed (%v)\n", os.Args[2], err) + } else { + fmt.Fprintf(os.Stderr, "\r- saved database [%s] (%.3fs - total[%s] strings[%s] numbers[%s] pairs[%s] clusters[%s] maps[%s] nodes[%s])\n", + os.Args[2], float64(time.Now().Sub(start))/float64(time.Second), size(pfdb.Total), size(pfdb.Strings[0]), + size(pfdb.Numbers[0]), size(pfdb.Pairs[0]), size(pfdb.Clusters[0]), size(pfdb.Maps[0]), size(pfdb.Nodes[0]), + ) + } +} + +func mkasn() { + for index := 3; index < len(os.Args); index++ { + count := 0 + if handle, err := os.Open(os.Args[index]); err == nil { + reader := bufio.NewReader(handle) + last := time.Now() + start := last + for { + if line, err := reader.ReadString('\n'); err != nil { + break + } else { + if fields := csvMatcher.FindAllStringSubmatch(strings.TrimSpace(line), -1); len(fields) == 3 { + for index := 0; index < len(fields); index++ { + fields[index][1] = strings.Trim(fields[index][1], `"`) + } + if asnum, _ := strconv.Atoi(fields[1][1]); asnum != 0 { + if _, prefix, err := net.ParseCIDR(fields[0][1]); err == nil { + pfdb.Add(*prefix, map[string]interface{}{ + "as_number": fmt.Sprintf("AS%d", asnum), + "as_name": fields[2][1], + }, nil) + count++ + } + } + } + } + if now := time.Now(); now.Sub(last) >= 250*time.Millisecond { + last = now + fmt.Fprintf(os.Stderr, "\r- adding prefixes [%s] %d", os.Args[index], count) + } + } + handle.Close() + fmt.Fprintf(os.Stderr, "\r- added prefixes [%s] (%.3fs - %d entries)\n", os.Args[index], float64(time.Now().Sub(start))/float64(time.Second), count) + } + } + + start := time.Now() + description := "" + if index := strings.Index(os.Args[2], "@"); index > 0 { + description = os.Args[2][index+1:] + os.Args[2] = os.Args[2][:index] + } + fmt.Fprintf(os.Stderr, "\r- saving database [%s]... ", os.Args[2]) + if _, err := pfdb.Save(os.Args[2], description); err != nil { + fmt.Fprintf(os.Stderr, "\r- saving database [%s] failed (%v)\n", os.Args[2], err) + } else { + fmt.Fprintf(os.Stderr, "\r- saved database [%s] (%.3fs - total[%s] strings[%s] numbers[%s] pairs[%s] clusters[%s] maps[%s] nodes[%s])\n", + os.Args[2], float64(time.Now().Sub(start))/float64(time.Second), size(pfdb.Total), size(pfdb.Strings[0]), + size(pfdb.Numbers[0]), size(pfdb.Pairs[0]), size(pfdb.Clusters[0]), size(pfdb.Maps[0]), size(pfdb.Nodes[0]), + ) + } +} + +func lookup() { + databases := []*prefixdb.PrefixDB{} + for index := 2; index < len(os.Args); index++ { + if strings.HasSuffix(os.Args[index], `.pfdb`) { + fmt.Fprintf(os.Stderr, "\r- loading database [%s]...", os.Args[index]) + database := prefixdb.New() + if err := database.Load(os.Args[index]); err == nil { + fmt.Fprintf(os.Stderr, "\r- loaded database [%s] (total[%s] version[%d.%d.%d] description[%s])\n", + os.Args[index], size(database.Total), (database.Version>>16)&0xff, (database.Version>>8)&0xff, database.Version&0xff, database.Description) + databases = append(databases, database) + } else { + fmt.Fprintf(os.Stderr, "\r- loading database [%s] failed (%v)\n", os.Args[index], err) + } + } else { + if ip := net.ParseIP(os.Args[index]); ip == nil { + fmt.Fprintf(os.Stderr, "- lookup [%s] failed (not a valid IP address)", os.Args[index]) + } else { + fmt.Fprintf(os.Stderr, "- lookup [%s] ", os.Args[index]) + lookup := map[string]interface{}{} + for _, database := range databases { + lookup, _ = database.Lookup(ip, lookup) + } + data, _ := json.Marshal(lookup) + fmt.Printf("%s\n", data) + } + } + } +} + +func server() { + databases := []*prefixdb.PrefixDB{} + for index := 3; index < len(os.Args); index++ { + fmt.Fprintf(os.Stderr, "\r- loading database [%s]...", os.Args[index]) + database := prefixdb.New() + if err := database.Load(os.Args[index]); err == nil { + fmt.Fprintf(os.Stderr, "\r- loaded database [%s] (total[%s] version[%d.%d.%d] description[%s])\n", + os.Args[index], size(database.Total), (database.Version>>16)&0xff, (database.Version>>8)&0xff, database.Version&0xff, database.Description) + databases = append(databases, database) + } else { + fmt.Fprintf(os.Stderr, "\r- loading database [%s] failed (%v)\n", os.Args[index], err) + } + } + http.HandleFunc("/", func(response http.ResponseWriter, request *http.Request) { + response.Header().Set("Content-Type", "application/json") + remote, _, _ := net.SplitHostPort(request.RemoteAddr) + if value := request.Header.Get("X-Forwarded-For"); value != "" { + remote = strings.Split(value, ",")[0] + } + parameters := request.URL.Query() + if value := parameters.Get("remote"); value != "" { + remote = value + } + lookup := map[string]interface{}{} + if ip := net.ParseIP(remote); ip != nil { + lookup["ip"] = fmt.Sprintf("%s", ip) + for _, database := range databases { + lookup, _ = database.Lookup(ip, lookup) + } + data, _ := json.Marshal(lookup) + response.Write(data) + } + }) + parts := strings.Split(os.Args[2], ",") + server := &http.Server{Addr: strings.TrimLeft(parts[0], "*"), ReadTimeout: 10 * time.Second, WriteTimeout: 10 * time.Second} + fmt.Fprintf(os.Stderr, "\r- listening to [%s]\n", os.Args[2]) + if len(parts) > 1 { + server.ListenAndServeTLS(parts[1], parts[2]) + } else { + server.ListenAndServe() + } +} + +func usage(status int) { + fmt.Fprintf(os.Stderr, "usage: prefixdb [parameters...]\n\n"+ + "help show this help screen\n"+ + "json ]> ... build database from generic JSON-formatted prefixes lists\n"+ + "city ]> ... build database from MaxMind GeoIP2 cities lists\n"+ + "asn ]> ... build database from MaxMind GeoLite2 asnums lists\n"+ + "lookup ...
... lookup entries in databases\n"+ + "server ... spawn an HTTP(S) server for entries lookup\n") + os.Exit(status) +} + +func main() { + if len(os.Args) < 2 { + usage(1) + } + switch os.Args[1] { + case "help": + usage(0) + case "json": + if len(os.Args) < 3 { + usage(1) + } + mkjson() + case "city": + if len(os.Args) < 5 { + usage(1) + } + mkcity() + case "asn": + if len(os.Args) < 3 { + usage(1) + } + mkasn() + case "lookup": + lookup() + case "server": + server() + default: + usage(2) + } +} diff --git a/vendor/github.com/pyke369/golang-support/prefixdb/prefixdb.go b/vendor/github.com/pyke369/golang-support/prefixdb/prefixdb.go new file mode 100644 index 0000000..14b9a25 --- /dev/null +++ b/vendor/github.com/pyke369/golang-support/prefixdb/prefixdb.go @@ -0,0 +1,827 @@ +package prefixdb + +import ( + "bytes" + "crypto/md5" + "encoding/binary" + "errors" + "fmt" + "io/ioutil" + "math" + "net" + "os" + "runtime" + "sort" + "strings" + "sync" + "time" +) + +const VERSION = 0x00010000 + +type fame struct { + fame int + value interface{} +} +type byfame []*fame + +func (a byfame) Len() int { return len(a) } +func (a byfame) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a byfame) Less(i, j int) bool { return a[i].fame > a[j].fame } + +type node struct { + down [2]*node + up *node + data []uint64 + offset int + explored [4]bool + emitted bool + id int +} +type cluster struct { + values [3]int // fame / initial index / final index + pairs []uint64 // cluster pairs + data []byte // reduced cluster pairs +} +type PrefixDB struct { + sync.RWMutex + tree node + strings map[string]*[3]int // fame / initial index / final index + numbers map[float64]*[3]int // fame / initial index / final index + pairs map[uint64]*[3]int // fame / initial index / final index + clusters map[[16]byte]*cluster + data []byte + Total int + Version uint32 + Description string + Strings [4]int // size / count / offset / strings index width (bytes) + Numbers [3]int // size / count / offset + Pairs [3]int // size / count / offset + Clusters [4]int // size / count / offset / clusters index width (bytes) + Maps [3]int // size / count / offset + Nodes [4]int // size / count / offset / nodes width (bits) +} + +var cores int + +func New() *PrefixDB { + if cores == 0 { + cores = runtime.NumCPU() + } + return &PrefixDB{strings: map[string]*[3]int{}, numbers: map[float64]*[3]int{}, pairs: map[uint64]*[3]int{}, clusters: map[[16]byte]*cluster{}} +} + +func (this *PrefixDB) Add(prefix net.IPNet, data map[string]interface{}, clusters [][]string) { + prefix.IP = prefix.IP.To16() + ones, bits := prefix.Mask.Size() + if bits == 32 { + ones += 96 + prefix.Mask = net.CIDRMask(ones, bits+96) + } + if cores > 1 { + this.Lock() + } + pnode := &this.tree + for bit := 0; bit < ones; bit++ { + down := 0 + if (prefix.IP[bit/8] & (1 << (7 - (byte(bit) % 8)))) != 0 { + down = 1 + } + if pnode.down[down] == nil { + pnode.down[down] = &node{} + pnode.down[down].up = pnode + } + if len(pnode.data) != 0 { + pnode.data = []uint64{} + } + pnode = pnode.down[down] + } + + skeys, ckeys, lkeys := "", [][]string{}, []string{} + for _, cluster := range clusters { + skeys += strings.Join(cluster, ` `) + ` ` + ckeys = append(ckeys, cluster) + } + for key, _ := range data { + if strings.Index(skeys, key) < 0 { + lkeys = append(lkeys, key) + } + } + ckeys = append(ckeys, lkeys) + for cindex, keys := range ckeys { + cpairs := []uint64{} + for _, key := range keys { + if len(key) > 255 { + continue + } + if value, ok := data[key]; ok { + index := 0 + if _, ok := this.strings[key]; !ok { + index = len(this.strings) + this.strings[key] = &[3]int{1, index} + } else { + index = this.strings[key][1] + this.strings[key][0]++ + } + pair := uint64((uint32(index)&0x0fffffff)|0x10000000) << 32 + if tvalue, ok := value.(string); ok { + if len(tvalue) <= 255 { + index = 0 + if _, ok := this.strings[tvalue]; !ok { + index = len(this.strings) + this.strings[tvalue] = &[3]int{1, index} + } else { + index = this.strings[tvalue][1] + this.strings[tvalue][0]++ + } + pair |= uint64((uint32(index) & 0x0fffffff) | 0x10000000) + } else { + pair |= uint64(0x50000000) + } + } else if tvalue, ok := value.(float64); ok { + index = 0 + if _, ok := this.numbers[tvalue]; !ok { + index = len(this.numbers) + this.numbers[tvalue] = &[3]int{1, index} + } else { + index = this.numbers[tvalue][1] + this.numbers[tvalue][0]++ + } + pair |= uint64((uint32(index) & 0x0fffffff) | 0x20000000) + } else if tvalue, ok := value.(bool); ok { + if tvalue { + pair |= uint64(0x30000000) + } else { + pair |= uint64(0x40000000) + } + } else { + pair |= uint64(0x50000000) + } + if _, ok := this.pairs[pair]; !ok { + index = len(this.pairs) + this.pairs[pair] = &[3]int{1, index} + } else { + this.pairs[pair][0]++ + } + if cindex < len(ckeys)-1 { + cpairs = append(cpairs, pair) + } else { + pnode.data = append(pnode.data, pair) + } + } + } + if len(cpairs) != 0 { + buffer := make([]byte, len(cpairs)*8) + for index, value := range cpairs { + binary.BigEndian.PutUint64(buffer[index*8:], value) + } + key := md5.Sum(buffer) + index := 0 + if _, ok := this.clusters[key]; !ok { + index = len(this.clusters) + this.clusters[key] = &cluster{pairs: cpairs, values: [3]int{1, index}} + } else { + index = this.clusters[key].values[1] + this.clusters[key].values[0]++ + } + pnode.data = append(pnode.data, 0x7000000000000000|((uint64(index)<<32)&0x0fffffff00000000)) + } + } + if cores > 1 { + this.Unlock() + } +} + +func wbytes(bytes, value int, data []byte) { + if len(data) >= bytes { + for index := bytes - 1; index >= 0; index-- { + data[bytes-index-1] = byte(value >> (uint(index * 8))) + } + } +} +func wpbits(prefix byte, value int) []byte { + if value <= 7 { + return []byte{prefix | (byte(value) & 0x07)} + } + bytes := int(math.Ceil(math.Ceil(math.Log2(float64(value+1))) / 8)) + data := []byte{prefix | 0x08 | byte(bytes)} + for nibble := bytes - 1; nibble >= 0; nibble-- { + data = append(data, byte(value>>(uint(nibble*8)))) + } + return data +} +func wnbits(bits, value0, value1 int, data []byte) { + if bits >= 8 && bits <= 32 && bits%4 == 0 && len(data) >= bits/4 { + switch bits { + case 8: + data[0], data[1] = byte(value0), byte(value1) + case 12: + data[0], data[1], data[2] = byte(value0>>4), byte(value0<<4)|(byte(value1>>8)&0x0f), byte(value1) + case 16: + binary.BigEndian.PutUint16(data[0:], uint16(value0)) + binary.BigEndian.PutUint16(data[2:], uint16(value1)) + case 20: + data[0], data[1] = byte(value0>>12), byte(value0>>4) + data[2] = byte(value0<<4) | (byte(value1>>16) & 0x0f) + data[3], data[4] = byte(value1>>8), byte(value1) + case 24: + data[0], data[1], data[2] = byte(value0>>16), byte(value0>>8), byte(value0) + data[3], data[4], data[5] = byte(value1>>16), byte(value1>>8), byte(value1) + case 28: + data[0], data[1], data[2] = byte(value0>>20), byte(value0>>12), byte(value0>>4) + data[3] = byte(value0<<4) | (byte(value1>>24) & 0x0f) + data[4], data[5], data[6] = byte(value1>>16), byte(value1>>8), byte(value1) + case 32: + binary.BigEndian.PutUint32(data[0:], uint32(value0)) + binary.BigEndian.PutUint32(data[4:], uint32(value1)) + } + } +} +func (this *PrefixDB) Save(path, description string) (content []byte, err error) { + // layout header + signature placeholder + description + if cores > 1 { + this.Lock() + } + this.data = []byte{'P', 'F', 'D', 'B', 0, (VERSION >> 16) & 0xff, (VERSION >> 8) & 0xff, (VERSION & 0xff), + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 'D', 'E', 'S', 'C', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} + if description == "" { + description = time.Now().Format(`20060102150405`) + } + this.Description = description + copy(this.data[28:47], []byte(description)) + + // layout strings dictionary (ordered by fame) + this.Strings[0] = 0 + for key, _ := range this.strings { + this.Strings[0] += len(key) + } + this.Strings[3] = int(math.Ceil(math.Ceil(math.Log2(float64(this.Strings[0]+1))) / 8)) + this.data = append(this.data, []byte{'S', 'T', 'R', 'S', byte(this.Strings[3]), 0, 0, 0, 0, 0, 0, 0, 0}...) + this.Strings[2] = len(this.data) + this.Strings[1] = len(this.strings) + this.Strings[0] += this.Strings[1] * this.Strings[3] + flist := make([]*fame, this.Strings[1]) + for key, values := range this.strings { + flist[values[1]] = &fame{values[0], key} + } + sort.Sort(byfame(flist)) + this.data = append(this.data, make([]byte, this.Strings[1]*this.Strings[3])...) + offset := 0 + for index, item := range flist { + this.strings[item.value.(string)][2] = index + this.data = append(this.data, []byte(item.value.(string))...) + wbytes(this.Strings[3], offset, this.data[this.Strings[2]+(index*this.Strings[3]):]) + offset += len(item.value.(string)) + } + binary.BigEndian.PutUint32(this.data[this.Strings[2]-8:], uint32(this.Strings[0])) + binary.BigEndian.PutUint32(this.data[this.Strings[2]-4:], uint32(this.Strings[1])) + strings := make([]*fame, this.Strings[1]) + for key, values := range this.strings { + strings[values[1]] = &fame{values[0], key} + } + + // layout numbers dictionary (ordered by fame) + this.data = append(this.data, []byte{'N', 'U', 'M', 'S', 0, 0, 0, 0, 0, 0, 0, 0}...) + this.Numbers[2] = len(this.data) + this.Numbers[1] = len(this.numbers) + this.Numbers[0] = this.Numbers[1] * 8 + flist = make([]*fame, this.Numbers[1]) + for key, values := range this.numbers { + flist[values[1]] = &fame{values[0], key} + } + sort.Sort(byfame(flist)) + this.data = append(this.data, make([]byte, this.Numbers[1]*8)...) + for index, item := range flist { + this.numbers[item.value.(float64)][2] = index + binary.BigEndian.PutUint64(this.data[this.Numbers[2]+(index*8):], math.Float64bits(item.value.(float64))) + } + binary.BigEndian.PutUint32(this.data[this.Numbers[2]-8:], uint32(this.Numbers[0])) + binary.BigEndian.PutUint32(this.data[this.Numbers[2]-4:], uint32(this.Numbers[1])) + numbers := make([]*fame, this.Numbers[1]) + for key, values := range this.numbers { + numbers[values[1]] = &fame{values[0], key} + } + + // layout pairs dictionary (ordered by fame) + this.data = append(this.data, []byte{'P', 'A', 'I', 'R', 0, 0, 0, 0, 0, 0, 0, 0}...) + this.Pairs[2] = len(this.data) + flist = make([]*fame, len(this.pairs)) + for key, values := range this.pairs { + flist[values[1]] = &fame{values[0], key} + } + sort.Sort(byfame(flist)) + for index, item := range flist { + if item.fame > 1 { + this.pairs[item.value.(uint64)][2] = index + } else { + delete(this.pairs, item.value.(uint64)) + } + } + this.Pairs[1] = len(this.pairs) + this.Pairs[0] = this.Pairs[1] * 8 + this.data = append(this.data, make([]byte, this.Pairs[0])...) + for index, item := range flist { + if item.fame <= 1 { + break + } + pair := 0x1000000000000000 | (uint64(this.strings[strings[(item.value.(uint64)>>32)&0x0fffffff].value.(string)][2]) << 32) + switch (item.value.(uint64) & 0xf0000000) >> 28 { + case 1: + pair |= 0x10000000 | uint64(this.strings[strings[item.value.(uint64)&0x0fffffff].value.(string)][2]) + case 2: + pair |= 0x20000000 | uint64(this.numbers[numbers[item.value.(uint64)&0x0fffffff].value.(float64)][2]) + default: + pair |= item.value.(uint64) & 0xf0000000 + } + binary.BigEndian.PutUint64(this.data[this.Pairs[2]+(index*8):], pair) + } + binary.BigEndian.PutUint32(this.data[this.Pairs[2]-8:], uint32(this.Pairs[0])) + binary.BigEndian.PutUint32(this.data[this.Pairs[2]-4:], uint32(this.Pairs[1])) + + // layout clusters dictionary (ordered by fame, and reduced for strings, numbers and pairs) + this.Clusters[0] = 0 + for _, cluster := range this.clusters { + for _, pair := range cluster.pairs { + if _, ok := this.pairs[pair]; ok { + cluster.data = append(cluster.data, wpbits(0x60, this.pairs[pair][2])...) + } else { + cluster.data = append(cluster.data, wpbits(0x10, this.strings[strings[(pair>>32)&0x0fffffff].value.(string)][2])...) + switch (pair & 0xf0000000) >> 28 { + case 1: + cluster.data = append(cluster.data, wpbits(0x10, this.strings[strings[pair&0x0fffffff].value.(string)][2])...) + case 2: + cluster.data = append(cluster.data, wpbits(0x20, this.numbers[numbers[pair&0x0fffffff].value.(float64)][2])...) + default: + cluster.data = append(cluster.data, byte((pair&0xf0000000)>>24)) + } + } + } + this.Clusters[0] += len(cluster.data) + } + this.Clusters[3] = int(math.Ceil(math.Ceil(math.Log2(float64(this.Clusters[0]+1))) / 8)) + this.data = append(this.data, []byte{'C', 'L', 'U', 'S', byte(this.Clusters[3]), 0, 0, 0, 0, 0, 0, 0, 0}...) + this.Clusters[2] = len(this.data) + this.Clusters[1] = len(this.clusters) + this.Clusters[0] += this.Clusters[1] * this.Clusters[3] + flist = make([]*fame, this.Clusters[1]) + for key, cluster := range this.clusters { + flist[cluster.values[1]] = &fame{cluster.values[0], key} + } + sort.Sort(byfame(flist)) + this.data = append(this.data, make([]byte, this.Clusters[1]*this.Clusters[3])...) + offset = 0 + for index, item := range flist { + this.clusters[item.value.([16]byte)].values[2] = index + this.data = append(this.data, this.clusters[item.value.([16]byte)].data...) + wbytes(this.Clusters[3], offset, this.data[this.Clusters[2]+(index*this.Clusters[3]):]) + offset += len(this.clusters[item.value.([16]byte)].data) + } + binary.BigEndian.PutUint32(this.data[this.Clusters[2]-8:], uint32(this.Clusters[0])) + binary.BigEndian.PutUint32(this.data[this.Clusters[2]-4:], uint32(this.Clusters[1])) + clusters := make([]*fame, this.Clusters[1]) + for key, cluster := range this.clusters { + clusters[cluster.values[1]] = &fame{cluster.values[0], key} + } + + // layout maps dictionary (reduced for strings, numbers, pairs and clusters) + this.Nodes[1] = 1 + this.data = append(this.data, []byte{'M', 'A', 'P', 'S', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x80}...) + pnode := &this.tree + this.Maps[2] = len(this.data) - 2 + this.Maps[1] = 1 + this.Maps[0] = 2 + for { + if pnode.down[0] != nil && !pnode.explored[0] { + pnode.explored[0] = true + pnode = pnode.down[0] + } else if pnode.down[1] != nil && !pnode.explored[1] { + pnode.explored[1] = true + pnode = pnode.down[1] + } else if pnode.up != nil { + pnode = pnode.up + } + if pnode.up == nil { + break + } + if pnode.down[0] == nil && pnode.down[1] == nil { + if len(pnode.data) == 0 { + pnode.offset = 1 + } else { + data := []byte{} + for index := 0; index < len(pnode.data); index++ { + last := byte(0x00) + if index == len(pnode.data)-1 { + last = 0x80 + } + if ((pnode.data[index]>>32)&0xf0000000)>>28 == 7 { + data = append(data, wpbits(last|0x70, this.clusters[clusters[(pnode.data[index]>>32)&0x0fffffff].value.([16]byte)].values[2])...) + } else { + if _, ok := this.pairs[pnode.data[index]]; ok { + data = append(data, wpbits(last|0x60, this.pairs[pnode.data[index]][2])...) + } else { + data = append(data, wpbits(0x10, this.strings[strings[(pnode.data[index]>>32)&0x0fffffff].value.(string)][2])...) + switch (pnode.data[index] & 0xf0000000) >> 28 { + case 1: + data = append(data, wpbits(last|0x10, this.strings[strings[pnode.data[index]&0x0fffffff].value.(string)][2])...) + case 2: + data = append(data, wpbits(last|0x20, this.numbers[numbers[pnode.data[index]&0x0fffffff].value.(float64)][2])...) + default: + data = append(data, last|byte((pnode.data[index]&0xf0000000)>>24)) + } + } + } + } + this.data = append(this.data, data...) + pnode.offset = this.Maps[0] + this.Maps[0] += len(data) + this.Maps[1]++ + } + } else if pnode.id == 0 { + pnode.id = this.Nodes[1] + this.Nodes[1]++ + } + } + binary.BigEndian.PutUint32(this.data[this.Maps[2]-8:], uint32(this.Maps[0])) + binary.BigEndian.PutUint32(this.data[this.Maps[2]-4:], uint32(this.Maps[1])) + + // layout nodes tree + this.Nodes[3] = int(math.Ceil(math.Ceil(math.Log2(float64(this.Nodes[1]+this.Maps[0]+1)))/4) * 4) + this.data = append(this.data, []byte{'N', 'O', 'D', 'E', byte(this.Nodes[3]), 0, 0, 0, 0, 0, 0, 0, 0}...) + this.Nodes[2] = len(this.data) + this.Nodes[0] = this.Nodes[1] * ((2 * this.Nodes[3]) / 8) + this.data = append(this.data, make([]byte, this.Nodes[0])...) + pnode = &this.tree + next := [2]int{} + for { + if (pnode == &this.tree || pnode.id != 0) && !pnode.emitted { + pnode.emitted = true + for index := 0; index <= 1; index++ { + next[index] = this.Nodes[1] + if pnode.down[index] != nil { + if pnode.down[index].id != 0 { + next[index] = pnode.down[index].id + } else { + next[index] += pnode.down[index].offset + } + } + } + wnbits(this.Nodes[3], next[0], next[1], this.data[this.Nodes[2]+(pnode.id*((2*this.Nodes[3])/8)):]) + } + if pnode.down[0] != nil && !pnode.explored[2] { + pnode.explored[2] = true + pnode = pnode.down[0] + } else if pnode.down[1] != nil && !pnode.explored[3] { + pnode.explored[3] = true + pnode = pnode.down[1] + } else if pnode.up != nil { + pnode = pnode.up + } + if pnode.up == nil { + break + } + } + binary.BigEndian.PutUint32(this.data[this.Nodes[2]-8:], uint32(this.Nodes[0])) + binary.BigEndian.PutUint32(this.data[this.Nodes[2]-4:], uint32(this.Nodes[1])) + + // finalize header + this.tree, this.strings, this.numbers, this.pairs, this.clusters = node{}, map[string]*[3]int{}, map[float64]*[3]int{}, map[uint64]*[3]int{}, map[[16]byte]*cluster{} + hash := md5.Sum(this.data[24:]) + copy(this.data[8:], hash[:]) + this.Total = len(this.data) + + // save database + if path != "" { + if path == "-" { + _, err = os.Stdout.Write(this.data) + } else { + err = ioutil.WriteFile(path, this.data, 0644) + } + } + + if cores > 1 { + this.Unlock() + } + return this.data, err +} + +func (this *PrefixDB) Load(path string) error { + if data, err := ioutil.ReadFile(path); err != nil { + return err + } else { + if len(data) < 8 || string(data[0:4]) != "PFDB" { + return errors.New(`invalid preamble`) + } + if version := (uint32(data[5]) << 16) + (uint32(data[6]) << 8) + uint32(data[7]); (version & 0xff0000) > (VERSION & 0xff0000) { + return errors.New(fmt.Sprintf(`library major version %d is incompatible with database major version %d`, (VERSION&0xff0000)>>16, (version&0xff0000)>>16)) + } else { + if len(data) < 24 || fmt.Sprintf("%x", md5.Sum(data[24:])) != fmt.Sprintf("%x", data[8:24]) { + return errors.New(`database checksum is invalid`) + } + if cores > 1 { + this.Lock() + } + this.data = data + this.Total = len(data) + this.Version = version + offset := 24 + if this.Total >= offset+4 && string(data[offset:offset+4]) == "DESC" { + offset += 4 + if this.Total >= offset+20 { + index := 0 + if index = bytes.Index(data[offset:offset+20], []byte{0}); index < 0 { + index = 20 + } + this.Description = fmt.Sprintf("%s", data[offset:offset+index]) + offset += 20 + if this.Total >= offset+4 && string(data[offset:offset+4]) == "STRS" { + offset += 4 + if this.Total >= offset+9 { + this.Strings[3] = int(data[offset]) + this.Strings[2] = offset + 9 + this.Strings[1] = int(binary.BigEndian.Uint32(this.data[offset+5:])) + this.Strings[0] = int(binary.BigEndian.Uint32(this.data[offset+1:])) + offset += 9 + this.Strings[0] + if this.Total >= offset+4 && string(data[offset:offset+4]) == "NUMS" { + offset += 4 + if this.Total >= offset+8 { + this.Numbers[2] = offset + 8 + this.Numbers[1] = int(binary.BigEndian.Uint32(this.data[offset+4:])) + this.Numbers[0] = int(binary.BigEndian.Uint32(this.data[offset:])) + offset += 8 + this.Numbers[0] + if this.Total >= offset+4 && string(data[offset:offset+4]) == "PAIR" { + offset += 4 + if this.Total >= offset+8 { + this.Pairs[2] = offset + 8 + this.Pairs[1] = int(binary.BigEndian.Uint32(this.data[offset+4:])) + this.Pairs[0] = int(binary.BigEndian.Uint32(this.data[offset:])) + offset += 8 + this.Pairs[0] + if this.Total >= offset+4 && string(data[offset:offset+4]) == "CLUS" { + offset += 4 + this.Clusters[3] = int(data[offset]) + this.Clusters[2] = offset + 9 + this.Clusters[1] = int(binary.BigEndian.Uint32(this.data[offset+5:])) + this.Clusters[0] = int(binary.BigEndian.Uint32(this.data[offset+1:])) + offset += 9 + this.Clusters[0] + if this.Total >= offset+4 && string(data[offset:offset+4]) == "MAPS" { + offset += 4 + if this.Total >= offset+8 { + this.Maps[2] = offset + 8 + this.Maps[1] = int(binary.BigEndian.Uint32(this.data[offset+4:])) + this.Maps[0] = int(binary.BigEndian.Uint32(this.data[offset:])) + offset += 8 + this.Maps[0] + if this.Total >= offset+9 && string(data[offset:offset+4]) == "NODE" { + offset += 4 + this.Nodes[3] = int(data[offset]) + this.Nodes[2] = offset + 9 + this.Nodes[1] = int(binary.BigEndian.Uint32(this.data[offset+5:])) + this.Nodes[0] = int(binary.BigEndian.Uint32(this.data[offset+1:])) + if offset+9+this.Nodes[0] != this.Total { + this.Nodes[2] = 0 + } + } + } + } + } + } + } + } + } + } + } + } + } + if cores > 1 { + this.Unlock() + } + if this.Strings[2] == 0 || this.Numbers[2] == 0 || this.Pairs[2] == 0 || this.Clusters[2] == 0 || this.Maps[2] == 0 || this.Nodes[2] == 0 { + return errors.New(`database structure is invalid`) + } + } + } + return nil +} +func rpbits(data []byte) (section, index, size int, last bool) { + section = int((data[0] & 0x70) >> 4) + if data[0]&0x80 != 0 || section == 0 { + last = true + } + if section == 1 || section == 2 || section == 6 || section == 7 { + if data[0]&0x08 != 0 { + size = int(data[0] & 0x07) + for nibble := 1; nibble <= size; nibble++ { + index |= int(data[nibble]) << (uint(size-nibble) * 8) + } + } else { + index = int(data[0] & 0x07) + } + } + size++ + return section, index, size, last +} +func rnbits(bits, index, down int, data []byte) int { + if bits >= 8 && bits <= 32 && bits%4 == 0 && (down == 0 || down == 1) && len(data) >= (index+1)*(bits/4) { + offset := index * (bits / 4) + switch bits { + case 8: + return int(data[offset+down]) + case 12: + if down == 0 { + return (int(data[offset]) << 4) | ((int(data[offset+1]) >> 4) & 0x0f) + } else { + return ((int(data[offset+1]) & 0x0f) << 8) | int(data[offset+2]) + } + case 16: + return int(binary.BigEndian.Uint16(data[offset+(down*2):])) + case 20: + if down == 0 { + return (int(data[offset]) << 12) | (int(data[offset+1]) << 4) | ((int(data[offset+2]) >> 4) & 0x0f) + } else { + return ((int(data[offset+2]) & 0x0f) << 16) | (int(data[offset+3]) << 8) | int(data[offset+4]) + } + case 24: + if down == 0 { + return (int(data[offset]) << 16) | (int(data[offset+1]) << 8) | int(data[offset+2]) + } else { + return (int(data[offset+3]) << 16) | (int(data[offset+4]) << 8) | int(data[offset+5]) + } + case 28: + if down == 0 { + return (int(data[offset]) << 20) | (int(data[offset+1]) << 12) | (int(data[offset+2]) << 4) | ((int(data[offset+3]) >> 4) & 0x0f) + } else { + return ((int(data[offset+3]) & 0x0f) << 24) | (int(data[offset+4]) << 16) | (int(data[offset+5]) << 8) | int(data[offset+6]) + } + case 32: + return int(binary.BigEndian.Uint32(data[offset+(down*4):])) + } + } + return index +} +func rbytes(width int, data []byte) (value int) { + for index := 0; index < width; index++ { + value |= int(data[index]) << (uint(width-1-index) * 8) + } + return value +} +func (this *PrefixDB) rstring(index int) string { + count, offset, width := this.Strings[1], this.Strings[2], this.Strings[3] + if index >= count { + return "" + } + start, end := rbytes(width, this.data[offset+(index*width):]), 0 + if index < count-1 { + end = rbytes(width, this.data[offset+(index+1)*width:]) + } else { + end = this.Strings[0] - (count * width) + } + return string(this.data[offset+(count*width)+start : offset+(count*width)+end]) +} +func (this *PrefixDB) rnumber(index int) float64 { + if index >= this.Numbers[1] { + return 0.0 + } + return math.Float64frombits(binary.BigEndian.Uint64(this.data[this.Numbers[2]+(index*8):])) +} +func (this *PrefixDB) rpair(index int, pairs map[string]interface{}) { + if index < this.Pairs[1] { + pair := binary.BigEndian.Uint64(this.data[this.Pairs[2]+(index*8):]) + if key := this.rstring(int((pair >> 32) & 0x0fffffff)); key != "" { + switch (pair & 0xf0000000) >> 28 { + case 1: + pairs[key] = this.rstring(int(pair & 0x0fffffff)) + case 2: + pairs[key] = this.rnumber(int(pair & 0x0fffffff)) + case 3: + pairs[key] = true + case 4: + pairs[key] = false + } + } + } +} +func (this *PrefixDB) rcluster(index int, pairs map[string]interface{}) { + count, offset, width := this.Clusters[1], this.Clusters[2], this.Clusters[3] + if index < count { + start, end := rbytes(width, this.data[offset+(index*width):]), 0 + if index < count-1 { + end = rbytes(width, this.data[offset+(index+1)*width:]) + } else { + end = this.Clusters[0] - (count * width) + } + start += offset + (count * width) + end += offset + (count * width) + key := "" + for start < end { + section, index, size, _ := rpbits(this.data[start:]) + switch section { + case 1: + if key != "" { + pairs[key] = this.rstring(index) + key = "" + } else { + key = this.rstring(index) + } + case 2: + if key != "" { + pairs[key] = this.rnumber(index) + key = "" + } + case 3: + if key != "" { + pairs[key] = true + key = "" + } + case 4: + if key != "" { + pairs[key] = false + key = "" + } + case 5: + if key != "" { + pairs[key] = nil + key = "" + } + case 6: + this.rpair(index, pairs) + } + start += size + } + } +} +func (this *PrefixDB) Lookup(address net.IP, input map[string]interface{}) (output map[string]interface{}, err error) { + output = input + if this.data == nil || this.Total == 0 || this.Version == 0 || this.Strings[2] == 0 || this.Numbers[2] == 0 || + this.Pairs[2] == 0 || this.Clusters[2] == 0 || this.Maps[2] == 0 || this.Nodes[2] == 0 || address == nil { + err = errors.New("record not found") + } else { + address = address.To16() + offset := 0 + if cores > 1 { + this.RLock() + } + for bit := 0; bit < 128; bit++ { + down := 0 + if (address[bit/8] & (1 << (7 - (byte(bit) % 8)))) != 0 { + down = 1 + } + offset = rnbits(this.Nodes[3], offset, down, this.data[this.Nodes[2]:]) + if offset == this.Nodes[1] || offset == 0 { + break + } + if output == nil { + output = map[string]interface{}{} + } + if offset > this.Nodes[1] { + offset -= this.Nodes[1] + if offset < this.Maps[0] { + offset += this.Maps[2] + key := "" + for offset < this.Maps[2]+this.Maps[0] { + section, index, size, last := rpbits(this.data[offset:]) + switch section { + case 1: + if key != "" { + output[key] = this.rstring(index) + key = "" + } else { + key = this.rstring(index) + } + case 2: + if key != "" { + output[key] = this.rnumber(index) + key = "" + } + case 3: + if key != "" { + output[key] = true + key = "" + } + case 4: + if key != "" { + output[key] = false + key = "" + } + case 5: + if key != "" { + output[key] = nil + key = "" + } + case 6: + this.rpair(index, output) + case 7: + this.rcluster(index, output) + } + if last { + break + } + offset += size + } + } + break + } + } + if cores > 1 { + this.RUnlock() + } + } + return output, err +} diff --git a/vendor/github.com/pyke369/golang-support/rcache/rcache.go b/vendor/github.com/pyke369/golang-support/rcache/rcache.go new file mode 100644 index 0000000..e02ed73 --- /dev/null +++ b/vendor/github.com/pyke369/golang-support/rcache/rcache.go @@ -0,0 +1,42 @@ +package rcache + +import ( + "crypto/md5" + "regexp" + "runtime" + "sync" +) + +var ( + cores int + cache map[[16]byte]*regexp.Regexp = map[[16]byte]*regexp.Regexp{} + lock sync.RWMutex +) + +func Get(expression string) *regexp.Regexp { + if cores == 0 { + cores = runtime.NumCPU() + } + key := md5.Sum([]byte(expression)) + if cores > 1 { + lock.RLock() + } + if cache[key] != nil { + if cores > 1 { + defer lock.RUnlock() + } + return cache[key].Copy() + } + if cores > 1 { + lock.RUnlock() + } + if regex, err := regexp.Compile(expression); err == nil { + if cores > 1 { + lock.Lock() + defer lock.Unlock() + } + cache[key] = regex + return cache[key].Copy() + } + return nil +} diff --git a/vendor/github.com/pyke369/golang-support/rpack/README.md b/vendor/github.com/pyke369/golang-support/rpack/README.md new file mode 100644 index 0000000..e69de29 diff --git a/vendor/github.com/pyke369/golang-support/rpack/cmd/rpack.go b/vendor/github.com/pyke369/golang-support/rpack/cmd/rpack.go new file mode 100644 index 0000000..0799c04 --- /dev/null +++ b/vendor/github.com/pyke369/golang-support/rpack/cmd/rpack.go @@ -0,0 +1,38 @@ +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + + "github.com/pyke369/golang-support/rpack" +) + +func main() { + var ( + options flag.FlagSet + usemain bool + ) + + options = flag.FlagSet{Usage: func() { + fmt.Fprintf(os.Stderr, "usage: %s [options] rootdir\n\noptions are:\n\n", filepath.Base(os.Args[0])) + options.PrintDefaults() + }, + } + options.String("output", "static.go", "the generated output file path") + options.String("pkgname", "main", "the package name to use in the generated output") + options.String("funcname", "resources", "the function name to use in the generated output") + options.Bool("main", false, "whether to generate a main func or not") + if err := options.Parse(os.Args[1:]); err != nil { + os.Exit(1) + } + if options.NArg() == 0 { + options.Usage() + os.Exit(1) + } + if options.Lookup("main").Value.String() == "true" { + usemain = true + } + rpack.Pack(options.Arg(0), options.Lookup("output").Value.String(), options.Lookup("pkgname").Value.String(), options.Lookup("funcname").Value.String(), usemain) +} diff --git a/vendor/github.com/pyke369/golang-support/rpack/rpack.go b/vendor/github.com/pyke369/golang-support/rpack/rpack.go new file mode 100644 index 0000000..998802a --- /dev/null +++ b/vendor/github.com/pyke369/golang-support/rpack/rpack.go @@ -0,0 +1,155 @@ +package rpack + +import ( + "bytes" + "compress/gzip" + "crypto/md5" + "encoding/base64" + "fmt" + "io/ioutil" + "math/rand" + "mime" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +type RPACK struct { + Compressed bool + Modified int64 + Mime string + Content string + raw []byte +} + +var guzpool = sync.Pool{ + New: func() interface{} { + return &gzip.Reader{} + }} + +func Pack(root, output, pkgname, funcname string, main bool) { + root = strings.TrimSuffix(root, "/") + if root == "" || output == "" { + return + } + if pkgname == "" || main { + pkgname = "main" + } + if funcname == "" { + funcname = "resources" + } + funcname = strings.ToUpper(funcname[:1]) + funcname[1:] + entries := map[string]*RPACK{} + compressor, _ := gzip.NewWriterLevel(nil, gzip.BestCompression) + count := 0 + size := int64(0) + start := time.Now() + filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + rpath := strings.TrimPrefix(path, root+"/") + if info.Mode()&os.ModeType == 0 { + for _, part := range strings.Split(rpath, "/") { + if len(part) > 0 && part[0] == '.' { + return nil + } + } + pack := &RPACK{Modified: info.ModTime().Unix(), Mime: "text/plain"} + if mime := mime.TypeByExtension(filepath.Ext(rpath)); mime != "" { + pack.Mime = mime + } + content, _ := ioutil.ReadFile(path) + compressed := bytes.Buffer{} + compressor.Reset(&compressed) + compressor.Write(content) + compressor.Close() + if compressed.Len() < len(content) { + pack.Content = base64.StdEncoding.EncodeToString(compressed.Bytes()) + pack.Compressed = true + } else { + pack.Content = base64.StdEncoding.EncodeToString(content) + } + entries[rpath] = pack + fmt.Fprintf(os.Stderr, "\r%-120.120s ", rpath) + count++ + size += info.Size() + } + return nil + }) + fmt.Fprintf(os.Stderr, "\r%-120.120s\rpacked %d file(s) %d byte(s) in %v\n", "", count, size, time.Now().Sub(start)) + if handle, err := os.OpenFile(output, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644); err == nil { + random := make([]byte, 64) + rand.Seed(time.Now().UnixNano() + int64(os.Getpid())) + rand.Read(random) + uid := fmt.Sprintf("rpack%8.8x", md5.Sum(random)) + fmt.Fprintf(handle, + `package %s + +import ( + "net/http" + "github.com/pyke369/golang-support/rpack" +) + +var %s map[string]*rpack.RPACK = map[string]*rpack.RPACK { +`, + pkgname, uid) + for path, entry := range entries { + fmt.Fprintf(handle, + ` "%s": &rpack.RPACK{Compressed:%t, Modified:%d, Mime:"%s", Content:"%s"}, +`, path, entry.Compressed, entry.Modified, entry.Mime, entry.Content) + } + fmt.Fprintf(handle, + `} + +func %s() http.Handler { + return rpack.Serve(%s) +} +`, funcname, uid) + if main { + fmt.Fprintf(handle, + ` +func main() { + http.Handle("/resources/", http.StripPrefix("/resources/", %s())) + http.ListenAndServe(":8000", nil) +} +`, funcname) + } + handle.Close() + } +} + +func Serve(pack map[string]*RPACK) http.Handler { + return http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + var err error + if pack == nil || pack[request.URL.Path] == nil { + response.WriteHeader(http.StatusNotFound) + return + } + if pack[request.URL.Path].raw == nil { + if pack[request.URL.Path].raw, err = base64.StdEncoding.DecodeString(pack[request.URL.Path].Content); err != nil { + response.WriteHeader(http.StatusNotFound) + return + } + } + resource := pack[request.URL.Path] + response.Header().Set("Content-Type", resource.Mime) + if strings.Index(request.Header.Get("Accept-Encoding"), "gzip") >= 0 && request.Header.Get("Range") == "" && resource.Compressed { + response.Header().Set("Content-Encoding", "gzip") + response.Header().Set("Content-Length", fmt.Sprintf("%d", len(resource.raw))) + http.ServeContent(response, request, request.URL.Path, time.Unix(resource.Modified, 0), bytes.NewReader(resource.raw)) + } else { + if resource.Compressed { + decompressor := guzpool.Get().(*gzip.Reader) + decompressor.Reset(bytes.NewReader(resource.raw)) + if raw, err := ioutil.ReadAll(decompressor); err == nil { + http.ServeContent(response, request, request.URL.Path, time.Unix(resource.Modified, 0), bytes.NewReader(raw)) + } + decompressor.Close() + guzpool.Put(decompressor) + } else { + http.ServeContent(response, request, request.URL.Path, time.Unix(resource.Modified, 0), bytes.NewReader(resource.raw)) + } + } + }) +} diff --git a/vendor/github.com/pyke369/golang-support/uconfig/README.md b/vendor/github.com/pyke369/golang-support/uconfig/README.md new file mode 100644 index 0000000..e69de29 diff --git a/vendor/github.com/pyke369/golang-support/uconfig/uconfig.go b/vendor/github.com/pyke369/golang-support/uconfig/uconfig.go new file mode 100644 index 0000000..ab52086 --- /dev/null +++ b/vendor/github.com/pyke369/golang-support/uconfig/uconfig.go @@ -0,0 +1,612 @@ +package uconfig + +import ( + "crypto/sha1" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "math" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strconv" + "strings" + "sync" + "time" +) + +type UConfig struct { + config interface{} + hash string + cache map[string]interface{} + cacheLock sync.RWMutex + sync.RWMutex +} + +type replacer struct { + search *regexp.Regexp + replace string + loop bool +} + +var ( + cores int + escaped string + unescaper *regexp.Regexp + requoter *regexp.Regexp + expander *regexp.Regexp + sizer *regexp.Regexp + duration1 *regexp.Regexp + duration2 *regexp.Regexp + replacers [16]replacer +) + +func init() { + escaped = "{}[],#/*;:= " // match characters within quotes to escape + unescaper = regexp.MustCompile("@\\d+@") // match escaped characters (to reverse previous escaping) + expander = regexp.MustCompile("{{([<|@&!\\-\\+])\\s*([^{}]*?)\\s*}}") // match external content macros + sizer = regexp.MustCompile("^(\\d+(?:\\.\\d*)?)\\s*([KMGTP]?)(B?)$") // match size value + duration1 = regexp.MustCompile("(\\d+)(Y|MO|D|H|MN|S|MS|US)?") // match duration value form1 (free) + duration2 = regexp.MustCompile("^(?:(\\d+):)?(\\d{2}):(\\d{2})(?:\\.(\\d{1,3}))?$") // match duration value form2 (timecode) + replacers[0] = replacer{regexp.MustCompile("(?m)^(.*?)(?:#|//).*?$"), "$1", false} // remove # and // commented portions + replacers[1] = replacer{regexp.MustCompile("/\\*[^\\*]*\\*/"), "", true} // remove /* */ commented portions + replacers[2] = replacer{regexp.MustCompile("(?m)^\\s+"), "", false} // trim leading spaces + replacers[3] = replacer{regexp.MustCompile("(?m)\\s+$"), "", false} // trim trailing spaces + replacers[4] = replacer{regexp.MustCompile("(?s)(^|[\r\n]+)\\[([^\\]\r\n]+?)\\](.+?)((?:[\r\n]+\\[)|$)"), "$1$2\n{$3\n}$4", true} // convert INI sections into JSON objects + replacers[5] = replacer{regexp.MustCompile("(?m)^(\\S+)\\s+([^{}\\[\\],;:=]+);$"), "$1 = $2;", false} // add missing key-value separators + replacers[6] = replacer{regexp.MustCompile("(?m);$"), ",", false} // replace ; line terminators by , + replacers[7] = replacer{regexp.MustCompile("(\\S+?)\\s*[:=]"), "$1:", false} // replace = key-value separators by : + replacers[8] = replacer{regexp.MustCompile("([}\\]])(\\s*)([^,}\\]\\s])"), "$1,$2$3", false} // add missing objects/arrays , separators + replacers[9] = replacer{regexp.MustCompile("(?m)(^[^:]+:.+?[^,])$"), "$1,", false} // add missing values trailing , seperators + replacers[10] = replacer{regexp.MustCompile("(^|[,{\\[]+\\s*)([^:{\\[]+?)(\\s*[{\\[])"), "$1$2:$3", true} // add missing key-(object/array-)value separator + replacers[11] = replacer{regexp.MustCompile("(?m)^([^\":{}\\[\\]]+)"), "\"$1\"", false} // add missing quotes around keys + replacers[12] = replacer{regexp.MustCompile("([:,\\[\\s]+)([^\",\\[\\]{}\n\r]+?)(\\s*[,\\]}])"), "$1\"$2\"$3", false} // add missing quotes around values + replacers[13] = replacer{regexp.MustCompile("\"[\r\n]"), "\",\n", false} // add still issing objects/arrays , separators + replacers[14] = replacer{regexp.MustCompile("\"\\s*(.+?)\\s*\""), "\"$1\"", false} // trim leading and trailing spaces in quoted strings + replacers[15] = replacer{regexp.MustCompile(",+(\\s*[}\\]])"), "$1", false} // remove objets/arrays last element extra , +} + +func escape(input string) string { + var output []byte + + instring := false + for index := 0; index < len(input); index++ { + if input[index:index+1] == "\"" && (index == 0 || input[index-1:index] != "\\") { + instring = !instring + } + if instring == true { + offset := strings.IndexAny(escaped, input[index:index+1]) + if offset >= 0 { + output = append(output, []byte(fmt.Sprintf("@%02d@", offset))...) + } else { + output = append(output, input[index:index+1]...) + } + } else { + output = append(output, input[index:index+1]...) + } + } + return string(output) +} + +func unescape(input string) string { + return unescaper.ReplaceAllStringFunc(input, func(a string) string { + offset, _ := strconv.Atoi(a[1:3]) + if offset < len(escaped) { + return escaped[offset : offset+1] + } + return "" + }) +} + +func reduce(input interface{}) { + if input != nil { + switch reflect.TypeOf(input).Kind() { + case reflect.Map: + for key, _ := range input.(map[string]interface{}) { + var parts []string + for _, value := range strings.Split(key, " ") { + if value != "" { + parts = append(parts, value) + } + } + if len(parts) > 1 { + if input.(map[string]interface{})[parts[0]] == nil || reflect.TypeOf(input.(map[string]interface{})[parts[0]]).Kind() != reflect.Map { + input.(map[string]interface{})[parts[0]] = make(map[string]interface{}) + } + input.(map[string]interface{})[parts[0]].(map[string]interface{})[parts[1]] = input.(map[string]interface{})[key] + delete(input.(map[string]interface{}), key) + } + } + for _, value := range input.(map[string]interface{}) { + reduce(value) + } + case reflect.Slice: + for index := 0; index < len(input.([]interface{})); index++ { + reduce(input.([]interface{})[index]) + } + } + } +} + +func New(input string, inline ...bool) (*UConfig, error) { + if cores == 0 { + cores = runtime.NumCPU() + } + config := &UConfig{ + config: nil, + } + return config, config.Load(input, inline...) +} + +func (this *UConfig) Load(input string, inline ...bool) error { + if cores > 1 { + this.Lock() + defer this.Unlock() + } + + this.cache = map[string]interface{}{} + base, _ := os.Getwd() + content := fmt.Sprintf("/*base:%s*/\n", base) + if len(inline) > 0 && inline[0] { + content += input + } else { + content += fmt.Sprintf("{{<%s}}", input) + } + + for { + indexes := expander.FindStringSubmatchIndex(content) + if indexes == nil { + content = escape(content) + for index := 0; index < len(replacers); index++ { + mcontent := "" + for mcontent != content { + mcontent = content + content = replacers[index].search.ReplaceAllString(content, replacers[index].replace) + if !replacers[index].loop { + break + } + } + } + content = unescape(strings.Trim(content, " \n,")) + break + } + + expanded := "" + arguments := strings.Split(content[indexes[4]:indexes[5]], " ") + if start := strings.LastIndex(content[0:indexes[2]], "/*base:"); start >= 0 { + if end := strings.Index(content[start+7:indexes[2]], "*/"); end > 0 { + base = content[start+7 : start+7+end] + } + } + switch content[indexes[2]:indexes[3]] { + case "<": + if arguments[0][0:1] != "/" { + arguments[0] = fmt.Sprintf("%s/%s", base, arguments[0]) + } + base = "" + if elements, err := filepath.Glob(arguments[0]); err == nil { + for _, element := range elements { + if mcontent, err := ioutil.ReadFile(element); err == nil { + base = filepath.Dir(element) + expanded += string(mcontent) + } + } + } + if base != "" && strings.Index(expanded, "\n") >= 0 { + expanded = fmt.Sprintf("/*base:%s*/\n%s\n", base, expanded) + } + case "|": + if arguments[0][0:1] != "/" { + arguments[0] = fmt.Sprintf("%s/%s", base, arguments[0]) + } + base = "" + if elements, err := filepath.Glob(arguments[0]); err == nil { + for _, element := range elements { + if element[0:1] != "/" { + element = fmt.Sprintf("%s/%s", base, element) + } + if mcontent, err := exec.Command(element, strings.Join(arguments[1:], " ")).Output(); err == nil { + base = filepath.Dir(element) + expanded += string(mcontent) + } + } + } + if base != "" && strings.Index(expanded, "\n") >= 0 { + expanded = fmt.Sprintf("/*base:%s*/\n%s\n", base, expanded) + } + case "@": + requester := http.Client{ + Timeout: time.Duration(5 * time.Second), + } + if response, err := requester.Get(arguments[0]); err == nil { + if (response.StatusCode / 100) == 2 { + if mcontent, err := ioutil.ReadAll(response.Body); err == nil { + expanded += string(mcontent) + } + } + } + case "&": + expanded += os.Getenv(arguments[0]) + case "!": + arguments[0] = strings.ToLower(arguments[0]) + for index := 1; index < len(os.Args); index++ { + option := strings.ToLower(os.Args[index]) + if option == "--"+arguments[0] { + if index == len(os.Args)-1 || strings.HasPrefix(os.Args[index+1], "--") { + expanded = "true" + } else { + expanded = os.Args[index+1] + } + break + } + } + case "-": + if index := strings.Index(os.Args[0], "-"); index >= 0 { + expanded = strings.ToLower(os.Args[0][index+1:]) + } + case "+": + if arguments[0][0:1] != "/" { + arguments[0] = fmt.Sprintf("%s/%s", base, arguments[0]) + } + if elements, err := filepath.Glob(arguments[0]); err == nil { + for _, element := range elements { + element = path.Base(element) + expanded += fmt.Sprintf("%s ", strings.TrimSuffix(element, filepath.Ext(element))) + } + } + expanded = strings.TrimSpace(expanded) + } + content = fmt.Sprintf("%s%s%s", content[0:indexes[0]], expanded, content[indexes[1]:len(content)]) + } + + this.hash = fmt.Sprintf("%x", sha1.Sum([]byte(content))) + if err := json.Unmarshal([]byte(content), &this.config); err != nil { + if err := json.Unmarshal([]byte("{"+content+"}"), &this.config); err != nil { + this.config = nil + if syntax, ok := err.(*json.SyntaxError); ok && syntax.Offset < int64(len(content)) { + if start := strings.LastIndex(content[:syntax.Offset], "\n") + 1; start >= 0 { + line := strings.Count(content[:start], "\n") + 1 + return errors.New(fmt.Sprintf("%s at line %d near %s", syntax, line, content[start:syntax.Offset])) + } + } + return err + } + } + + reduce(this.config) + return nil +} + +func (this *UConfig) Loaded() bool { + if cores > 1 { + this.RLock() + defer this.RUnlock() + } + return !(this.config == nil) +} + +func (this *UConfig) Hash() string { + if cores > 1 { + this.RLock() + defer this.RUnlock() + } + return this.hash +} + +func (this *UConfig) String() string { + if config, err := json.MarshalIndent(this.config, " ", " "); this.config != nil && err == nil { + return string(config) + } + return "{}" +} + +func (this *UConfig) GetPaths(path string) []string { + var ( + current interface{} = this.config + paths []string = []string{} + ) + + if cores > 1 { + this.RLock() + defer this.RUnlock() + } + prefix := "" + if current == nil || path == "" { + return paths + } + if cores > 1 { + this.cacheLock.RLock() + } + if this.cache[path] != nil { + if paths, ok := this.cache[path].([]string); ok { + if cores > 1 { + this.cacheLock.RUnlock() + } + return paths + } + } + if cores > 1 { + this.cacheLock.RUnlock() + } + if path != "" { + prefix = "." + for _, part := range strings.Split(path, ".") { + kind := reflect.TypeOf(current).Kind() + index, err := strconv.Atoi(part) + if (kind == reflect.Slice && (err != nil || index < 0 || index >= len(current.([]interface{})))) || (kind != reflect.Slice && kind != reflect.Map) { + if cores > 1 { + this.cacheLock.Lock() + } + this.cache[path] = paths + if cores > 1 { + this.cacheLock.Unlock() + } + return paths + } + if kind == reflect.Slice { + current = current.([]interface{})[index] + } else { + if current = current.(map[string]interface{})[strings.TrimSpace(part)]; current == nil { + if cores > 1 { + this.cacheLock.Lock() + } + this.cache[path] = paths + if cores > 1 { + this.cacheLock.Unlock() + } + return paths + } + } + } + } + switch reflect.TypeOf(current).Kind() { + case reflect.Slice: + for index := 0; index < len(current.([]interface{})); index++ { + paths = append(paths, fmt.Sprintf("%s%s%d", path, prefix, index)) + } + case reflect.Map: + for key, _ := range current.(map[string]interface{}) { + paths = append(paths, fmt.Sprintf("%s%s%s", path, prefix, key)) + } + } + if cores > 1 { + this.cacheLock.Lock() + } + this.cache[path] = paths + if cores > 1 { + this.cacheLock.Unlock() + } + return paths +} + +func (this *UConfig) value(path string) (string, error) { + var current interface{} = this.config + + if cores > 1 { + this.RLock() + defer this.RUnlock() + } + if current == nil || path == "" { + return "", fmt.Errorf("invalid parameter") + } + if cores > 1 { + this.cacheLock.RLock() + } + if this.cache[path] != nil { + if current, ok := this.cache[path].(bool); ok && !current { + if cores > 1 { + this.cacheLock.RUnlock() + } + return "", fmt.Errorf("invalid path") + } + if current, ok := this.cache[path].(string); ok { + if cores > 1 { + this.cacheLock.RUnlock() + } + return current, nil + } + } + if cores > 1 { + this.cacheLock.RUnlock() + } + for _, part := range strings.Split(path, ".") { + kind := reflect.TypeOf(current).Kind() + index, err := strconv.Atoi(part) + if (kind == reflect.Slice && (err != nil || index < 0 || index >= len(current.([]interface{})))) || (kind != reflect.Slice && kind != reflect.Map) { + if cores > 1 { + this.cacheLock.Lock() + } + this.cache[path] = false + if cores > 1 { + this.cacheLock.Unlock() + } + return "", fmt.Errorf("invalid path") + } + if kind == reflect.Slice { + current = current.([]interface{})[index] + } else { + if current = current.(map[string]interface{})[strings.TrimSpace(part)]; current == nil { + if cores > 1 { + this.cacheLock.Lock() + } + this.cache[path] = false + if cores > 1 { + this.cacheLock.Unlock() + } + return "", fmt.Errorf("invalid path") + } + } + } + if reflect.TypeOf(current).Kind() == reflect.String { + if cores > 1 { + this.cacheLock.Lock() + } + this.cache[path] = current.(string) + if cores > 1 { + this.cacheLock.Unlock() + } + return current.(string), nil + } + if cores > 1 { + this.cacheLock.Lock() + } + this.cache[path] = false + if cores > 1 { + this.cacheLock.Unlock() + } + return "", fmt.Errorf("invalid path") +} + +func (this *UConfig) GetBoolean(path string, fallback bool) bool { + value, err := this.value(path) + if err != nil { + return fallback + } + if value = strings.ToLower(strings.TrimSpace(value)); value == "1" || value == "on" || value == "yes" || value == "true" { + return true + } + return false +} + +func (this *UConfig) GetString(path string, fallback string) string { + return this.GetStringMatch(path, fallback, "") +} +func (this *UConfig) GetStringMatch(path string, fallback, match string) string { + return this.GetStringMatchCaptures(path, fallback, match)[0] +} +func (this *UConfig) GetStringMatchCaptures(path string, fallback, match string) []string { + value, err := this.value(path) + if err != nil { + return []string{fallback} + } + if match != "" { + if matcher, err := regexp.Compile(match); err == nil { + if matches := matcher.FindStringSubmatch(value); matches != nil { + return matches + } else { + return []string{fallback} + } + } else { + return []string{fallback} + } + } + return []string{value} +} + +func (this *UConfig) GetInteger(path string, fallback int64) int64 { + return this.GetIntegerBounds(path, fallback, math.MinInt64, math.MaxInt64) +} +func (this *UConfig) GetIntegerBounds(path string, fallback, min, max int64) int64 { + value, err := this.value(path) + if err != nil { + return fallback + } + nvalue, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64) + if err != nil { + return fallback + } + if nvalue < min { + nvalue = min + } + if nvalue > max { + nvalue = max + } + return nvalue +} + +func (this *UConfig) GetFloat(path string, fallback float64) float64 { + return this.GetFloatBounds(path, fallback, -math.MaxFloat64, math.MaxFloat64) +} +func (this *UConfig) GetFloatBounds(path string, fallback, min, max float64) float64 { + value, err := this.value(path) + if err != nil { + return fallback + } + nvalue, err := strconv.ParseFloat(strings.TrimSpace(value), 64) + if err != nil { + return fallback + } + return math.Max(math.Min(nvalue, max), min) +} + +func (this *UConfig) GetSize(path string, fallback int64) int64 { + return this.GetSizeBounds(path, fallback, math.MinInt64, math.MaxInt64) +} +func (this *UConfig) GetSizeBounds(path string, fallback, min, max int64) int64 { + value, err := this.value(path) + if err != nil { + return fallback + } + nvalue := int64(0) + if matches := sizer.FindStringSubmatch(strings.TrimSpace(strings.ToUpper(value))); matches != nil { + fvalue, err := strconv.ParseFloat(matches[1], 64) + if err != nil { + return fallback + } + scale := float64(1000) + if matches[3] == "B" { + scale = float64(1024) + } + nvalue = int64(fvalue * math.Pow(scale, float64(strings.Index("_KMGTP", matches[2])))) + } else { + return fallback + } + if nvalue < min { + nvalue = min + } + if nvalue > max { + nvalue = max + } + return nvalue +} + +func (this *UConfig) GetDuration(path string, fallback float64) float64 { + return this.GetDurationBounds(path, fallback, -math.MaxFloat64, math.MaxFloat64) +} +func (this *UConfig) GetDurationBounds(path string, fallback, min, max float64) float64 { + value, err := this.value(path) + if err != nil { + return fallback + } + nvalue := float64(0.0) + if matches := duration1.FindAllStringSubmatch(strings.TrimSpace(strings.ToUpper(value)), -1); matches != nil { + for index := 0; index < len(matches); index++ { + if uvalue, err := strconv.ParseFloat(matches[index][1], 64); err == nil { + switch matches[index][2] { + case "Y": + nvalue += uvalue * 86400 * 365.256 + case "MO": + nvalue += uvalue * 86400 * 365.256 / 12 + case "D": + nvalue += uvalue * 86400 + case "H": + nvalue += uvalue * 3600 + case "MN": + nvalue += uvalue * 60 + case "S": + nvalue += uvalue + case "": + nvalue += uvalue + case "MS": + nvalue += uvalue / 1000 + case "US": + nvalue += uvalue / 1000000 + } + } + } + } + if matches := duration2.FindStringSubmatch(strings.TrimSpace(value)); matches != nil { + hours, _ := strconv.ParseFloat(matches[1], 64) + minutes, _ := strconv.ParseFloat(matches[2], 64) + seconds, _ := strconv.ParseFloat(matches[3], 64) + milliseconds, _ := strconv.ParseFloat(matches[4], 64) + nvalue = (hours * 3600) + (math.Min(minutes, 59) * 60) + math.Min(seconds, 59) + (milliseconds / 1000) + } + return math.Max(math.Min(nvalue, max), min) +} diff --git a/vendor/github.com/pyke369/golang-support/ulog/README.md b/vendor/github.com/pyke369/golang-support/ulog/README.md new file mode 100644 index 0000000..e69de29 diff --git a/vendor/github.com/pyke369/golang-support/ulog/ulog.go b/vendor/github.com/pyke369/golang-support/ulog/ulog.go new file mode 100644 index 0000000..59033f3 --- /dev/null +++ b/vendor/github.com/pyke369/golang-support/ulog/ulog.go @@ -0,0 +1,545 @@ +package ulog + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "log/syslog" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "sync" + "syscall" + "time" + "unsafe" +) + +const ( + TIME_NONE int = iota + TIME_DATETIME + TIME_MSDATETIME + TIME_TIMESTAMP + TIME_MSTIMESTAMP +) + +var ( + cores int + facilities = map[string]syslog.Priority{ + "user": syslog.LOG_USER, + "daemon": syslog.LOG_DAEMON, + "local0": syslog.LOG_LOCAL0, + "local1": syslog.LOG_LOCAL1, + "local2": syslog.LOG_LOCAL2, + "local3": syslog.LOG_LOCAL3, + "local4": syslog.LOG_LOCAL4, + "local5": syslog.LOG_LOCAL5, + "local6": syslog.LOG_LOCAL6, + "local7": syslog.LOG_LOCAL7, + } + severities = map[string]syslog.Priority{ + "error": syslog.LOG_ERR, + "warning": syslog.LOG_WARNING, + "info": syslog.LOG_INFO, + "debug": syslog.LOG_DEBUG, + } + severityLabels = map[syslog.Priority]string{ + syslog.LOG_ERR: "ERRO ", + syslog.LOG_WARNING: "WARN ", + syslog.LOG_INFO: "INFO ", + syslog.LOG_DEBUG: "DBUG ", + } + severityColors = map[syslog.Priority]string{ + syslog.LOG_ERR: "\x1b[31m", + syslog.LOG_WARNING: "\x1b[33m", + syslog.LOG_INFO: "\x1b[36m", + syslog.LOG_DEBUG: "\x1b[32m", + } +) + +type FileOutput struct { + handle *os.File + last time.Time +} +type ULog struct { + file, console, syslog bool + fileOutputs map[string]*FileOutput + filePath string + fileTime int + fileLast time.Time + fileSeverity bool + consoleHandle io.Writer + consoleTime int + consoleSeverity bool + consoleColors bool + syslogHandle *syslog.Writer + syslogRemote string + syslogName string + syslogFacility syslog.Priority + optionUTC bool + level syslog.Priority + sync.Mutex +} + +func New(target string) *ULog { + if cores == 0 { + cores = runtime.NumCPU() + } + log := &ULog{ + fileOutputs: map[string]*FileOutput{}, + syslogHandle: nil, + } + return log.Load(target) +} + +func (this *ULog) Load(target string) *ULog { + this.Close() + if cores > 1 { + this.Lock() + defer this.Unlock() + } + + this.file = false + this.filePath = "" + this.fileTime = TIME_DATETIME + this.fileSeverity = true + this.console = false + this.consoleTime = TIME_DATETIME + this.consoleSeverity = true + this.consoleColors = true + this.consoleHandle = os.Stderr + this.syslog = false + this.syslogRemote = "" + this.syslogName = filepath.Base(os.Args[0]) + this.syslogFacility = syslog.LOG_DAEMON + this.optionUTC = false + this.level = syslog.LOG_INFO + for _, target := range regexp.MustCompile("(file|console|syslog|option)\\s*\\(([^\\)]*)\\)").FindAllStringSubmatch(target, -1) { + switch strings.ToLower(target[1]) { + case "file": + this.file = true + for _, option := range regexp.MustCompile("([^:=,\\s]+)\\s*[:=]\\s*([^,\\s]+)").FindAllStringSubmatch(target[2], -1) { + switch strings.ToLower(option[1]) { + case "path": + this.filePath = option[2] + case "time": + option[2] = strings.ToLower(option[2]) + switch { + case option[2] == "datetime": + this.fileTime = TIME_DATETIME + case option[2] == "msdatetime": + this.fileTime = TIME_MSDATETIME + case option[2] == "stamp" || option[2] == "timestamp": + this.fileTime = TIME_TIMESTAMP + case option[2] == "msstamp" || option[2] == "mstimestamp": + this.fileTime = TIME_MSTIMESTAMP + case option[2] != "1" && option[2] != "true" && option[2] != "on" && option[2] != "yes": + this.fileTime = TIME_NONE + } + case "severity": + option[2] = strings.ToLower(option[2]) + if option[2] != "1" && option[2] != "true" && option[2] != "on" && option[2] != "yes" { + this.fileSeverity = false + } + } + } + if this.filePath == "" { + this.file = false + } + case "console": + this.console = true + for _, option := range regexp.MustCompile("([^:=,\\s]+)\\s*[:=]\\s*([^,\\s]+)").FindAllStringSubmatch(target[2], -1) { + option[2] = strings.ToLower(option[2]) + switch strings.ToLower(option[1]) { + case "output": + if option[2] == "stdout" { + this.consoleHandle = os.Stdout + } + case "time": + switch { + case option[2] == "datetime": + this.consoleTime = TIME_DATETIME + case option[2] == "msdatetime": + this.consoleTime = TIME_MSDATETIME + case option[2] == "stamp" || option[2] == "timestamp": + this.consoleTime = TIME_TIMESTAMP + case option[2] == "msstamp" || option[2] == "mstimestamp": + this.consoleTime = TIME_MSTIMESTAMP + case option[2] != "1" && option[2] != "true" && option[2] != "on" && option[2] != "yes": + this.consoleTime = TIME_NONE + } + case "severity": + if option[2] != "1" && option[2] != "true" && option[2] != "on" && option[2] != "yes" { + this.consoleSeverity = false + } + case "colors": + if option[2] != "1" && option[2] != "true" && option[2] != "on" && option[2] != "yes" { + this.consoleColors = false + } + } + } + case "syslog": + this.syslog = true + for _, option := range regexp.MustCompile("([^:=,\\s]+)\\s*[:=]\\s*([^,\\s]+)").FindAllStringSubmatch(target[2], -1) { + switch strings.ToLower(option[1]) { + case "remote": + this.syslogRemote = option[2] + if !regexp.MustCompile(":\\d+$").MatchString(this.syslogRemote) { + this.syslogRemote += ":514" + } + case "name": + this.syslogName = option[2] + case "facility": + this.syslogFacility = facilities[strings.ToLower(option[2])] + } + } + case "option": + for _, option := range regexp.MustCompile("([^:=,\\s]+)\\s*[:=]\\s*([^,\\s]+)").FindAllStringSubmatch(target[2], -1) { + option[2] = strings.ToLower(option[2]) + switch strings.ToLower(option[1]) { + case "utc": + if option[2] == "1" || option[2] == "true" || option[2] == "on" || option[2] == "yes" { + this.optionUTC = true + } + case "level": + this.level = severities[strings.ToLower(option[2])] + } + } + } + } + + var info syscall.Termios + if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, this.consoleHandle.(*os.File).Fd(), syscall.TCGETS, uintptr(unsafe.Pointer(&info)), 0, 0, 0); err != 0 { + this.consoleColors = false + } + return this +} + +func (this *ULog) Close() { + if cores > 1 { + this.Lock() + defer this.Unlock() + } + if this.syslogHandle != nil { + this.syslogHandle.Close() + this.syslogHandle = nil + } + for path, output := range this.fileOutputs { + if output.handle != nil { + output.handle.Close() + } + delete(this.fileOutputs, path) + } +} + +func (this *ULog) SetLevel(level string) { + level = strings.ToLower(level) + switch level { + case "error": + this.level = syslog.LOG_ERR + case "warning": + this.level = syslog.LOG_WARNING + case "info": + this.level = syslog.LOG_INFO + case "debug": + this.level = syslog.LOG_DEBUG + } +} + +func strftime(layout string, base time.Time) string { + var output []string + + length := len(layout) + for index := 0; index < length; index++ { + switch layout[index] { + case '%': + if index < length-1 { + switch layout[index+1] { + case 'a': + output = append(output, base.Format("Mon")) + case 'A': + output = append(output, base.Format("Monday")) + case 'b': + output = append(output, base.Format("Jan")) + case 'B': + output = append(output, base.Format("January")) + case 'c': + output = append(output, base.Format("Mon Jan 2 15:04:05 2006")) + case 'C': + output = append(output, fmt.Sprintf("%02d", base.Year()/100)) + case 'd': + output = append(output, fmt.Sprintf("%02d", base.Day())) + case 'D': + output = append(output, fmt.Sprintf("%02d/%02d/%02d", base.Month(), base.Day(), base.Year()%100)) + case 'e': + output = append(output, fmt.Sprintf("%2d", base.Day())) + case 'f': + output = append(output, fmt.Sprintf("%06d", base.Nanosecond()/1000)) + case 'F': + output = append(output, fmt.Sprintf("%04d-%02d-%02d", base.Year(), base.Month(), base.Day())) + case 'g': + year, _ := base.ISOWeek() + output = append(output, fmt.Sprintf("%02d", year%100)) + case 'G': + year, _ := base.ISOWeek() + output = append(output, fmt.Sprintf("%04d", year)) + case 'h': + output = append(output, base.Format("Jan")) + case 'H': + output = append(output, fmt.Sprintf("%02d", base.Hour())) + case 'I': + if base.Hour() == 0 || base.Hour() == 12 { + output = append(output, "12") + } else { + output = append(output, fmt.Sprintf("%02d", base.Hour()%12)) + } + case 'j': + output = append(output, fmt.Sprintf("%03d", base.YearDay())) + case 'k': + output = append(output, fmt.Sprintf("%2d", base.Hour())) + case 'l': + if base.Hour() == 0 || base.Hour() == 12 { + output = append(output, "12") + } else { + output = append(output, fmt.Sprintf("%2d", base.Hour()%12)) + } + case 'm': + output = append(output, fmt.Sprintf("%02d", base.Month())) + case 'M': + output = append(output, fmt.Sprintf("%02d", base.Minute())) + case 'n': + output = append(output, "\n") + case 'p': + if base.Hour() < 12 { + output = append(output, "AM") + } else { + output = append(output, "PM") + } + case 'P': + if base.Hour() < 12 { + output = append(output, "am") + } else { + output = append(output, "pm") + } + case 'r': + if base.Hour() == 0 || base.Hour() == 12 { + output = append(output, "12") + } else { + output = append(output, fmt.Sprintf("%02d", base.Hour()%12)) + } + output = append(output, fmt.Sprintf(":%02d:%02d", base.Minute(), base.Second())) + if base.Hour() < 12 { + output = append(output, " AM") + } else { + output = append(output, " PM") + } + case 'R': + output = append(output, fmt.Sprintf("%02d:%02d", base.Hour(), base.Minute())) + case 's': + output = append(output, fmt.Sprintf("%d", base.Unix())) + case 'S': + output = append(output, fmt.Sprintf("%02d", base.Second())) + case 't': + output = append(output, "\t") + case 'T': + output = append(output, fmt.Sprintf("%02d:%02d:%02d", base.Hour(), base.Minute(), base.Second())) + case 'u': + day := base.Weekday() + if day == 0 { + day = 7 + } + output = append(output, fmt.Sprintf("%d", day)) + case 'U': + output = append(output, fmt.Sprintf("%d", (base.YearDay()+6-int(base.Weekday()))/7)) + case 'V': + _, week := base.ISOWeek() + output = append(output, fmt.Sprintf("%02d", week)) + case 'w': + output = append(output, fmt.Sprintf("%d", base.Weekday())) + case 'W': + day := int(base.Weekday()) + if day == 0 { + day = 6 + } else { + day -= 1 + } + output = append(output, fmt.Sprintf("%d", (base.YearDay()+6-day)/7)) + case 'x': + output = append(output, fmt.Sprintf("%02d/%02d/%02d", base.Month(), base.Day(), base.Year()%100)) + case 'X': + output = append(output, fmt.Sprintf("%02d:%02d:%02d", base.Hour(), base.Minute(), base.Second())) + case 'y': + output = append(output, fmt.Sprintf("%02d", base.Year()%100)) + case 'Y': + output = append(output, fmt.Sprintf("%04d", base.Year())) + case 'z': + output = append(output, base.Format("-0700")) + case 'Z': + output = append(output, base.Format("MST")) + case '%': + output = append(output, "%") + } + index++ + } + default: + output = append(output, string(layout[index])) + } + } + return strings.Join(output, "") +} + +func (this *ULog) log(now time.Time, severity syslog.Priority, xlayout interface{}, a ...interface{}) { + var err error + if this.level < severity || (!this.syslog && !this.file && !this.console) { + return + } + layout := "" + switch reflect.TypeOf(xlayout).Kind() { + case reflect.Map: + var buffer bytes.Buffer + + encoder := json.NewEncoder(&buffer) + encoder.SetEscapeHTML(false) + if err := encoder.Encode(xlayout); err == nil { + layout = "%s" + a = []interface{}{bytes.TrimSpace(buffer.Bytes())} + } + case reflect.String: + layout = xlayout.(string) + } + layout = strings.TrimSpace(layout) + if this.syslog { + if this.syslogHandle == nil { + if cores > 1 { + this.Lock() + } + if this.syslogHandle == nil { + protocol := "" + if this.syslogRemote != "" { + protocol = "udp" + } + if this.syslogHandle, err = syslog.Dial(protocol, this.syslogRemote, this.syslogFacility, this.syslogName); err != nil { + this.syslogHandle = nil + } + } + if cores > 1 { + this.Unlock() + } + } + if this.syslogHandle != nil { + switch severity { + case syslog.LOG_ERR: + this.syslogHandle.Err(fmt.Sprintf(layout, a...)) + case syslog.LOG_WARNING: + this.syslogHandle.Warning(fmt.Sprintf(layout, a...)) + case syslog.LOG_INFO: + this.syslogHandle.Info(fmt.Sprintf(layout, a...)) + case syslog.LOG_DEBUG: + this.syslogHandle.Debug(fmt.Sprintf(layout, a...)) + } + } + } + if this.optionUTC { + now = now.UTC() + } else { + now = now.Local() + } + if this.file { + path := strftime(this.filePath, now) + if cores > 1 { + this.Lock() + } + if this.fileOutputs[path] == nil { + os.MkdirAll(filepath.Dir(path), 0755) + if handle, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644); err == nil { + this.fileOutputs[path] = &FileOutput{handle: handle} + } + } + if this.fileOutputs[path] != nil && this.fileOutputs[path].handle != nil { + prefix := "" + switch this.fileTime { + case TIME_DATETIME: + prefix = fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d ", now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second()) + case TIME_MSDATETIME: + prefix = fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d.%03d ", now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second(), now.Nanosecond()/int(time.Millisecond)) + case TIME_TIMESTAMP: + prefix = fmt.Sprintf("%d ", now.Unix()) + case TIME_MSTIMESTAMP: + prefix = fmt.Sprintf("%d ", now.UnixNano()/int64(time.Millisecond)) + } + if this.fileSeverity { + prefix += severityLabels[severity] + } + this.fileOutputs[path].handle.WriteString(fmt.Sprintf(prefix+layout+"\n", a...)) + this.fileOutputs[path].last = now + } + if now.Sub(this.fileLast) >= 5*time.Second { + this.fileLast = now + for path, output := range this.fileOutputs { + if now.Sub(output.last) >= 5*time.Second { + output.handle.Close() + delete(this.fileOutputs, path) + } + } + } + if cores > 1 { + this.Unlock() + } + } + if this.console { + prefix := "" + switch this.consoleTime { + case TIME_DATETIME: + prefix = fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d ", now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second()) + case TIME_MSDATETIME: + prefix = fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d.%03d ", now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second(), now.Nanosecond()/int(time.Millisecond)) + case TIME_TIMESTAMP: + prefix = fmt.Sprintf("%d ", now.Unix()) + case TIME_MSTIMESTAMP: + prefix = fmt.Sprintf("%d ", now.UnixNano()/int64(time.Millisecond)) + } + if this.consoleSeverity { + if this.consoleColors { + prefix += fmt.Sprintf("%s%s\x1b[0m", severityColors[severity], severityLabels[severity]) + } else { + prefix += severityLabels[severity] + } + } + if cores > 1 { + this.Lock() + } + fmt.Fprintf(this.consoleHandle, prefix+layout+"\n", a...) + if cores > 1 { + this.Unlock() + } + } +} + +func (this *ULog) Error(layout interface{}, a ...interface{}) { + this.log(time.Now(), syslog.LOG_ERR, layout, a...) +} +func (this *ULog) Warn(layout interface{}, a ...interface{}) { + this.log(time.Now(), syslog.LOG_WARNING, layout, a...) +} +func (this *ULog) Info(layout interface{}, a ...interface{}) { + this.log(time.Now(), syslog.LOG_INFO, layout, a...) +} +func (this *ULog) Debug(layout interface{}, a ...interface{}) { + this.log(time.Now(), syslog.LOG_DEBUG, layout, a...) +} + +func (this *ULog) ErrorTime(now time.Time, layout interface{}, a ...interface{}) { + this.log(now, syslog.LOG_ERR, layout, a...) +} +func (this *ULog) WarnTime(now time.Time, layout interface{}, a ...interface{}) { + this.log(now, syslog.LOG_WARNING, layout, a...) +} +func (this *ULog) InfoTime(now time.Time, layout interface{}, a ...interface{}) { + this.log(now, syslog.LOG_INFO, layout, a...) +} +func (this *ULog) DebugTime(now time.Time, layout interface{}, a ...interface{}) { + this.log(now, syslog.LOG_DEBUG, layout, a...) +} diff --git a/vendor/github.com/pyke369/golang-support/uuid/README.md b/vendor/github.com/pyke369/golang-support/uuid/README.md new file mode 100644 index 0000000..e69de29 diff --git a/vendor/github.com/pyke369/golang-support/uuid/uuid.go b/vendor/github.com/pyke369/golang-support/uuid/uuid.go new file mode 100644 index 0000000..5814371 --- /dev/null +++ b/vendor/github.com/pyke369/golang-support/uuid/uuid.go @@ -0,0 +1,30 @@ +package uuid + +import ( + "fmt" + "math/rand" + "os" + "time" +) + +var initialized bool + +func init() { + if !initialized { + rand.Seed(time.Now().UnixNano() + int64(os.Getpid())) + initialized = true + } +} + +func UUID() string { + var entropy = make([]byte, 16) + + if !initialized { + rand.Seed(time.Now().UnixNano() + int64(os.Getpid())) + initialized = true + } + rand.Read(entropy) + entropy[6] = (entropy[6] & 0x0f) | 0x40 + entropy[8] = (entropy[8] & 0x3f) | 0x80 + return fmt.Sprintf("%08x-%04x-%04x-%04x-%12x", entropy[0:4], entropy[4:6], entropy[6:8], entropy[8:10], entropy[10:16]) +} diff --git a/vendor/github.com/pyke369/golang-support/whohas/README.md b/vendor/github.com/pyke369/golang-support/whohas/README.md new file mode 100644 index 0000000..e69de29 diff --git a/vendor/github.com/pyke369/golang-support/whohas/whohas.go b/vendor/github.com/pyke369/golang-support/whohas/whohas.go new file mode 100644 index 0000000..a40892a --- /dev/null +++ b/vendor/github.com/pyke369/golang-support/whohas/whohas.go @@ -0,0 +1,244 @@ +package whohas + +import ( + "context" + "mime" + "net/http" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "time" +) + +type BACKEND struct { + Host string + Secure bool + Path string + Headers map[string]string + Penalty time.Duration +} +type CACHE struct { + TTL time.Duration + last time.Time + items map[string]*LOOKUP + sync.RWMutex +} +type LOOKUP struct { + index int + deadline time.Time + Protocol string + Host string + Headers map[string]string + Size int64 + Mime string + Ranges bool + Date time.Time + Modified time.Time + Expires time.Time +} + +var cores int + +func Lookup(path string, backends []BACKEND, timeout time.Duration, cache *CACHE, ckey string) (lookup *LOOKUP) { + if cores == 0 { + cores = runtime.NumCPU() + } + if path == "" || backends == nil || len(backends) < 1 || timeout < 100*time.Millisecond { + return + } + + cpath := path + if index := strings.Index(path, "?"); index >= 0 { + cpath = path[:index] + } + cbackends := backends + if cache != nil && cache.items != nil { + now := time.Now() + if cores > 1 { + cache.RLock() + } + if cache.items[cpath] != nil && now.Sub(cache.items[cpath].deadline) < 0 { + lookup = cache.items[cpath] + if cache.items[cpath].Host != "" { + cbackends = []BACKEND{} + for _, backend := range backends { + if backend.Host == cache.items[cpath].Host { + cbackends = append(cbackends, backend) + break + } + } + if len(cbackends) < 1 { + cbackends = backends + } + } + } + if len(cbackends) == len(backends) && ckey != "" && cache.items["k"+ckey] != nil && now.Sub(cache.items["k"+ckey].deadline) < 0 { + cbackends = []BACKEND{} + for _, backend := range backends { + if backend.Host == cache.items["k"+ckey].Host { + cbackends = append(cbackends, backend) + break + } + } + if len(cbackends) < 1 { + cbackends = backends + } + } + if cores > 1 { + cache.RUnlock() + } + } + + if lookup == nil { + inflight := len(cbackends) + sink := make(chan LOOKUP, inflight+1) + cancels := make([]context.CancelFunc, inflight) + for index, backend := range cbackends { + var ctx context.Context + + ctx, cancels[index] = context.WithCancel(context.Background()) + go func(index int, backend BACKEND, ctx context.Context) { + lookup := LOOKUP{index: index} + if backend.Penalty != 0 && len(cbackends) > 1 { + select { + case <-time.After(backend.Penalty): + case <-ctx.Done(): + sink <- lookup + return + } + } + lookup.Protocol = "http" + if backend.Secure { + lookup.Protocol = "https" + } + rpath := path + if backend.Path != "" { + rpath = backend.Path + } + if request, err := http.NewRequest(http.MethodHead, lookup.Protocol+"://"+backend.Host+rpath, nil); err == nil { + request = request.WithContext(ctx) + request.Header.Set("User-Agent", "whohas") + if backend.Headers != nil { + lookup.Headers = map[string]string{} + for name, value := range backend.Headers { + lookup.Headers[name] = value + request.Header.Set(name, value) + } + } + if response, err := http.DefaultClient.Do(request); err == nil { + if response.StatusCode == 200 { + lookup.Host = backend.Host + lookup.Size, _ = strconv.ParseInt(response.Header.Get("Content-Length"), 10, 64) + lookup.Mime = response.Header.Get("Content-Type") + if lookup.Mime == "" || lookup.Mime == "application/octet-stream" || lookup.Mime == "text/plain" { + if extension := filepath.Ext(path); extension != "" { + lookup.Mime = mime.TypeByExtension(extension) + } + } + if response.Header.Get("Accept-Ranges") != "" { + lookup.Ranges = true + } + if header := response.Header.Get("Date"); header != "" { + lookup.Date, _ = http.ParseTime(header) + } else { + lookup.Date = time.Now() + } + if header := response.Header.Get("Last-Modified"); header != "" { + lookup.Modified, _ = http.ParseTime(header) + } + if header := response.Header.Get("Expires"); header != "" { + lookup.Expires, _ = http.ParseTime(header) + } else { + lookup.Expires = lookup.Date.Add(time.Hour) + } + if lookup.Expires.Sub(lookup.Date) < 2*time.Second { + lookup.Expires = lookup.Date.Add(2 * time.Second) + } + } + response.Body.Close() + } + } + sink <- lookup + }(index, backend, ctx) + } + + for inflight > 0 { + select { + case result := <-sink: + inflight-- + cancels[result.index] = nil + if result.Host != "" { + lookup = &result + for index, cancel := range cancels { + if cancels[index] != nil && index != result.index { + cancel() + cancels[index] = nil + } + } + } + case <-time.After(timeout): + for index, cancel := range cancels { + if cancels[index] != nil { + cancel() + } + } + } + } + close(sink) + } + + if cache != nil { + now := time.Now() + if cores > 1 { + cache.Lock() + } + if cache.items == nil { + cache.items = map[string]*LOOKUP{} + } + if now.Sub(cache.last) >= 5*time.Second { + cache.last = now + for key, item := range cache.items { + if now.Sub(item.deadline) >= 0 { + delete(cache.items, key) + } + } + } + if lookup == nil || lookup.Host == "" { + if ckey != "" { + delete(cache.items, "k"+ckey) + } + if cache.items[cpath] == nil { + cache.items[cpath] = &LOOKUP{deadline: now.Add(5 * time.Second)} + } + lookup = nil + } else { + if cache.TTL < 2*time.Second { + cache.TTL = 2 * time.Second + } + if ckey != "" { + cache.items["k"+ckey] = &LOOKUP{Host: lookup.Host, deadline: now.Add(cache.TTL)} + } + if cache.items[cpath] == nil { + ttl := lookup.Expires.Sub(lookup.Date) + if ttl < 2*time.Second { + ttl = 2 * time.Second + } + if ttl > cache.TTL { + ttl = cache.TTL + } + if ttl > 10*time.Minute { + ttl = 10 * time.Minute + } + lookup.deadline = now.Add(ttl) + cache.items[cpath] = lookup + } + } + if cores > 1 { + cache.Unlock() + } + } + + return +}