euclide
/
openvpn-mgt
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

read configuration file

This commit is contained in:
Xavier Henner 2019-07-08 22:32:12 +02:00
parent 9dc7d19811
commit dd38706b0b
30 changed files with 3621 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
go.mod Normal file
View File

@ -0,0 +1 @@
module github.com/pyke369/golang-support/uconfig

49
httpd.go
View File

@ -1,15 +1,24 @@
package main package main
import ( import (
"bufio"
"bytes"
"crypto/tls"
"crypto/x509"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io"
"log" "log"
"net/http" "net/http"
"os"
) )
type HttpServer struct { type HttpServer struct {
Port string Port string
ovpn *OpenVpnMgt ovpn *OpenVpnMgt
key string
cert string
certPool *x509.CertPool
} }
func (h *HttpServer) handler(w http.ResponseWriter, r *http.Request) { func (h *HttpServer) handler(w http.ResponseWriter, r *http.Request) {
@ -33,19 +42,51 @@ func (h *HttpServer) helpHandler(w http.ResponseWriter, r *http.Request) {
err, message := h.ovpn.Help() err, message := h.ovpn.Help()
if err != nil { if err != nil {
fmt.Fprintf(w, "Error : %s", err) fmt.Fprintf(w, "Error : %s", err)
} else {
fmt.Fprintf(w, "%s", message)
} }
jsonStr, err := json.Marshal(message)
if err != nil {
fmt.Fprintf(w, "Error : %s", err)
}
fmt.Fprintf(w, "%s", jsonStr)
} }
func NewHTTPServer(port string, s *OpenVpnMgt) { func NewHTTPServer(port, key, cert, ca string, s *OpenVpnMgt) {
h := &HttpServer{ h := &HttpServer{
Port: port, Port: port,
ovpn: s, ovpn: s,
key: key,
cert: cert,
} }
http.HandleFunc("/help", h.helpHandler) http.HandleFunc("/help", h.helpHandler)
http.HandleFunc("/version", h.versionHandler) http.HandleFunc("/version", h.versionHandler)
http.HandleFunc("/", h.handler) http.HandleFunc("/", h.handler)
switch {
case key == "" || cert == "":
log.Fatal(http.ListenAndServe(port, nil)) log.Fatal(http.ListenAndServe(port, nil))
case ca != "":
h.certPool = x509.NewCertPool()
fi, err := os.Open(ca)
if err != nil {
log.Fatal(err)
}
defer fi.Close()
buf := new(bytes.Buffer)
reader := bufio.NewReader(fi)
io.Copy(buf, reader)
if ok := h.certPool.AppendCertsFromPEM(buf.Bytes()); !ok {
log.Fatal("Failed to append PEM.")
}
server := &http.Server{
Addr: port,
TLSConfig: &tls.Config{
ClientAuth: tls.RequestClientCert,
ClientCAs: h.certPool,
},
}
log.Fatal(server.ListenAndServeTLS(cert, key))
default:
log.Fatal(http.ListenAndServeTLS(port, cert, key, nil))
}
} }

54
main.go
View File

@ -1,7 +1,57 @@
package main package main
import (
"flag"
"log"
"log/syslog"
"os"
"github.com/pyke369/golang-support/uconfig"
)
var config *uconfig.UConfig
func main() { func main() {
server := NewVPNServer(":4000") var err error
// default configuration file is ./openvpn-dm-mgt-server.conf
configFile := flag.String("config", "openvpn-dm-mgt-server.conf", "configuration file")
logToSyslog := flag.Bool("syslog", false, "Log to syslog")
flag.Parse()
// parseconfig
if config, err = uconfig.New(*configFile); err != nil {
log.Println(err)
os.Exit(1)
}
server := NewVPNServer(config.GetString("config.openvpnPort", "127.0.0.01:5000"))
server.vpnlogUrl = config.GetString("config.vpnLogUrl", "")
server.mailRelay = config.GetString("config.mailRelay", "")
server.MailFrom = config.GetString("config.mailFrom", "")
server.CcPwnPassword = config.GetString("config.ccPwnPassword", "")
server.pwnTemplate = config.GetString("config.pwnTemplate", "")
server.newAsTemplate = config.GetString("config.newAsTemplate", "")
server.slackTemplate = config.GetString("config.slackTemplate", "")
server.slackTemplate2 = config.GetString("config.slackTemplate2", "")
server.cacheDir = config.GetString("config.cacheDir", "")
server.authCa = config.GetString("config.authCa", "")
server.syslog = false
if *logToSyslog {
log.SetFlags(0)
server.syslog = true
logWriter, e := syslog.New(syslog.LOG_NOTICE, "")
if e == nil {
log.SetOutput(logWriter)
defer logWriter.Close()
}
}
go server.Run() go server.Run()
NewHTTPServer(":8080", server) NewHTTPServer(
config.GetString("config.httpPort", "127.0.0.01:8080"),
config.GetString("config.httpKey", ""),
config.GetString("config.httpCert", ""),
config.GetString("config.httpCa", ""),
server)
} }

12
tcpserver.go → openvpn.go
View File

@ -3,10 +3,10 @@ package main
import ( import (
"bufio" "bufio"
"errors" "errors"
"fmt"
"io" "io"
"log" "log"
"net" "net"
"os"
"strings" "strings"
"sync" "sync"
) )
@ -18,12 +18,16 @@ type OpenVpnMgt struct {
connected bool connected bool
m sync.RWMutex m sync.RWMutex
ret chan []string ret chan []string
authCa string
vpnlogUrl string vpnlogUrl string
mailRelay string mailRelay string
MailFrom string MailFrom string
CcPwnPassword string CcPwnPassword string
pwnTemplate string pwnTemplate string
newAsTemplate string newAsTemplate string
slackTemplate string
slackTemplate2 string
cacheDir string
syslog bool syslog bool
} }
@ -40,12 +44,14 @@ func (s *OpenVpnMgt) Run() {
// Resolve the passed port into an address // Resolve the passed port into an address
addrs, err := net.ResolveTCPAddr("tcp", s.Port) addrs, err := net.ResolveTCPAddr("tcp", s.Port)
if err != nil { if err != nil {
return log.Println(err)
os.Exit(1)
} }
// start listening to client connections // start listening to client connections
listener, err := net.ListenTCP("tcp", addrs) listener, err := net.ListenTCP("tcp", addrs)
if err != nil { if err != nil {
fmt.Println(err) log.Println(err)
os.Exit(1)
} }
// Infinite loop since we dont want the server to shut down // Infinite loop since we dont want the server to shut down
for { for {

0
vendor/github.com/pyke369/golang-support/README.md generated vendored Normal file
View File

89
vendor/github.com/pyke369/golang-support/bslab/bslab.go generated vendored Normal file
View File

@ -0,0 +1,89 @@
package bslab
import "sync/atomic"
type slab struct {
queue chan []byte
get, put, alloc, lost int64
}
var (
slabs = map[int]*slab{}
)
func init() {
slabs[0] = &slab{}
for size := uint(9); size <= 26; size++ {
slabs[1<<size] = &slab{queue: make(chan []byte, 1024)}
}
}
func Stats() (info map[int][5]int64) {
info = map[int][5]int64{}
for size, slab := range slabs {
info[size] = [5]int64{slab.get, slab.put, slab.alloc, slab.lost, int64(len(slab.queue))}
}
return info
}
func Get(size int, item []byte) []byte {
if size <= 0 {
return nil
}
if item != nil {
if cap(item) >= size {
return item[:0]
}
Put(item)
}
bits, power := uint(0), uint(0)
if size&(size-1) == 0 {
power = 1
}
for size != 0 {
size >>= 1
bits++
}
size = 1 << (bits - power)
if slab, ok := slabs[size]; ok {
atomic.AddInt64(&(slab.get), 1)
select {
case item := <-slab.queue:
return item[:0]
default:
atomic.AddInt64(&(slab.alloc), 1)
return make([]byte, 0, size)
}
}
atomic.AddInt64(&(slabs[0].get), 1)
atomic.AddInt64(&(slabs[0].alloc), int64(size))
return make([]byte, 0, size)
}
func Put(item []byte) {
if item == nil || cap(item) <= 0 {
return
}
size, bits := cap(item), uint(0)
for size != 0 {
size >>= 1
bits++
}
size = 1 << (bits - 1)
if size > 0 && float64(cap(item))/float64(size) <= 1.2 {
if slab, ok := slabs[size]; ok {
atomic.AddInt64(&(slab.put), 1)
select {
case slab.queue <- item:
default:
atomic.AddInt64(&(slab.lost), 1)
}
} else {
atomic.AddInt64(&(slabs[0].put), 1)
atomic.AddInt64(&(slabs[0].lost), int64(cap(item)))
}
} else {
atomic.AddInt64(&(slabs[0].put), 1)
atomic.AddInt64(&(slabs[0].lost), int64(cap(item)))
}
}

0
vendor/github.com/pyke369/golang-support/chash/README.md generated vendored Normal file
View File

356
vendor/github.com/pyke369/golang-support/chash/chash.go generated vendored Normal file
View File

@ -0,0 +1,356 @@
package chash
import (
"math/rand"
"os"
"runtime"
"sort"
"strconv"
"sync"
"time"
)
const (
CHASH_MAGIC uint32 = 0x48414843
CHASH_REPLICAS = 128
)
type item struct {
hash uint32
target uint16
}
type CHash struct {
targets map[string]uint8
names []string
ring []item
ringSize uint32
replicas uint8
frozen bool
sync.RWMutex
}
var cores int
func init() {
rand.Seed(time.Now().UnixNano() + int64(os.Getpid()))
}
func mmhash2(key []byte, keySize int) uint32 {
var magic, hash, current, value uint32 = 0x5bd1e995, uint32(0x4d4d4832 ^ keySize), 0, 0
if keySize < 0 {
keySize = len(key)
}
for keySize >= 4 {
value = uint32(key[current]) | uint32(key[current+1])<<8 |
uint32(key[current+2])<<16 | uint32(key[current+3])<<24
value *= magic
value ^= value >> 24
value *= magic
hash *= magic
hash ^= value
current += 4
keySize -= 4
}
if keySize >= 3 {
hash ^= uint32(key[current+2]) << 16
}
if keySize >= 2 {
hash ^= uint32(key[current+1]) << 8
}
if keySize >= 1 {
hash ^= uint32(key[current])
}
if keySize != 0 {
hash *= magic
}
hash ^= hash >> 13
hash *= magic
hash ^= hash >> 15
return hash
}
type ByHash []item
func (a ByHash) Len() int { return len(a) }
func (a ByHash) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByHash) Less(i, j int) bool { return a[i].hash < a[j].hash }
func (this *CHash) freeze() {
if cores > 1 {
this.Lock()
defer this.Unlock()
}
if this.frozen {
return
}
this.ringSize = 0
for _, tweight := range this.targets {
this.ringSize += uint32(tweight) * uint32(this.replicas)
}
if this.ringSize == 0 {
this.frozen = true
return
}
var (
target uint16 = 0
offset uint32 = 0
key []byte = make([]byte, 128)
)
this.names = make([]string, len(this.targets))
this.ring = make([]item, this.ringSize)
for tname, tweight := range this.targets {
this.names[target] = tname
for weight := uint8(0); weight < tweight; weight++ {
for replica := uint8(0); replica < this.replicas; replica++ {
key = append(key[:0], tname...)
key = strconv.AppendInt(key, int64(weight), 10)
key = strconv.AppendInt(key, int64(replica), 10)
this.ring[offset] = item{mmhash2(key, -1), target}
offset++
}
}
target++
}
sort.Sort(ByHash(this.ring))
this.frozen = true
}
func New(replicas ...uint8) *CHash {
if cores == 0 {
cores = runtime.NumCPU()
}
chash := &CHash{
targets: make(map[string]uint8),
names: nil,
ring: nil,
ringSize: 0,
replicas: CHASH_REPLICAS,
frozen: false,
}
if len(replicas) > 0 {
chash.replicas = replicas[0]
}
if chash.replicas < 1 {
chash.replicas = 1
}
if chash.replicas > CHASH_REPLICAS {
chash.replicas = CHASH_REPLICAS
}
return chash
}
func (this *CHash) AddTarget(name string, weight uint8) bool {
if weight > 0 && weight <= 100 && len(name) <= 128 && this.targets[name] != weight {
if cores > 1 {
this.Lock()
defer this.Unlock()
}
this.targets[name] = weight
this.frozen = false
return true
}
return false
}
func (this *CHash) RemoveTarget(name string) bool {
if cores > 1 {
this.Lock()
defer this.Unlock()
}
delete(this.targets, name)
this.frozen = false
return true
}
func (this *CHash) ClearTargets() bool {
if cores > 1 {
this.Lock()
defer this.Unlock()
}
this.targets = make(map[string]uint8)
this.frozen = false
return true
}
func (this *CHash) Serialize() []byte {
this.freeze()
if cores > 1 {
this.RLock()
defer this.RUnlock()
}
size := uint32(4) + 4 + 1 + 2 + 4
for _, name := range this.names {
size += 1 + 1 + uint32(len(name))
}
size += (this.ringSize * 6)
serialized := make([]byte, size)
offset := uint32(0)
serialized[offset] = byte(CHASH_MAGIC & 0xff)
serialized[offset+1] = byte((CHASH_MAGIC >> 8) & 0xff)
serialized[offset+2] = byte((CHASH_MAGIC >> 16) & 0xff)
serialized[offset+3] = byte((CHASH_MAGIC >> 24) & 0xff)
serialized[offset+4] = byte(size & 0xff)
serialized[offset+5] = byte((size >> 8) & 0xff)
serialized[offset+6] = byte((size >> 16) & 0xff)
serialized[offset+7] = byte((size >> 24) & 0xff)
serialized[offset+8] = this.replicas
serialized[offset+9] = byte(uint16(len(this.names)) & 0xff)
serialized[offset+10] = byte(((uint16(len(this.names))) >> 8) & 0xff)
serialized[offset+11] = byte(this.ringSize & 0xff)
serialized[offset+12] = byte((this.ringSize >> 8) & 0xff)
serialized[offset+13] = byte((this.ringSize >> 16) & 0xff)
serialized[offset+14] = byte((this.ringSize >> 24) & 0xff)
offset += 15
for _, name := range this.names {
serialized[offset] = this.targets[name]
serialized[offset+1] = byte(len(name) & 0xff)
copy(serialized[offset+2:offset+2+uint32(serialized[offset+1])], []byte(name))
offset += 2 + uint32(serialized[offset+1])
}
for _, item := range this.ring {
serialized[offset] = byte(item.hash & 0xff)
serialized[offset+1] = byte((item.hash >> 8) & 0xff)
serialized[offset+2] = byte((item.hash >> 16) & 0xff)
serialized[offset+3] = byte((item.hash >> 24) & 0xff)
serialized[offset+4] = byte(item.target & 0xff)
serialized[offset+5] = byte((item.target >> 8) & 0xff)
offset += 6
}
return serialized
}
func (this *CHash) FileSerialize(path string) bool {
handle, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
return false
}
defer handle.Close()
if _, err := handle.Write(this.Serialize()); err != nil {
return false
}
return true
}
func (this *CHash) Unserialize(serialized []byte) bool {
if cores > 1 {
this.Lock()
defer this.Unlock()
}
if len(serialized) < 15 {
return false
}
magic := uint32(serialized[0]) + (uint32(serialized[1]) << 8) + (uint32(serialized[2]) << 16) + (uint32(serialized[3]) << 24)
size := uint32(serialized[4]) + (uint32(serialized[5]) << 8) + (uint32(serialized[6]) << 16) + (uint32(serialized[7]) << 24)
replicas := serialized[8]
names := uint16(serialized[9]) + (uint16(serialized[10]) << 8)
ringSize := uint32(serialized[11]) + (uint32(serialized[12]) << 8) + (uint32(serialized[13]) << 16) + (uint32(serialized[14]) << 24)
if magic != CHASH_MAGIC || size != uint32(len(serialized)) {
return false
}
this.targets = make(map[string]uint8)
this.names = make([]string, names)
this.ring = make([]item, ringSize)
this.ringSize = ringSize
this.replicas = replicas
offset := uint32(15)
for index := uint16(0); index < names && offset < size; index++ {
len := uint32(serialized[offset+1])
this.names[index] = string(serialized[offset+2 : offset+2+len])
this.targets[this.names[index]] = serialized[offset]
offset += 2 + len
}
if offset > size {
return false
}
for item := uint32(0); item < ringSize && offset < size; item++ {
this.ring[item].hash = uint32(serialized[offset]) + (uint32(serialized[offset+1]) << 8) + (uint32(serialized[offset+2]) << 16) + (uint32(serialized[offset+3]) << 24)
this.ring[item].target = uint16(serialized[offset+4]) + (uint16(serialized[offset+5]) << 8)
offset += 6
}
if offset != size {
return false
}
this.frozen = true
return true
}
func (this *CHash) FileUnserialize(path string) bool {
handle, err := os.OpenFile(path, os.O_RDONLY, 0)
if err != nil {
return false
}
defer handle.Close()
info, err := handle.Stat()
if err != nil {
return false
}
if info.Size() > 128*1024*1024 {
return false
}
serialized := make([]byte, info.Size())
read, err := handle.Read(serialized)
if int64(read) != info.Size() || err != nil {
return false
}
return this.Unserialize(serialized)
}
func (this *CHash) Lookup(candidate string, count int) []string {
var start uint32 = 0
this.freeze()
if cores > 1 {
this.RLock()
defer this.RUnlock()
}
if count > len(this.targets) {
count = len(this.targets)
}
if this.ringSize == 0 || count < 1 {
return []string{}
}
hash := mmhash2([]byte(candidate), -1)
if hash > this.ring[0].hash && hash <= this.ring[this.ringSize-1].hash {
start = this.ringSize / 2
span := start / 2
for {
if hash > this.ring[start].hash && hash <= this.ring[start+1].hash {
break
}
if hash > this.ring[start].hash {
start += span
} else {
start -= span
}
span /= 2
if span < 1 {
span = 1
}
}
}
result := make([]string, count)
rank := 0
for rank < count {
index := 0
for index = 0; index < rank; index++ {
if result[index] == this.names[this.ring[start].target] {
break
}
}
if index >= rank {
result[rank] = this.names[this.ring[start].target]
rank++
}
start++
if start >= this.ringSize {
start = 0
}
}
return result
}
func (this *CHash) LookupBalance(candidate string, count int) string {
result := this.Lookup(candidate, count)
if len(result) > 0 {
return result[rand.Intn(len(result))]
}
return ""
}

54
vendor/github.com/pyke369/golang-support/dynacert/dynacert.go generated vendored Normal file
View File

@ -0,0 +1,54 @@
package dynacert
import (
"crypto/tls"
"os"
"runtime"
"sync"
"time"
)
type DYNACERT struct {
Public, Key string
Certificate *tls.Certificate
Last, Modified time.Time
sync.RWMutex
}
var cores int
func (this *DYNACERT) GetCertificate(*tls.ClientHelloInfo) (cert *tls.Certificate, err error) {
var info os.FileInfo
if cores == 0 {
cores = runtime.NumCPU()
}
if this.Certificate == nil || time.Now().Sub(this.Last) >= 10*time.Second {
this.Last = time.Now()
if info, err = os.Stat(this.Public); err != nil {
return nil, err
}
if _, err = os.Stat(this.Key); err != nil {
return nil, err
}
if this.Certificate == nil || info.ModTime().Sub(this.Modified) != 0 {
if certificate, err := tls.LoadX509KeyPair(this.Public, this.Key); err != nil {
return nil, err
} else {
if cores > 1 {
this.Lock()
}
this.Modified = info.ModTime()
this.Certificate = &certificate
if cores > 1 {
this.Unlock()
}
}
}
}
if cores > 1 {
this.RLock()
defer this.RUnlock()
}
return this.Certificate, nil
}

0
vendor/github.com/pyke369/golang-support/fqdn/README.md generated vendored Normal file
View File

28
vendor/github.com/pyke369/golang-support/fqdn/fqdn.go generated vendored Normal file
View File

@ -0,0 +1,28 @@
package fqdn
import (
"net"
"os"
"strings"
)
func FQDN() (string, string) {
if hostname, err := os.Hostname(); err == nil {
if addresses, err := net.LookupHost(hostname); err != nil {
return hostname, "*"
} else {
for _, address := range addresses {
if hostnames, err := net.LookupAddr(address); err == nil && len(hostnames) > 0 {
for _, hostname := range hostnames {
if strings.Count(hostname, ".") > 1 {
hostname = strings.TrimSuffix(hostname, ".")
addresses, _ = net.LookupHost(hostname)
return hostname, addresses[0]
}
}
}
}
}
}
return "unknown", "*"
}

60
vendor/github.com/pyke369/golang-support/listener/listener.go generated vendored Normal file
View File

@ -0,0 +1,60 @@
package listener
import (
"context"
"net"
"syscall"
"golang.org/x/sys/unix"
)
type TCPListener struct {
*net.TCPListener
ReadBufferSize int
WriteBufferSize int
}
func (this *TCPListener) Accept() (net.Conn, error) {
connection, err := this.AcceptTCP()
if err != nil {
return nil, err
}
if rconnection, err := connection.SyscallConn(); err == nil {
rconnection.Control(
func(handle uintptr) {
syscall.SetsockoptInt(int(handle), syscall.SOL_SOCKET, syscall.SO_KEEPALIVE, 1)
syscall.SetsockoptInt(int(handle), syscall.IPPROTO_TCP, syscall.TCP_KEEPIDLE, 60)
syscall.SetsockoptInt(int(handle), syscall.IPPROTO_TCP, syscall.TCP_KEEPINTVL, 10)
syscall.SetsockoptInt(int(handle), syscall.IPPROTO_TCP, syscall.TCP_KEEPCNT, 3)
})
}
if this.ReadBufferSize > 0 {
connection.SetReadBuffer(this.ReadBufferSize)
}
if this.WriteBufferSize > 0 {
connection.SetWriteBuffer(this.WriteBufferSize)
}
return connection, nil
}
func NewTCPListener(network, address string, reuseport bool, read, write int) (listener *TCPListener, err error) {
config := net.ListenConfig{
Control: func(network, address string, connection syscall.RawConn) error {
connection.Control(func(handle uintptr) {
if err := syscall.SetsockoptInt(int(handle), syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1); err != nil {
return
}
if reuseport {
if err := syscall.SetsockoptInt(int(handle), syscall.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil {
return
}
}
})
return nil
}}
if clistener, err := config.Listen(context.Background(), network, address); err != nil {
return nil, err
} else {
return &TCPListener{clistener.(*net.TCPListener), read, write}, nil
}
}

34
vendor/github.com/pyke369/golang-support/prefixdb/README.md generated vendored Normal file
View File

@ -0,0 +1,34 @@
PFDB binary layout
------------------
"PFDB"
VVVV (4)
HHHHHHHHHHHHHHHH (16)
"DESC"
"DDDDDDDDDDDDDDDDDDDD" (20)
"STRS"
W (1)
SSSS (4)
CCCC (4)
...
"NUMS"
SSSS (4)
CCCC (4)
...
"PAIR"
SSSS (4)
CCCC (4)
...
"CLUS"
W (1)
SSSS (4)
CCCC (4)
...
"MAPS"
SSSS (4)
CCCC (4)
...
"NODE"
W (1)
SSSS (4)
CCCC (4)
...

11
vendor/github.com/pyke369/golang-support/prefixdb/cmd/Makefile generated vendored Normal file
View File

@ -0,0 +1,11 @@
prefixdb: prefixdb.go
@go build prefixdb.go && strip prefixdb
run: prefixdb
@#./prefixdb city city.pfdb@'MMCITY 20190402' GeoIP2-City-Locations-en.csv GeoIP2-City-Blocks-IPv4.csv GeoIP2-City-Blocks-IPv6.csv
@#./prefixdb asn asn.pfdb@'MMASN 20190402' GeoLite2-ASN-Blocks-IPv4.csv GeoLite2-ASN-Blocks-IPv6.csv
@./prefixdb lookup city.pfdb asn.pfdb 78.193.67.63 188.65.124.26
@#./prefixdb server *:8000 city.pfdb asn.pfdb
clean:
@rm -f prefixdb *.pfdb

5
vendor/github.com/pyke369/golang-support/prefixdb/cmd/go.mod generated vendored Normal file
View File

@ -0,0 +1,5 @@
module .
go 1.12
require github.com/pyke369/golang-support v0.0.0-20190428173758-fae1fcd33c43 // indirect

2
vendor/github.com/pyke369/golang-support/prefixdb/cmd/go.sum generated vendored Normal file
View File

@ -0,0 +1,2 @@
github.com/pyke369/golang-support v0.0.0-20190428173758-fae1fcd33c43 h1:638A4GSCbTc/Z8N1TyymmC8iWQOE3BWnWJv9fzZeHJc=
github.com/pyke369/golang-support v0.0.0-20190428173758-fae1fcd33c43/go.mod h1:0XGrzgrEp0fa/+JSV8XZePUwyjnU6C3bMc7Xz2bHHKI=

367
vendor/github.com/pyke369/golang-support/prefixdb/cmd/prefixdb.go generated vendored Normal file
View File

@ -0,0 +1,367 @@
package main
import (
"bufio"
"encoding/json"
"fmt"
"net"
"net/http"
"os"
"regexp"
"strconv"
"strings"
"time"
"github.com/pyke369/golang-support/prefixdb"
)
type LOCATION struct {
ContinentCode string
ContinentName string
CountryCode string
CountryName string
RegionCode string
RegionName string
StateCode string
StateName string
CityName string
TimeZone string
InEurope bool
}
var (
csvMatcher = regexp.MustCompile(`(?:,|\n|^)("(?:(?:"")*[^"]*)*"|[^",\n]*|(?:\n|$))`)
jsonMatcher = regexp.MustCompile(`^(\S+)(?:\s(\{.+?\}))?$`)
pfdb = prefixdb.New()
)
func size(input int) string {
if input < 1024*1024 {
return fmt.Sprintf("%.1fkB", float64(input)/1024)
}
return fmt.Sprintf("%.1fMB", float64(input)/(1024*1024))
}
func mkjson() {
for index := 3; index < len(os.Args); index++ {
count := 0
if handle, err := os.Open(os.Args[index]); err == nil {
reader := bufio.NewReader(handle)
last := time.Now()
start := last
for {
if line, err := reader.ReadString('\n'); err != nil {
break
} else {
if fields := jsonMatcher.FindStringSubmatch(strings.TrimSpace(line)); fields != nil {
if _, prefix, err := net.ParseCIDR(fields[1]); err == nil {
data := map[string]interface{}{}
json.Unmarshal([]byte(fields[2]), &data)
pfdb.Add(*prefix, data, [][]string{[]string{"key1", "key2"}})
count++
}
}
}
if now := time.Now(); now.Sub(last) >= 250*time.Millisecond {
last = now
fmt.Fprintf(os.Stderr, "\r- adding prefixes [%s] %d", os.Args[index], count)
}
}
handle.Close()
fmt.Fprintf(os.Stderr, "\r- added prefixes [%s] (%.3fs - %d entries)\n", os.Args[index], float64(time.Now().Sub(start))/float64(time.Second), count)
}
}
start := time.Now()
description := ""
if index := strings.Index(os.Args[2], "@"); index > 0 {
description = os.Args[2][index+1:]
os.Args[2] = os.Args[2][:index]
}
fmt.Fprintf(os.Stderr, "\r- saving database [%s]... ", os.Args[2])
if _, err := pfdb.Save(os.Args[2], description); err != nil {
fmt.Fprintf(os.Stderr, "\r- saving database [%s] failed (%v)\n", os.Args[2], err)
} else {
fmt.Fprintf(os.Stderr, "\r- saved database [%s] (%.3fs - total[%s] strings[%s] numbers[%s] pairs[%s] clusters[%s] maps[%s] nodes[%s])\n",
os.Args[2], float64(time.Now().Sub(start))/float64(time.Second), size(pfdb.Total), size(pfdb.Strings[0]),
size(pfdb.Numbers[0]), size(pfdb.Pairs[0]), size(pfdb.Clusters[0]), size(pfdb.Maps[0]), size(pfdb.Nodes[0]),
)
}
}
func mkcity() {
locations := map[int]*LOCATION{}
if handle, err := os.Open(os.Args[3]); err == nil {
reader := bufio.NewReader(handle)
last := time.Now()
start := last
for {
if line, err := reader.ReadString('\n'); err != nil {
break
} else {
if fields := csvMatcher.FindAllStringSubmatch(strings.TrimSpace(line), -1); len(fields) == 14 {
for index := 0; index < len(fields); index++ {
fields[index][1] = strings.Trim(fields[index][1], `"`)
}
if id, err := strconv.Atoi(fields[0][1]); err == nil {
locations[id] = &LOCATION{
ContinentCode: fields[2][1],
ContinentName: fields[3][1],
CountryCode: fields[4][1],
CountryName: fields[5][1],
RegionCode: fields[6][1],
RegionName: fields[7][1],
StateCode: fields[8][1],
StateName: fields[9][1],
CityName: fields[10][1],
TimeZone: fields[12][1],
InEurope: fields[13][1] == "1",
}
}
}
}
if now := time.Now(); now.Sub(last) >= 250*time.Millisecond {
last = now
fmt.Fprintf(os.Stderr, "\r- loading locations [%s] %d", os.Args[3], len(locations))
}
}
handle.Close()
fmt.Fprintf(os.Stderr, "\r- loaded locations [%s] (%.3fs - %d entries)\n", os.Args[3], float64(time.Now().Sub(start))/float64(time.Second), len(locations))
}
clusters := [][]string{
[]string{"continent_code", "continent_name", "country_code", "country_name", "region_code", "region_name", "state_code", "state_name", "timezone", "in_europe"},
[]string{"city_name", "postal_code", "latitude", "longitude"},
}
for index := 4; index < len(os.Args); index++ {
count := 0
if handle, err := os.Open(os.Args[index]); err == nil {
reader := bufio.NewReader(handle)
last := time.Now()
start := last
for {
if line, err := reader.ReadString('\n'); err != nil {
break
} else {
if fields := strings.Split(strings.TrimSpace(line), ","); len(fields) == 10 {
id := 0
if id, _ = strconv.Atoi(fields[1]); id == 0 {
id, _ = strconv.Atoi(fields[2])
}
if id != 0 && locations[id] != nil {
if _, prefix, err := net.ParseCIDR(fields[0]); err == nil {
latitude, _ := strconv.ParseFloat(fields[7], 64)
longitude, _ := strconv.ParseFloat(fields[8], 64)
pfdb.Add(*prefix, map[string]interface{}{
"continent_code": locations[id].ContinentCode,
"continent_name": locations[id].ContinentName,
"country_code": locations[id].CountryCode,
"country_name": locations[id].CountryName,
"region_code": locations[id].RegionCode,
"region_name": locations[id].RegionName,
"state_code": locations[id].StateCode,
"state_name": locations[id].StateName,
"city_name": locations[id].CityName,
"in_europe": locations[id].InEurope,
"timezone": locations[id].TimeZone,
"postal_code": fields[6],
"latitude": latitude,
"longitude": longitude,
}, clusters)
count++
}
}
}
}
if now := time.Now(); now.Sub(last) >= 250*time.Millisecond {
last = now
fmt.Fprintf(os.Stderr, "\r- adding prefixes [%s] %d", os.Args[index], count)
}
}
handle.Close()
fmt.Fprintf(os.Stderr, "\r- added prefixes [%s] (%.3fs - %d entries)\n", os.Args[index], float64(time.Now().Sub(start))/float64(time.Second), count)
}
}
start := time.Now()
description := ""
if index := strings.Index(os.Args[2], "@"); index > 0 {
description = os.Args[2][index+1:]
os.Args[2] = os.Args[2][:index]
}
fmt.Fprintf(os.Stderr, "\r- saving database [%s]... ", os.Args[2])
if _, err := pfdb.Save(os.Args[2], description); err != nil {
fmt.Fprintf(os.Stderr, "\r- saving database [%s] failed (%v)\n", os.Args[2], err)
} else {
fmt.Fprintf(os.Stderr, "\r- saved database [%s] (%.3fs - total[%s] strings[%s] numbers[%s] pairs[%s] clusters[%s] maps[%s] nodes[%s])\n",
os.Args[2], float64(time.Now().Sub(start))/float64(time.Second), size(pfdb.Total), size(pfdb.Strings[0]),
size(pfdb.Numbers[0]), size(pfdb.Pairs[0]), size(pfdb.Clusters[0]), size(pfdb.Maps[0]), size(pfdb.Nodes[0]),
)
}
}
func mkasn() {
for index := 3; index < len(os.Args); index++ {
count := 0
if handle, err := os.Open(os.Args[index]); err == nil {
reader := bufio.NewReader(handle)
last := time.Now()
start := last
for {
if line, err := reader.ReadString('\n'); err != nil {
break
} else {
if fields := csvMatcher.FindAllStringSubmatch(strings.TrimSpace(line), -1); len(fields) == 3 {
for index := 0; index < len(fields); index++ {
fields[index][1] = strings.Trim(fields[index][1], `"`)
}
if asnum, _ := strconv.Atoi(fields[1][1]); asnum != 0 {
if _, prefix, err := net.ParseCIDR(fields[0][1]); err == nil {
pfdb.Add(*prefix, map[string]interface{}{
"as_number": fmt.Sprintf("AS%d", asnum),
"as_name": fields[2][1],
}, nil)
count++
}
}
}
}
if now := time.Now(); now.Sub(last) >= 250*time.Millisecond {
last = now
fmt.Fprintf(os.Stderr, "\r- adding prefixes [%s] %d", os.Args[index], count)
}
}
handle.Close()
fmt.Fprintf(os.Stderr, "\r- added prefixes [%s] (%.3fs - %d entries)\n", os.Args[index], float64(time.Now().Sub(start))/float64(time.Second), count)
}
}
start := time.Now()
description := ""
if index := strings.Index(os.Args[2], "@"); index > 0 {
description = os.Args[2][index+1:]
os.Args[2] = os.Args[2][:index]
}
fmt.Fprintf(os.Stderr, "\r- saving database [%s]... ", os.Args[2])
if _, err := pfdb.Save(os.Args[2], description); err != nil {
fmt.Fprintf(os.Stderr, "\r- saving database [%s] failed (%v)\n", os.Args[2], err)
} else {
fmt.Fprintf(os.Stderr, "\r- saved database [%s] (%.3fs - total[%s] strings[%s] numbers[%s] pairs[%s] clusters[%s] maps[%s] nodes[%s])\n",
os.Args[2], float64(time.Now().Sub(start))/float64(time.Second), size(pfdb.Total), size(pfdb.Strings[0]),
size(pfdb.Numbers[0]), size(pfdb.Pairs[0]), size(pfdb.Clusters[0]), size(pfdb.Maps[0]), size(pfdb.Nodes[0]),
)
}
}
func lookup() {
databases := []*prefixdb.PrefixDB{}
for index := 2; index < len(os.Args); index++ {
if strings.HasSuffix(os.Args[index], `.pfdb`) {
fmt.Fprintf(os.Stderr, "\r- loading database [%s]...", os.Args[index])
database := prefixdb.New()
if err := database.Load(os.Args[index]); err == nil {
fmt.Fprintf(os.Stderr, "\r- loaded database [%s] (total[%s] version[%d.%d.%d] description[%s])\n",
os.Args[index], size(database.Total), (database.Version>>16)&0xff, (database.Version>>8)&0xff, database.Version&0xff, database.Description)
databases = append(databases, database)
} else {
fmt.Fprintf(os.Stderr, "\r- loading database [%s] failed (%v)\n", os.Args[index], err)
}
} else {
if ip := net.ParseIP(os.Args[index]); ip == nil {
fmt.Fprintf(os.Stderr, "- lookup [%s] failed (not a valid IP address)", os.Args[index])
} else {
fmt.Fprintf(os.Stderr, "- lookup [%s] ", os.Args[index])
lookup := map[string]interface{}{}
for _, database := range databases {
lookup, _ = database.Lookup(ip, lookup)
}
data, _ := json.Marshal(lookup)
fmt.Printf("%s\n", data)
}
}
}
}
func server() {
databases := []*prefixdb.PrefixDB{}
for index := 3; index < len(os.Args); index++ {
fmt.Fprintf(os.Stderr, "\r- loading database [%s]...", os.Args[index])
database := prefixdb.New()
if err := database.Load(os.Args[index]); err == nil {
fmt.Fprintf(os.Stderr, "\r- loaded database [%s] (total[%s] version[%d.%d.%d] description[%s])\n",
os.Args[index], size(database.Total), (database.Version>>16)&0xff, (database.Version>>8)&0xff, database.Version&0xff, database.Description)
databases = append(databases, database)
} else {
fmt.Fprintf(os.Stderr, "\r- loading database [%s] failed (%v)\n", os.Args[index], err)
}
}
http.HandleFunc("/", func(response http.ResponseWriter, request *http.Request) {
response.Header().Set("Content-Type", "application/json")
remote, _, _ := net.SplitHostPort(request.RemoteAddr)
if value := request.Header.Get("X-Forwarded-For"); value != "" {
remote = strings.Split(value, ",")[0]
}
parameters := request.URL.Query()
if value := parameters.Get("remote"); value != "" {
remote = value
}
lookup := map[string]interface{}{}
if ip := net.ParseIP(remote); ip != nil {
lookup["ip"] = fmt.Sprintf("%s", ip)
for _, database := range databases {
lookup, _ = database.Lookup(ip, lookup)
}
data, _ := json.Marshal(lookup)
response.Write(data)
}
})
parts := strings.Split(os.Args[2], ",")
server := &http.Server{Addr: strings.TrimLeft(parts[0], "*"), ReadTimeout: 10 * time.Second, WriteTimeout: 10 * time.Second}
fmt.Fprintf(os.Stderr, "\r- listening to [%s]\n", os.Args[2])
if len(parts) > 1 {
server.ListenAndServeTLS(parts[1], parts[2])
} else {
server.ListenAndServe()
}
}
func usage(status int) {
fmt.Fprintf(os.Stderr, "usage: prefixdb <action> [parameters...]\n\n"+
"help show this help screen\n"+
"json <database[@<description>]> <JSON prefixes>... build database from generic JSON-formatted prefixes lists\n"+
"city <database[@<description>]> <CSV locations> <CSV prefixes>... build database from MaxMind GeoIP2 cities lists\n"+
"asn <database[@<description>]> <CSV prefixes>... build database from MaxMind GeoLite2 asnums lists\n"+
"lookup <database>... <address>... lookup entries in databases\n"+
"server <bind address> <database>... spawn an HTTP(S) server for entries lookup\n")
os.Exit(status)
}
func main() {
if len(os.Args) < 2 {
usage(1)
}
switch os.Args[1] {
case "help":
usage(0)
case "json":
if len(os.Args) < 3 {
usage(1)
}
mkjson()
case "city":
if len(os.Args) < 5 {
usage(1)
}
mkcity()
case "asn":
if len(os.Args) < 3 {
usage(1)
}
mkasn()
case "lookup":
lookup()
case "server":
server()
default:
usage(2)
}
}

827
vendor/github.com/pyke369/golang-support/prefixdb/prefixdb.go generated vendored Normal file
View File

@ -0,0 +1,827 @@
package prefixdb
import (
"bytes"
"crypto/md5"
"encoding/binary"
"errors"
"fmt"
"io/ioutil"
"math"
"net"
"os"
"runtime"
"sort"
"strings"
"sync"
"time"
)
const VERSION = 0x00010000
type fame struct {
fame int
value interface{}
}
type byfame []*fame
func (a byfame) Len() int { return len(a) }
func (a byfame) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a byfame) Less(i, j int) bool { return a[i].fame > a[j].fame }
type node struct {
down [2]*node
up *node
data []uint64
offset int
explored [4]bool
emitted bool
id int
}
type cluster struct {
values [3]int // fame / initial index / final index
pairs []uint64 // cluster pairs
data []byte // reduced cluster pairs
}
type PrefixDB struct {
sync.RWMutex
tree node
strings map[string]*[3]int // fame / initial index / final index
numbers map[float64]*[3]int // fame / initial index / final index
pairs map[uint64]*[3]int // fame / initial index / final index
clusters map[[16]byte]*cluster
data []byte
Total int
Version uint32
Description string
Strings [4]int // size / count / offset / strings index width (bytes)
Numbers [3]int // size / count / offset
Pairs [3]int // size / count / offset
Clusters [4]int // size / count / offset / clusters index width (bytes)
Maps [3]int // size / count / offset
Nodes [4]int // size / count / offset / nodes width (bits)
}
var cores int
func New() *PrefixDB {
if cores == 0 {
cores = runtime.NumCPU()
}
return &PrefixDB{strings: map[string]*[3]int{}, numbers: map[float64]*[3]int{}, pairs: map[uint64]*[3]int{}, clusters: map[[16]byte]*cluster{}}
}
func (this *PrefixDB) Add(prefix net.IPNet, data map[string]interface{}, clusters [][]string) {
prefix.IP = prefix.IP.To16()
ones, bits := prefix.Mask.Size()
if bits == 32 {
ones += 96
prefix.Mask = net.CIDRMask(ones, bits+96)
}
if cores > 1 {
this.Lock()
}
pnode := &this.tree
for bit := 0; bit < ones; bit++ {
down := 0
if (prefix.IP[bit/8] & (1 << (7 - (byte(bit) % 8)))) != 0 {
down = 1
}
if pnode.down[down] == nil {
pnode.down[down] = &node{}
pnode.down[down].up = pnode
}
if len(pnode.data) != 0 {
pnode.data = []uint64{}
}
pnode = pnode.down[down]
}
skeys, ckeys, lkeys := "", [][]string{}, []string{}
for _, cluster := range clusters {
skeys += strings.Join(cluster, ` `) + ` `
ckeys = append(ckeys, cluster)
}
for key, _ := range data {
if strings.Index(skeys, key) < 0 {
lkeys = append(lkeys, key)
}
}
ckeys = append(ckeys, lkeys)
for cindex, keys := range ckeys {
cpairs := []uint64{}
for _, key := range keys {
if len(key) > 255 {
continue
}
if value, ok := data[key]; ok {
index := 0
if _, ok := this.strings[key]; !ok {
index = len(this.strings)
this.strings[key] = &[3]int{1, index}
} else {
index = this.strings[key][1]
this.strings[key][0]++
}
pair := uint64((uint32(index)&0x0fffffff)|0x10000000) << 32
if tvalue, ok := value.(string); ok {
if len(tvalue) <= 255 {
index = 0
if _, ok := this.strings[tvalue]; !ok {
index = len(this.strings)
this.strings[tvalue] = &[3]int{1, index}
} else {
index = this.strings[tvalue][1]
this.strings[tvalue][0]++
}
pair |= uint64((uint32(index) & 0x0fffffff) | 0x10000000)
} else {
pair |= uint64(0x50000000)
}
} else if tvalue, ok := value.(float64); ok {
index = 0
if _, ok := this.numbers[tvalue]; !ok {
index = len(this.numbers)
this.numbers[tvalue] = &[3]int{1, index}
} else {
index = this.numbers[tvalue][1]
this.numbers[tvalue][0]++
}
pair |= uint64((uint32(index) & 0x0fffffff) | 0x20000000)
} else if tvalue, ok := value.(bool); ok {
if tvalue {
pair |= uint64(0x30000000)
} else {
pair |= uint64(0x40000000)
}
} else {
pair |= uint64(0x50000000)
}
if _, ok := this.pairs[pair]; !ok {
index = len(this.pairs)
this.pairs[pair] = &[3]int{1, index}
} else {
this.pairs[pair][0]++
}
if cindex < len(ckeys)-1 {
cpairs = append(cpairs, pair)
} else {
pnode.data = append(pnode.data, pair)
}
}
}
if len(cpairs) != 0 {
buffer := make([]byte, len(cpairs)*8)
for index, value := range cpairs {
binary.BigEndian.PutUint64(buffer[index*8:], value)
}
key := md5.Sum(buffer)
index := 0
if _, ok := this.clusters[key]; !ok {
index = len(this.clusters)
this.clusters[key] = &cluster{pairs: cpairs, values: [3]int{1, index}}
} else {
index = this.clusters[key].values[1]
this.clusters[key].values[0]++
}
pnode.data = append(pnode.data, 0x7000000000000000|((uint64(index)<<32)&0x0fffffff00000000))
}
}
if cores > 1 {
this.Unlock()
}
}
func wbytes(bytes, value int, data []byte) {
if len(data) >= bytes {
for index := bytes - 1; index >= 0; index-- {
data[bytes-index-1] = byte(value >> (uint(index * 8)))
}
}
}
func wpbits(prefix byte, value int) []byte {
if value <= 7 {
return []byte{prefix | (byte(value) & 0x07)}
}
bytes := int(math.Ceil(math.Ceil(math.Log2(float64(value+1))) / 8))
data := []byte{prefix | 0x08 | byte(bytes)}
for nibble := bytes - 1; nibble >= 0; nibble-- {
data = append(data, byte(value>>(uint(nibble*8))))
}
return data
}
func wnbits(bits, value0, value1 int, data []byte) {
if bits >= 8 && bits <= 32 && bits%4 == 0 && len(data) >= bits/4 {
switch bits {
case 8:
data[0], data[1] = byte(value0), byte(value1)
case 12:
data[0], data[1], data[2] = byte(value0>>4), byte(value0<<4)|(byte(value1>>8)&0x0f), byte(value1)
case 16:
binary.BigEndian.PutUint16(data[0:], uint16(value0))
binary.BigEndian.PutUint16(data[2:], uint16(value1))
case 20:
data[0], data[1] = byte(value0>>12), byte(value0>>4)
data[2] = byte(value0<<4) | (byte(value1>>16) & 0x0f)
data[3], data[4] = byte(value1>>8), byte(value1)
case 24:
data[0], data[1], data[2] = byte(value0>>16), byte(value0>>8), byte(value0)
data[3], data[4], data[5] = byte(value1>>16), byte(value1>>8), byte(value1)
case 28:
data[0], data[1], data[2] = byte(value0>>20), byte(value0>>12), byte(value0>>4)
data[3] = byte(value0<<4) | (byte(value1>>24) & 0x0f)
data[4], data[5], data[6] = byte(value1>>16), byte(value1>>8), byte(value1)
case 32:
binary.BigEndian.PutUint32(data[0:], uint32(value0))
binary.BigEndian.PutUint32(data[4:], uint32(value1))
}
}
}
func (this *PrefixDB) Save(path, description string) (content []byte, err error) {
// layout header + signature placeholder + description
if cores > 1 {
this.Lock()
}
this.data = []byte{'P', 'F', 'D', 'B', 0, (VERSION >> 16) & 0xff, (VERSION >> 8) & 0xff, (VERSION & 0xff),
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 'D', 'E', 'S', 'C', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
if description == "" {
description = time.Now().Format(`20060102150405`)
}
this.Description = description
copy(this.data[28:47], []byte(description))
// layout strings dictionary (ordered by fame)
this.Strings[0] = 0
for key, _ := range this.strings {
this.Strings[0] += len(key)
}
this.Strings[3] = int(math.Ceil(math.Ceil(math.Log2(float64(this.Strings[0]+1))) / 8))
this.data = append(this.data, []byte{'S', 'T', 'R', 'S', byte(this.Strings[3]), 0, 0, 0, 0, 0, 0, 0, 0}...)
this.Strings[2] = len(this.data)
this.Strings[1] = len(this.strings)
this.Strings[0] += this.Strings[1] * this.Strings[3]
flist := make([]*fame, this.Strings[1])
for key, values := range this.strings {
flist[values[1]] = &fame{values[0], key}
}
sort.Sort(byfame(flist))
this.data = append(this.data, make([]byte, this.Strings[1]*this.Strings[3])...)
offset := 0
for index, item := range flist {
this.strings[item.value.(string)][2] = index
this.data = append(this.data, []byte(item.value.(string))...)
wbytes(this.Strings[3], offset, this.data[this.Strings[2]+(index*this.Strings[3]):])
offset += len(item.value.(string))
}
binary.BigEndian.PutUint32(this.data[this.Strings[2]-8:], uint32(this.Strings[0]))
binary.BigEndian.PutUint32(this.data[this.Strings[2]-4:], uint32(this.Strings[1]))
strings := make([]*fame, this.Strings[1])
for key, values := range this.strings {
strings[values[1]] = &fame{values[0], key}
}
// layout numbers dictionary (ordered by fame)
this.data = append(this.data, []byte{'N', 'U', 'M', 'S', 0, 0, 0, 0, 0, 0, 0, 0}...)
this.Numbers[2] = len(this.data)
this.Numbers[1] = len(this.numbers)
this.Numbers[0] = this.Numbers[1] * 8
flist = make([]*fame, this.Numbers[1])
for key, values := range this.numbers {
flist[values[1]] = &fame{values[0], key}
}
sort.Sort(byfame(flist))
this.data = append(this.data, make([]byte, this.Numbers[1]*8)...)
for index, item := range flist {
this.numbers[item.value.(float64)][2] = index
binary.BigEndian.PutUint64(this.data[this.Numbers[2]+(index*8):], math.Float64bits(item.value.(float64)))
}
binary.BigEndian.PutUint32(this.data[this.Numbers[2]-8:], uint32(this.Numbers[0]))
binary.BigEndian.PutUint32(this.data[this.Numbers[2]-4:], uint32(this.Numbers[1]))
numbers := make([]*fame, this.Numbers[1])
for key, values := range this.numbers {
numbers[values[1]] = &fame{values[0], key}
}
// layout pairs dictionary (ordered by fame)
this.data = append(this.data, []byte{'P', 'A', 'I', 'R', 0, 0, 0, 0, 0, 0, 0, 0}...)
this.Pairs[2] = len(this.data)
flist = make([]*fame, len(this.pairs))
for key, values := range this.pairs {
flist[values[1]] = &fame{values[0], key}
}
sort.Sort(byfame(flist))
for index, item := range flist {
if item.fame > 1 {
this.pairs[item.value.(uint64)][2] = index
} else {
delete(this.pairs, item.value.(uint64))
}
}
this.Pairs[1] = len(this.pairs)
this.Pairs[0] = this.Pairs[1] * 8
this.data = append(this.data, make([]byte, this.Pairs[0])...)
for index, item := range flist {
if item.fame <= 1 {
break
}
pair := 0x1000000000000000 | (uint64(this.strings[strings[(item.value.(uint64)>>32)&0x0fffffff].value.(string)][2]) << 32)
switch (item.value.(uint64) & 0xf0000000) >> 28 {
case 1:
pair |= 0x10000000 | uint64(this.strings[strings[item.value.(uint64)&0x0fffffff].value.(string)][2])
case 2:
pair |= 0x20000000 | uint64(this.numbers[numbers[item.value.(uint64)&0x0fffffff].value.(float64)][2])
default:
pair |= item.value.(uint64) & 0xf0000000
}
binary.BigEndian.PutUint64(this.data[this.Pairs[2]+(index*8):], pair)
}
binary.BigEndian.PutUint32(this.data[this.Pairs[2]-8:], uint32(this.Pairs[0]))
binary.BigEndian.PutUint32(this.data[this.Pairs[2]-4:], uint32(this.Pairs[1]))
// layout clusters dictionary (ordered by fame, and reduced for strings, numbers and pairs)
this.Clusters[0] = 0
for _, cluster := range this.clusters {
for _, pair := range cluster.pairs {
if _, ok := this.pairs[pair]; ok {
cluster.data = append(cluster.data, wpbits(0x60, this.pairs[pair][2])...)
} else {
cluster.data = append(cluster.data, wpbits(0x10, this.strings[strings[(pair>>32)&0x0fffffff].value.(string)][2])...)
switch (pair & 0xf0000000) >> 28 {
case 1:
cluster.data = append(cluster.data, wpbits(0x10, this.strings[strings[pair&0x0fffffff].value.(string)][2])...)
case 2:
cluster.data = append(cluster.data, wpbits(0x20, this.numbers[numbers[pair&0x0fffffff].value.(float64)][2])...)
default:
cluster.data = append(cluster.data, byte((pair&0xf0000000)>>24))
}
}
}
this.Clusters[0] += len(cluster.data)
}
this.Clusters[3] = int(math.Ceil(math.Ceil(math.Log2(float64(this.Clusters[0]+1))) / 8))
this.data = append(this.data, []byte{'C', 'L', 'U', 'S', byte(this.Clusters[3]), 0, 0, 0, 0, 0, 0, 0, 0}...)
this.Clusters[2] = len(this.data)
this.Clusters[1] = len(this.clusters)
this.Clusters[0] += this.Clusters[1] * this.Clusters[3]
flist = make([]*fame, this.Clusters[1])
for key, cluster := range this.clusters {
flist[cluster.values[1]] = &fame{cluster.values[0], key}
}
sort.Sort(byfame(flist))
this.data = append(this.data, make([]byte, this.Clusters[1]*this.Clusters[3])...)
offset = 0
for index, item := range flist {
this.clusters[item.value.([16]byte)].values[2] = index
this.data = append(this.data, this.clusters[item.value.([16]byte)].data...)
wbytes(this.Clusters[3], offset, this.data[this.Clusters[2]+(index*this.Clusters[3]):])
offset += len(this.clusters[item.value.([16]byte)].data)
}
binary.BigEndian.PutUint32(this.data[this.Clusters[2]-8:], uint32(this.Clusters[0]))
binary.BigEndian.PutUint32(this.data[this.Clusters[2]-4:], uint32(this.Clusters[1]))
clusters := make([]*fame, this.Clusters[1])
for key, cluster := range this.clusters {
clusters[cluster.values[1]] = &fame{cluster.values[0], key}
}
// layout maps dictionary (reduced for strings, numbers, pairs and clusters)
this.Nodes[1] = 1
this.data = append(this.data, []byte{'M', 'A', 'P', 'S', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x80}...)
pnode := &this.tree
this.Maps[2] = len(this.data) - 2
this.Maps[1] = 1
this.Maps[0] = 2
for {
if pnode.down[0] != nil && !pnode.explored[0] {
pnode.explored[0] = true
pnode = pnode.down[0]
} else if pnode.down[1] != nil && !pnode.explored[1] {
pnode.explored[1] = true
pnode = pnode.down[1]
} else if pnode.up != nil {
pnode = pnode.up
}
if pnode.up == nil {
break
}
if pnode.down[0] == nil && pnode.down[1] == nil {
if len(pnode.data) == 0 {
pnode.offset = 1
} else {
data := []byte{}
for index := 0; index < len(pnode.data); index++ {
last := byte(0x00)
if index == len(pnode.data)-1 {
last = 0x80
}
if ((pnode.data[index]>>32)&0xf0000000)>>28 == 7 {
data = append(data, wpbits(last|0x70, this.clusters[clusters[(pnode.data[index]>>32)&0x0fffffff].value.([16]byte)].values[2])...)
} else {
if _, ok := this.pairs[pnode.data[index]]; ok {
data = append(data, wpbits(last|0x60, this.pairs[pnode.data[index]][2])...)
} else {
data = append(data, wpbits(0x10, this.strings[strings[(pnode.data[index]>>32)&0x0fffffff].value.(string)][2])...)
switch (pnode.data[index] & 0xf0000000) >> 28 {
case 1:
data = append(data, wpbits(last|0x10, this.strings[strings[pnode.data[index]&0x0fffffff].value.(string)][2])...)
case 2:
data = append(data, wpbits(last|0x20, this.numbers[numbers[pnode.data[index]&0x0fffffff].value.(float64)][2])...)
default:
data = append(data, last|byte((pnode.data[index]&0xf0000000)>>24))
}
}
}
}
this.data = append(this.data, data...)
pnode.offset = this.Maps[0]
this.Maps[0] += len(data)
this.Maps[1]++
}
} else if pnode.id == 0 {
pnode.id = this.Nodes[1]
this.Nodes[1]++
}
}
binary.BigEndian.PutUint32(this.data[this.Maps[2]-8:], uint32(this.Maps[0]))
binary.BigEndian.PutUint32(this.data[this.Maps[2]-4:], uint32(this.Maps[1]))
// layout nodes tree
this.Nodes[3] = int(math.Ceil(math.Ceil(math.Log2(float64(this.Nodes[1]+this.Maps[0]+1)))/4) * 4)
this.data = append(this.data, []byte{'N', 'O', 'D', 'E', byte(this.Nodes[3]), 0, 0, 0, 0, 0, 0, 0, 0}...)
this.Nodes[2] = len(this.data)
this.Nodes[0] = this.Nodes[1] * ((2 * this.Nodes[3]) / 8)
this.data = append(this.data, make([]byte, this.Nodes[0])...)
pnode = &this.tree
next := [2]int{}
for {
if (pnode == &this.tree || pnode.id != 0) && !pnode.emitted {
pnode.emitted = true
for index := 0; index <= 1; index++ {
next[index] = this.Nodes[1]
if pnode.down[index] != nil {
if pnode.down[index].id != 0 {
next[index] = pnode.down[index].id
} else {
next[index] += pnode.down[index].offset
}
}
}
wnbits(this.Nodes[3], next[0], next[1], this.data[this.Nodes[2]+(pnode.id*((2*this.Nodes[3])/8)):])
}
if pnode.down[0] != nil && !pnode.explored[2] {
pnode.explored[2] = true
pnode = pnode.down[0]
} else if pnode.down[1] != nil && !pnode.explored[3] {
pnode.explored[3] = true
pnode = pnode.down[1]
} else if pnode.up != nil {
pnode = pnode.up
}
if pnode.up == nil {
break
}
}
binary.BigEndian.PutUint32(this.data[this.Nodes[2]-8:], uint32(this.Nodes[0]))
binary.BigEndian.PutUint32(this.data[this.Nodes[2]-4:], uint32(this.Nodes[1]))
// finalize header
this.tree, this.strings, this.numbers, this.pairs, this.clusters = node{}, map[string]*[3]int{}, map[float64]*[3]int{}, map[uint64]*[3]int{}, map[[16]byte]*cluster{}
hash := md5.Sum(this.data[24:])
copy(this.data[8:], hash[:])
this.Total = len(this.data)
// save database
if path != "" {
if path == "-" {
_, err = os.Stdout.Write(this.data)
} else {
err = ioutil.WriteFile(path, this.data, 0644)
}
}
if cores > 1 {
this.Unlock()
}
return this.data, err
}
func (this *PrefixDB) Load(path string) error {
if data, err := ioutil.ReadFile(path); err != nil {
return err
} else {
if len(data) < 8 || string(data[0:4]) != "PFDB" {
return errors.New(`invalid preamble`)
}
if version := (uint32(data[5]) << 16) + (uint32(data[6]) << 8) + uint32(data[7]); (version & 0xff0000) > (VERSION & 0xff0000) {
return errors.New(fmt.Sprintf(`library major version %d is incompatible with database major version %d`, (VERSION&0xff0000)>>16, (version&0xff0000)>>16))
} else {
if len(data) < 24 || fmt.Sprintf("%x", md5.Sum(data[24:])) != fmt.Sprintf("%x", data[8:24]) {
return errors.New(`database checksum is invalid`)
}
if cores > 1 {
this.Lock()
}
this.data = data
this.Total = len(data)
this.Version = version
offset := 24
if this.Total >= offset+4 && string(data[offset:offset+4]) == "DESC" {
offset += 4
if this.Total >= offset+20 {
index := 0
if index = bytes.Index(data[offset:offset+20], []byte{0}); index < 0 {
index = 20
}
this.Description = fmt.Sprintf("%s", data[offset:offset+index])
offset += 20
if this.Total >= offset+4 && string(data[offset:offset+4]) == "STRS" {
offset += 4
if this.Total >= offset+9 {
this.Strings[3] = int(data[offset])
this.Strings[2] = offset + 9
this.Strings[1] = int(binary.BigEndian.Uint32(this.data[offset+5:]))
this.Strings[0] = int(binary.BigEndian.Uint32(this.data[offset+1:]))
offset += 9 + this.Strings[0]
if this.Total >= offset+4 && string(data[offset:offset+4]) == "NUMS" {
offset += 4
if this.Total >= offset+8 {
this.Numbers[2] = offset + 8
this.Numbers[1] = int(binary.BigEndian.Uint32(this.data[offset+4:]))
this.Numbers[0] = int(binary.BigEndian.Uint32(this.data[offset:]))
offset += 8 + this.Numbers[0]
if this.Total >= offset+4 && string(data[offset:offset+4]) == "PAIR" {
offset += 4
if this.Total >= offset+8 {
this.Pairs[2] = offset + 8
this.Pairs[1] = int(binary.BigEndian.Uint32(this.data[offset+4:]))
this.Pairs[0] = int(binary.BigEndian.Uint32(this.data[offset:]))
offset += 8 + this.Pairs[0]
if this.Total >= offset+4 && string(data[offset:offset+4]) == "CLUS" {
offset += 4
this.Clusters[3] = int(data[offset])
this.Clusters[2] = offset + 9
this.Clusters[1] = int(binary.BigEndian.Uint32(this.data[offset+5:]))
this.Clusters[0] = int(binary.BigEndian.Uint32(this.data[offset+1:]))
offset += 9 + this.Clusters[0]
if this.Total >= offset+4 && string(data[offset:offset+4]) == "MAPS" {
offset += 4
if this.Total >= offset+8 {
this.Maps[2] = offset + 8
this.Maps[1] = int(binary.BigEndian.Uint32(this.data[offset+4:]))
this.Maps[0] = int(binary.BigEndian.Uint32(this.data[offset:]))
offset += 8 + this.Maps[0]
if this.Total >= offset+9 && string(data[offset:offset+4]) == "NODE" {
offset += 4
this.Nodes[3] = int(data[offset])
this.Nodes[2] = offset + 9
this.Nodes[1] = int(binary.BigEndian.Uint32(this.data[offset+5:]))
this.Nodes[0] = int(binary.BigEndian.Uint32(this.data[offset+1:]))
if offset+9+this.Nodes[0] != this.Total {
this.Nodes[2] = 0
}
}
}
}
}
}
}
}
}
}
}
}
}
if cores > 1 {
this.Unlock()
}
if this.Strings[2] == 0 || this.Numbers[2] == 0 || this.Pairs[2] == 0 || this.Clusters[2] == 0 || this.Maps[2] == 0 || this.Nodes[2] == 0 {
return errors.New(`database structure is invalid`)
}
}
}
return nil
}
func rpbits(data []byte) (section, index, size int, last bool) {
section = int((data[0] & 0x70) >> 4)
if data[0]&0x80 != 0 || section == 0 {
last = true
}
if section == 1 || section == 2 || section == 6 || section == 7 {
if data[0]&0x08 != 0 {
size = int(data[0] & 0x07)
for nibble := 1; nibble <= size; nibble++ {
index |= int(data[nibble]) << (uint(size-nibble) * 8)
}
} else {
index = int(data[0] & 0x07)
}
}
size++
return section, index, size, last
}
func rnbits(bits, index, down int, data []byte) int {
if bits >= 8 && bits <= 32 && bits%4 == 0 && (down == 0 || down == 1) && len(data) >= (index+1)*(bits/4) {
offset := index * (bits / 4)
switch bits {
case 8:
return int(data[offset+down])
case 12:
if down == 0 {
return (int(data[offset]) << 4) | ((int(data[offset+1]) >> 4) & 0x0f)
} else {
return ((int(data[offset+1]) & 0x0f) << 8) | int(data[offset+2])
}
case 16:
return int(binary.BigEndian.Uint16(data[offset+(down*2):]))
case 20:
if down == 0 {
return (int(data[offset]) << 12) | (int(data[offset+1]) << 4) | ((int(data[offset+2]) >> 4) & 0x0f)
} else {
return ((int(data[offset+2]) & 0x0f) << 16) | (int(data[offset+3]) << 8) | int(data[offset+4])
}
case 24:
if down == 0 {
return (int(data[offset]) << 16) | (int(data[offset+1]) << 8) | int(data[offset+2])
} else {
return (int(data[offset+3]) << 16) | (int(data[offset+4]) << 8) | int(data[offset+5])
}
case 28:
if down == 0 {
return (int(data[offset]) << 20) | (int(data[offset+1]) << 12) | (int(data[offset+2]) << 4) | ((int(data[offset+3]) >> 4) & 0x0f)
} else {
return ((int(data[offset+3]) & 0x0f) << 24) | (int(data[offset+4]) << 16) | (int(data[offset+5]) << 8) | int(data[offset+6])
}
case 32:
return int(binary.BigEndian.Uint32(data[offset+(down*4):]))
}
}
return index
}
func rbytes(width int, data []byte) (value int) {
for index := 0; index < width; index++ {
value |= int(data[index]) << (uint(width-1-index) * 8)
}
return value
}
func (this *PrefixDB) rstring(index int) string {
count, offset, width := this.Strings[1], this.Strings[2], this.Strings[3]
if index >= count {
return ""
}
start, end := rbytes(width, this.data[offset+(index*width):]), 0
if index < count-1 {
end = rbytes(width, this.data[offset+(index+1)*width:])
} else {
end = this.Strings[0] - (count * width)
}
return string(this.data[offset+(count*width)+start : offset+(count*width)+end])
}
func (this *PrefixDB) rnumber(index int) float64 {
if index >= this.Numbers[1] {
return 0.0
}
return math.Float64frombits(binary.BigEndian.Uint64(this.data[this.Numbers[2]+(index*8):]))
}
func (this *PrefixDB) rpair(index int, pairs map[string]interface{}) {
if index < this.Pairs[1] {
pair := binary.BigEndian.Uint64(this.data[this.Pairs[2]+(index*8):])
if key := this.rstring(int((pair >> 32) & 0x0fffffff)); key != "" {
switch (pair & 0xf0000000) >> 28 {
case 1:
pairs[key] = this.rstring(int(pair & 0x0fffffff))
case 2:
pairs[key] = this.rnumber(int(pair & 0x0fffffff))
case 3:
pairs[key] = true
case 4:
pairs[key] = false
}
}
}
}
func (this *PrefixDB) rcluster(index int, pairs map[string]interface{}) {
count, offset, width := this.Clusters[1], this.Clusters[2], this.Clusters[3]
if index < count {
start, end := rbytes(width, this.data[offset+(index*width):]), 0
if index < count-1 {
end = rbytes(width, this.data[offset+(index+1)*width:])
} else {
end = this.Clusters[0] - (count * width)
}
start += offset + (count * width)
end += offset + (count * width)
key := ""
for start < end {
section, index, size, _ := rpbits(this.data[start:])
switch section {
case 1:
if key != "" {
pairs[key] = this.rstring(index)
key = ""
} else {
key = this.rstring(index)
}
case 2:
if key != "" {
pairs[key] = this.rnumber(index)
key = ""
}
case 3:
if key != "" {
pairs[key] = true
key = ""
}
case 4:
if key != "" {
pairs[key] = false
key = ""
}
case 5:
if key != "" {
pairs[key] = nil
key = ""
}
case 6:
this.rpair(index, pairs)
}
start += size
}
}
}
func (this *PrefixDB) Lookup(address net.IP, input map[string]interface{}) (output map[string]interface{}, err error) {
output = input
if this.data == nil || this.Total == 0 || this.Version == 0 || this.Strings[2] == 0 || this.Numbers[2] == 0 ||
this.Pairs[2] == 0 || this.Clusters[2] == 0 || this.Maps[2] == 0 || this.Nodes[2] == 0 || address == nil {
err = errors.New("record not found")
} else {
address = address.To16()
offset := 0
if cores > 1 {
this.RLock()
}
for bit := 0; bit < 128; bit++ {
down := 0
if (address[bit/8] & (1 << (7 - (byte(bit) % 8)))) != 0 {
down = 1
}
offset = rnbits(this.Nodes[3], offset, down, this.data[this.Nodes[2]:])
if offset == this.Nodes[1] || offset == 0 {
break
}
if output == nil {
output = map[string]interface{}{}
}
if offset > this.Nodes[1] {
offset -= this.Nodes[1]
if offset < this.Maps[0] {
offset += this.Maps[2]
key := ""
for offset < this.Maps[2]+this.Maps[0] {
section, index, size, last := rpbits(this.data[offset:])
switch section {
case 1:
if key != "" {
output[key] = this.rstring(index)
key = ""
} else {
key = this.rstring(index)
}
case 2:
if key != "" {
output[key] = this.rnumber(index)
key = ""
}
case 3:
if key != "" {
output[key] = true
key = ""
}
case 4:
if key != "" {
output[key] = false
key = ""
}
case 5:
if key != "" {
output[key] = nil
key = ""
}
case 6:
this.rpair(index, output)
case 7:
this.rcluster(index, output)
}
if last {
break
}
offset += size
}
}
break
}
}
if cores > 1 {
this.RUnlock()
}
}
return output, err
}

42
vendor/github.com/pyke369/golang-support/rcache/rcache.go generated vendored Normal file
View File

@ -0,0 +1,42 @@
package rcache
import (
"crypto/md5"
"regexp"
"runtime"
"sync"
)
var (
cores int
cache map[[16]byte]*regexp.Regexp = map[[16]byte]*regexp.Regexp{}
lock sync.RWMutex
)
func Get(expression string) *regexp.Regexp {
if cores == 0 {
cores = runtime.NumCPU()
}
key := md5.Sum([]byte(expression))
if cores > 1 {
lock.RLock()
}
if cache[key] != nil {
if cores > 1 {
defer lock.RUnlock()
}
return cache[key].Copy()
}
if cores > 1 {
lock.RUnlock()
}
if regex, err := regexp.Compile(expression); err == nil {
if cores > 1 {
lock.Lock()
defer lock.Unlock()
}
cache[key] = regex
return cache[key].Copy()
}
return nil
}

0
vendor/github.com/pyke369/golang-support/rpack/README.md generated vendored Normal file
View File

38
vendor/github.com/pyke369/golang-support/rpack/cmd/rpack.go generated vendored Normal file
View File

@ -0,0 +1,38 @@
package main
import (
"flag"
"fmt"
"os"
"path/filepath"
"github.com/pyke369/golang-support/rpack"
)
func main() {
var (
options flag.FlagSet
usemain bool
)
options = flag.FlagSet{Usage: func() {
fmt.Fprintf(os.Stderr, "usage: %s [options] rootdir\n\noptions are:\n\n", filepath.Base(os.Args[0]))
options.PrintDefaults()
},
}
options.String("output", "static.go", "the generated output file path")
options.String("pkgname", "main", "the package name to use in the generated output")
options.String("funcname", "resources", "the function name to use in the generated output")
options.Bool("main", false, "whether to generate a main func or not")
if err := options.Parse(os.Args[1:]); err != nil {
os.Exit(1)
}
if options.NArg() == 0 {
options.Usage()
os.Exit(1)
}
if options.Lookup("main").Value.String() == "true" {
usemain = true
}
rpack.Pack(options.Arg(0), options.Lookup("output").Value.String(), options.Lookup("pkgname").Value.String(), options.Lookup("funcname").Value.String(), usemain)
}

155
vendor/github.com/pyke369/golang-support/rpack/rpack.go generated vendored Normal file
View File

@ -0,0 +1,155 @@
package rpack
import (
"bytes"
"compress/gzip"
"crypto/md5"
"encoding/base64"
"fmt"
"io/ioutil"
"math/rand"
"mime"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"time"
)
type RPACK struct {
Compressed bool
Modified int64
Mime string
Content string
raw []byte
}
var guzpool = sync.Pool{
New: func() interface{} {
return &gzip.Reader{}
}}
func Pack(root, output, pkgname, funcname string, main bool) {
root = strings.TrimSuffix(root, "/")
if root == "" || output == "" {
return
}
if pkgname == "" || main {
pkgname = "main"
}
if funcname == "" {
funcname = "resources"
}
funcname = strings.ToUpper(funcname[:1]) + funcname[1:]
entries := map[string]*RPACK{}
compressor, _ := gzip.NewWriterLevel(nil, gzip.BestCompression)
count := 0
size := int64(0)
start := time.Now()
filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
rpath := strings.TrimPrefix(path, root+"/")
if info.Mode()&os.ModeType == 0 {
for _, part := range strings.Split(rpath, "/") {
if len(part) > 0 && part[0] == '.' {
return nil
}
}
pack := &RPACK{Modified: info.ModTime().Unix(), Mime: "text/plain"}
if mime := mime.TypeByExtension(filepath.Ext(rpath)); mime != "" {
pack.Mime = mime
}
content, _ := ioutil.ReadFile(path)
compressed := bytes.Buffer{}
compressor.Reset(&compressed)
compressor.Write(content)
compressor.Close()
if compressed.Len() < len(content) {
pack.Content = base64.StdEncoding.EncodeToString(compressed.Bytes())
pack.Compressed = true
} else {
pack.Content = base64.StdEncoding.EncodeToString(content)
}
entries[rpath] = pack
fmt.Fprintf(os.Stderr, "\r%-120.120s ", rpath)
count++
size += info.Size()
}
return nil
})
fmt.Fprintf(os.Stderr, "\r%-120.120s\rpacked %d file(s) %d byte(s) in %v\n", "", count, size, time.Now().Sub(start))
if handle, err := os.OpenFile(output, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644); err == nil {
random := make([]byte, 64)
rand.Seed(time.Now().UnixNano() + int64(os.Getpid()))
rand.Read(random)
uid := fmt.Sprintf("rpack%8.8x", md5.Sum(random))
fmt.Fprintf(handle,
`package %s
import (
"net/http"
"github.com/pyke369/golang-support/rpack"
)
var %s map[string]*rpack.RPACK = map[string]*rpack.RPACK {
`,
pkgname, uid)
for path, entry := range entries {
fmt.Fprintf(handle,
` "%s": &rpack.RPACK{Compressed:%t, Modified:%d, Mime:"%s", Content:"%s"},
`, path, entry.Compressed, entry.Modified, entry.Mime, entry.Content)
}
fmt.Fprintf(handle,
`}
func %s() http.Handler {
return rpack.Serve(%s)
}
`, funcname, uid)
if main {
fmt.Fprintf(handle,
`
func main() {
http.Handle("/resources/", http.StripPrefix("/resources/", %s()))
http.ListenAndServe(":8000", nil)
}
`, funcname)
}
handle.Close()
}
}
func Serve(pack map[string]*RPACK) http.Handler {
return http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
var err error
if pack == nil || pack[request.URL.Path] == nil {
response.WriteHeader(http.StatusNotFound)
return
}
if pack[request.URL.Path].raw == nil {
if pack[request.URL.Path].raw, err = base64.StdEncoding.DecodeString(pack[request.URL.Path].Content); err != nil {
response.WriteHeader(http.StatusNotFound)
return
}
}
resource := pack[request.URL.Path]
response.Header().Set("Content-Type", resource.Mime)
if strings.Index(request.Header.Get("Accept-Encoding"), "gzip") >= 0 && request.Header.Get("Range") == "" && resource.Compressed {
response.Header().Set("Content-Encoding", "gzip")
response.Header().Set("Content-Length", fmt.Sprintf("%d", len(resource.raw)))
http.ServeContent(response, request, request.URL.Path, time.Unix(resource.Modified, 0), bytes.NewReader(resource.raw))
} else {
if resource.Compressed {
decompressor := guzpool.Get().(*gzip.Reader)
decompressor.Reset(bytes.NewReader(resource.raw))
if raw, err := ioutil.ReadAll(decompressor); err == nil {
http.ServeContent(response, request, request.URL.Path, time.Unix(resource.Modified, 0), bytes.NewReader(raw))
}
decompressor.Close()
guzpool.Put(decompressor)
} else {
http.ServeContent(response, request, request.URL.Path, time.Unix(resource.Modified, 0), bytes.NewReader(resource.raw))
}
}
})
}

0
vendor/github.com/pyke369/golang-support/uconfig/README.md generated vendored Normal file
View File

612
vendor/github.com/pyke369/golang-support/uconfig/uconfig.go generated vendored Normal file
View File

@ -0,0 +1,612 @@
package uconfig
import (
"crypto/sha1"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"math"
"net/http"
"os"
"os/exec"
"path"
"path/filepath"
"reflect"
"regexp"
"runtime"
"strconv"
"strings"
"sync"
"time"
)
type UConfig struct {
config interface{}
hash string
cache map[string]interface{}
cacheLock sync.RWMutex
sync.RWMutex
}
type replacer struct {
search *regexp.Regexp
replace string
loop bool
}
var (
cores int
escaped string
unescaper *regexp.Regexp
requoter *regexp.Regexp
expander *regexp.Regexp
sizer *regexp.Regexp
duration1 *regexp.Regexp
duration2 *regexp.Regexp
replacers [16]replacer
)
func init() {
escaped = "{}[],#/*;:= " // match characters within quotes to escape
unescaper = regexp.MustCompile("@\\d+@") // match escaped characters (to reverse previous escaping)
expander = regexp.MustCompile("{{([<|@&!\\-\\+])\\s*([^{}]*?)\\s*}}") // match external content macros
sizer = regexp.MustCompile("^(\\d+(?:\\.\\d*)?)\\s*([KMGTP]?)(B?)$") // match size value
duration1 = regexp.MustCompile("(\\d+)(Y|MO|D|H|MN|S|MS|US)?") // match duration value form1 (free)
duration2 = regexp.MustCompile("^(?:(\\d+):)?(\\d{2}):(\\d{2})(?:\\.(\\d{1,3}))?$") // match duration value form2 (timecode)
replacers[0] = replacer{regexp.MustCompile("(?m)^(.*?)(?:#|//).*?$"), "$1", false} // remove # and // commented portions
replacers[1] = replacer{regexp.MustCompile("/\\*[^\\*]*\\*/"), "", true} // remove /* */ commented portions
replacers[2] = replacer{regexp.MustCompile("(?m)^\\s+"), "", false} // trim leading spaces
replacers[3] = replacer{regexp.MustCompile("(?m)\\s+$"), "", false} // trim trailing spaces
replacers[4] = replacer{regexp.MustCompile("(?s)(^|[\r\n]+)\\[([^\\]\r\n]+?)\\](.+?)((?:[\r\n]+\\[)|$)"), "$1$2\n{$3\n}$4", true} // convert INI sections into JSON objects
replacers[5] = replacer{regexp.MustCompile("(?m)^(\\S+)\\s+([^{}\\[\\],;:=]+);$"), "$1 = $2;", false} // add missing key-value separators
replacers[6] = replacer{regexp.MustCompile("(?m);$"), ",", false} // replace ; line terminators by ,
replacers[7] = replacer{regexp.MustCompile("(\\S+?)\\s*[:=]"), "$1:", false} // replace = key-value separators by :
replacers[8] = replacer{regexp.MustCompile("([}\\]])(\\s*)([^,}\\]\\s])"), "$1,$2$3", false} // add missing objects/arrays , separators
replacers[9] = replacer{regexp.MustCompile("(?m)(^[^:]+:.+?[^,])$"), "$1,", false} // add missing values trailing , seperators
replacers[10] = replacer{regexp.MustCompile("(^|[,{\\[]+\\s*)([^:{\\[]+?)(\\s*[{\\[])"), "$1$2:$3", true} // add missing key-(object/array-)value separator
replacers[11] = replacer{regexp.MustCompile("(?m)^([^\":{}\\[\\]]+)"), "\"$1\"", false} // add missing quotes around keys
replacers[12] = replacer{regexp.MustCompile("([:,\\[\\s]+)([^\",\\[\\]{}\n\r]+?)(\\s*[,\\]}])"), "$1\"$2\"$3", false} // add missing quotes around values
replacers[13] = replacer{regexp.MustCompile("\"[\r\n]"), "\",\n", false} // add still issing objects/arrays , separators
replacers[14] = replacer{regexp.MustCompile("\"\\s*(.+?)\\s*\""), "\"$1\"", false} // trim leading and trailing spaces in quoted strings
replacers[15] = replacer{regexp.MustCompile(",+(\\s*[}\\]])"), "$1", false} // remove objets/arrays last element extra ,
}
func escape(input string) string {
var output []byte
instring := false
for index := 0; index < len(input); index++ {
if input[index:index+1] == "\"" && (index == 0 || input[index-1:index] != "\\") {
instring = !instring
}
if instring == true {
offset := strings.IndexAny(escaped, input[index:index+1])
if offset >= 0 {
output = append(output, []byte(fmt.Sprintf("@%02d@", offset))...)
} else {
output = append(output, input[index:index+1]...)
}
} else {
output = append(output, input[index:index+1]...)
}
}
return string(output)
}
func unescape(input string) string {
return unescaper.ReplaceAllStringFunc(input, func(a string) string {
offset, _ := strconv.Atoi(a[1:3])
if offset < len(escaped) {
return escaped[offset : offset+1]
}
return ""
})
}
func reduce(input interface{}) {
if input != nil {
switch reflect.TypeOf(input).Kind() {
case reflect.Map:
for key, _ := range input.(map[string]interface{}) {
var parts []string
for _, value := range strings.Split(key, " ") {
if value != "" {
parts = append(parts, value)
}
}
if len(parts) > 1 {
if input.(map[string]interface{})[parts[0]] == nil || reflect.TypeOf(input.(map[string]interface{})[parts[0]]).Kind() != reflect.Map {
input.(map[string]interface{})[parts[0]] = make(map[string]interface{})
}
input.(map[string]interface{})[parts[0]].(map[string]interface{})[parts[1]] = input.(map[string]interface{})[key]
delete(input.(map[string]interface{}), key)
}
}
for _, value := range input.(map[string]interface{}) {
reduce(value)
}
case reflect.Slice:
for index := 0; index < len(input.([]interface{})); index++ {
reduce(input.([]interface{})[index])
}
}
}
}
func New(input string, inline ...bool) (*UConfig, error) {
if cores == 0 {
cores = runtime.NumCPU()
}
config := &UConfig{
config: nil,
}
return config, config.Load(input, inline...)
}
func (this *UConfig) Load(input string, inline ...bool) error {
if cores > 1 {
this.Lock()
defer this.Unlock()
}
this.cache = map[string]interface{}{}
base, _ := os.Getwd()
content := fmt.Sprintf("/*base:%s*/\n", base)
if len(inline) > 0 && inline[0] {
content += input
} else {
content += fmt.Sprintf("{{<%s}}", input)
}
for {
indexes := expander.FindStringSubmatchIndex(content)
if indexes == nil {
content = escape(content)
for index := 0; index < len(replacers); index++ {
mcontent := ""
for mcontent != content {
mcontent = content
content = replacers[index].search.ReplaceAllString(content, replacers[index].replace)
if !replacers[index].loop {
break
}
}
}
content = unescape(strings.Trim(content, " \n,"))
break
}
expanded := ""
arguments := strings.Split(content[indexes[4]:indexes[5]], " ")
if start := strings.LastIndex(content[0:indexes[2]], "/*base:"); start >= 0 {
if end := strings.Index(content[start+7:indexes[2]], "*/"); end > 0 {
base = content[start+7 : start+7+end]
}
}
switch content[indexes[2]:indexes[3]] {
case "<":
if arguments[0][0:1] != "/" {
arguments[0] = fmt.Sprintf("%s/%s", base, arguments[0])
}
base = ""
if elements, err := filepath.Glob(arguments[0]); err == nil {
for _, element := range elements {
if mcontent, err := ioutil.ReadFile(element); err == nil {
base = filepath.Dir(element)
expanded += string(mcontent)
}
}
}
if base != "" && strings.Index(expanded, "\n") >= 0 {
expanded = fmt.Sprintf("/*base:%s*/\n%s\n", base, expanded)
}
case "|":
if arguments[0][0:1] != "/" {
arguments[0] = fmt.Sprintf("%s/%s", base, arguments[0])
}
base = ""
if elements, err := filepath.Glob(arguments[0]); err == nil {
for _, element := range elements {
if element[0:1] != "/" {
element = fmt.Sprintf("%s/%s", base, element)
}
if mcontent, err := exec.Command(element, strings.Join(arguments[1:], " ")).Output(); err == nil {
base = filepath.Dir(element)
expanded += string(mcontent)
}
}
}
if base != "" && strings.Index(expanded, "\n") >= 0 {
expanded = fmt.Sprintf("/*base:%s*/\n%s\n", base, expanded)
}
case "@":
requester := http.Client{
Timeout: time.Duration(5 * time.Second),
}
if response, err := requester.Get(arguments[0]); err == nil {
if (response.StatusCode / 100) == 2 {
if mcontent, err := ioutil.ReadAll(response.Body); err == nil {
expanded += string(mcontent)
}
}
}
case "&":
expanded += os.Getenv(arguments[0])
case "!":
arguments[0] = strings.ToLower(arguments[0])
for index := 1; index < len(os.Args); index++ {
option := strings.ToLower(os.Args[index])
if option == "--"+arguments[0] {
if index == len(os.Args)-1 || strings.HasPrefix(os.Args[index+1], "--") {
expanded = "true"
} else {
expanded = os.Args[index+1]
}
break
}
}
case "-":
if index := strings.Index(os.Args[0], "-"); index >= 0 {
expanded = strings.ToLower(os.Args[0][index+1:])
}
case "+":
if arguments[0][0:1] != "/" {
arguments[0] = fmt.Sprintf("%s/%s", base, arguments[0])
}
if elements, err := filepath.Glob(arguments[0]); err == nil {
for _, element := range elements {
element = path.Base(element)
expanded += fmt.Sprintf("%s ", strings.TrimSuffix(element, filepath.Ext(element)))
}
}
expanded = strings.TrimSpace(expanded)
}
content = fmt.Sprintf("%s%s%s", content[0:indexes[0]], expanded, content[indexes[1]:len(content)])
}
this.hash = fmt.Sprintf("%x", sha1.Sum([]byte(content)))
if err := json.Unmarshal([]byte(content), &this.config); err != nil {
if err := json.Unmarshal([]byte("{"+content+"}"), &this.config); err != nil {
this.config = nil
if syntax, ok := err.(*json.SyntaxError); ok && syntax.Offset < int64(len(content)) {
if start := strings.LastIndex(content[:syntax.Offset], "\n") + 1; start >= 0 {
line := strings.Count(content[:start], "\n") + 1
return errors.New(fmt.Sprintf("%s at line %d near %s", syntax, line, content[start:syntax.Offset]))
}
}
return err
}
}
reduce(this.config)
return nil
}
func (this *UConfig) Loaded() bool {
if cores > 1 {
this.RLock()
defer this.RUnlock()
}
return !(this.config == nil)
}
func (this *UConfig) Hash() string {
if cores > 1 {
this.RLock()
defer this.RUnlock()
}
return this.hash
}
func (this *UConfig) String() string {
if config, err := json.MarshalIndent(this.config, " ", " "); this.config != nil && err == nil {
return string(config)
}
return "{}"
}
func (this *UConfig) GetPaths(path string) []string {
var (
current interface{} = this.config
paths []string = []string{}
)
if cores > 1 {
this.RLock()
defer this.RUnlock()
}
prefix := ""
if current == nil || path == "" {
return paths
}
if cores > 1 {
this.cacheLock.RLock()
}
if this.cache[path] != nil {
if paths, ok := this.cache[path].([]string); ok {
if cores > 1 {
this.cacheLock.RUnlock()
}
return paths
}
}
if cores > 1 {
this.cacheLock.RUnlock()
}
if path != "" {
prefix = "."
for _, part := range strings.Split(path, ".") {
kind := reflect.TypeOf(current).Kind()
index, err := strconv.Atoi(part)
if (kind == reflect.Slice && (err != nil || index < 0 || index >= len(current.([]interface{})))) || (kind != reflect.Slice && kind != reflect.Map) {
if cores > 1 {
this.cacheLock.Lock()
}
this.cache[path] = paths
if cores > 1 {
this.cacheLock.Unlock()
}
return paths
}
if kind == reflect.Slice {
current = current.([]interface{})[index]
} else {
if current = current.(map[string]interface{})[strings.TrimSpace(part)]; current == nil {
if cores > 1 {
this.cacheLock.Lock()
}
this.cache[path] = paths
if cores > 1 {
this.cacheLock.Unlock()
}
return paths
}
}
}
}
switch reflect.TypeOf(current).Kind() {
case reflect.Slice:
for index := 0; index < len(current.([]interface{})); index++ {
paths = append(paths, fmt.Sprintf("%s%s%d", path, prefix, index))
}
case reflect.Map:
for key, _ := range current.(map[string]interface{}) {
paths = append(paths, fmt.Sprintf("%s%s%s", path, prefix, key))
}
}
if cores > 1 {
this.cacheLock.Lock()
}
this.cache[path] = paths
if cores > 1 {
this.cacheLock.Unlock()
}
return paths
}
func (this *UConfig) value(path string) (string, error) {
var current interface{} = this.config
if cores > 1 {
this.RLock()
defer this.RUnlock()
}
if current == nil || path == "" {
return "", fmt.Errorf("invalid parameter")
}
if cores > 1 {
this.cacheLock.RLock()
}
if this.cache[path] != nil {
if current, ok := this.cache[path].(bool); ok && !current {
if cores > 1 {
this.cacheLock.RUnlock()
}
return "", fmt.Errorf("invalid path")
}
if current, ok := this.cache[path].(string); ok {
if cores > 1 {
this.cacheLock.RUnlock()
}
return current, nil
}
}
if cores > 1 {
this.cacheLock.RUnlock()
}
for _, part := range strings.Split(path, ".") {
kind := reflect.TypeOf(current).Kind()
index, err := strconv.Atoi(part)
if (kind == reflect.Slice && (err != nil || index < 0 || index >= len(current.([]interface{})))) || (kind != reflect.Slice && kind != reflect.Map) {
if cores > 1 {
this.cacheLock.Lock()
}
this.cache[path] = false
if cores > 1 {
this.cacheLock.Unlock()
}
return "", fmt.Errorf("invalid path")
}
if kind == reflect.Slice {
current = current.([]interface{})[index]
} else {
if current = current.(map[string]interface{})[strings.TrimSpace(part)]; current == nil {
if cores > 1 {
this.cacheLock.Lock()
}
this.cache[path] = false
if cores > 1 {
this.cacheLock.Unlock()
}
return "", fmt.Errorf("invalid path")
}
}
}
if reflect.TypeOf(current).Kind() == reflect.String {
if cores > 1 {
this.cacheLock.Lock()
}
this.cache[path] = current.(string)
if cores > 1 {
this.cacheLock.Unlock()
}
return current.(string), nil
}
if cores > 1 {
this.cacheLock.Lock()
}
this.cache[path] = false
if cores > 1 {
this.cacheLock.Unlock()
}
return "", fmt.Errorf("invalid path")
}
func (this *UConfig) GetBoolean(path string, fallback bool) bool {
value, err := this.value(path)
if err != nil {
return fallback
}
if value = strings.ToLower(strings.TrimSpace(value)); value == "1" || value == "on" || value == "yes" || value == "true" {
return true
}
return false
}
func (this *UConfig) GetString(path string, fallback string) string {
return this.GetStringMatch(path, fallback, "")
}
func (this *UConfig) GetStringMatch(path string, fallback, match string) string {
return this.GetStringMatchCaptures(path, fallback, match)[0]
}
func (this *UConfig) GetStringMatchCaptures(path string, fallback, match string) []string {
value, err := this.value(path)
if err != nil {
return []string{fallback}
}
if match != "" {
if matcher, err := regexp.Compile(match); err == nil {
if matches := matcher.FindStringSubmatch(value); matches != nil {
return matches
} else {
return []string{fallback}
}
} else {
return []string{fallback}
}
}
return []string{value}
}
func (this *UConfig) GetInteger(path string, fallback int64) int64 {
return this.GetIntegerBounds(path, fallback, math.MinInt64, math.MaxInt64)
}
func (this *UConfig) GetIntegerBounds(path string, fallback, min, max int64) int64 {
value, err := this.value(path)
if err != nil {
return fallback
}
nvalue, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64)
if err != nil {
return fallback
}
if nvalue < min {
nvalue = min
}
if nvalue > max {
nvalue = max
}
return nvalue
}
func (this *UConfig) GetFloat(path string, fallback float64) float64 {
return this.GetFloatBounds(path, fallback, -math.MaxFloat64, math.MaxFloat64)
}
func (this *UConfig) GetFloatBounds(path string, fallback, min, max float64) float64 {
value, err := this.value(path)
if err != nil {
return fallback
}
nvalue, err := strconv.ParseFloat(strings.TrimSpace(value), 64)
if err != nil {
return fallback
}
return math.Max(math.Min(nvalue, max), min)
}
func (this *UConfig) GetSize(path string, fallback int64) int64 {
return this.GetSizeBounds(path, fallback, math.MinInt64, math.MaxInt64)
}
func (this *UConfig) GetSizeBounds(path string, fallback, min, max int64) int64 {
value, err := this.value(path)
if err != nil {
return fallback
}
nvalue := int64(0)
if matches := sizer.FindStringSubmatch(strings.TrimSpace(strings.ToUpper(value))); matches != nil {
fvalue, err := strconv.ParseFloat(matches[1], 64)
if err != nil {
return fallback
}
scale := float64(1000)
if matches[3] == "B" {
scale = float64(1024)
}
nvalue = int64(fvalue * math.Pow(scale, float64(strings.Index("_KMGTP", matches[2]))))
} else {
return fallback
}
if nvalue < min {
nvalue = min
}
if nvalue > max {
nvalue = max
}
return nvalue
}
func (this *UConfig) GetDuration(path string, fallback float64) float64 {
return this.GetDurationBounds(path, fallback, -math.MaxFloat64, math.MaxFloat64)
}
func (this *UConfig) GetDurationBounds(path string, fallback, min, max float64) float64 {
value, err := this.value(path)
if err != nil {
return fallback
}
nvalue := float64(0.0)
if matches := duration1.FindAllStringSubmatch(strings.TrimSpace(strings.ToUpper(value)), -1); matches != nil {
for index := 0; index < len(matches); index++ {
if uvalue, err := strconv.ParseFloat(matches[index][1], 64); err == nil {
switch matches[index][2] {
case "Y":
nvalue += uvalue * 86400 * 365.256
case "MO":
nvalue += uvalue * 86400 * 365.256 / 12
case "D":
nvalue += uvalue * 86400
case "H":
nvalue += uvalue * 3600
case "MN":
nvalue += uvalue * 60
case "S":
nvalue += uvalue
case "":
nvalue += uvalue
case "MS":
nvalue += uvalue / 1000
case "US":
nvalue += uvalue / 1000000
}
}
}
}
if matches := duration2.FindStringSubmatch(strings.TrimSpace(value)); matches != nil {
hours, _ := strconv.ParseFloat(matches[1], 64)
minutes, _ := strconv.ParseFloat(matches[2], 64)
seconds, _ := strconv.ParseFloat(matches[3], 64)
milliseconds, _ := strconv.ParseFloat(matches[4], 64)
nvalue = (hours * 3600) + (math.Min(minutes, 59) * 60) + math.Min(seconds, 59) + (milliseconds / 1000)
}
return math.Max(math.Min(nvalue, max), min)
}

0
vendor/github.com/pyke369/golang-support/ulog/README.md generated vendored Normal file
View File

545
vendor/github.com/pyke369/golang-support/ulog/ulog.go generated vendored Normal file
View File

@ -0,0 +1,545 @@
package ulog
import (
"bytes"
"encoding/json"
"fmt"
"io"
"log/syslog"
"os"
"path/filepath"
"reflect"
"regexp"
"runtime"
"strings"
"sync"
"syscall"
"time"
"unsafe"
)
const (
TIME_NONE int = iota
TIME_DATETIME
TIME_MSDATETIME
TIME_TIMESTAMP
TIME_MSTIMESTAMP
)
var (
cores int
facilities = map[string]syslog.Priority{
"user": syslog.LOG_USER,
"daemon": syslog.LOG_DAEMON,
"local0": syslog.LOG_LOCAL0,
"local1": syslog.LOG_LOCAL1,
"local2": syslog.LOG_LOCAL2,
"local3": syslog.LOG_LOCAL3,
"local4": syslog.LOG_LOCAL4,
"local5": syslog.LOG_LOCAL5,
"local6": syslog.LOG_LOCAL6,
"local7": syslog.LOG_LOCAL7,
}
severities = map[string]syslog.Priority{
"error": syslog.LOG_ERR,
"warning": syslog.LOG_WARNING,
"info": syslog.LOG_INFO,
"debug": syslog.LOG_DEBUG,
}
severityLabels = map[syslog.Priority]string{
syslog.LOG_ERR: "ERRO ",
syslog.LOG_WARNING: "WARN ",
syslog.LOG_INFO: "INFO ",
syslog.LOG_DEBUG: "DBUG ",
}
severityColors = map[syslog.Priority]string{
syslog.LOG_ERR: "\x1b[31m",
syslog.LOG_WARNING: "\x1b[33m",
syslog.LOG_INFO: "\x1b[36m",
syslog.LOG_DEBUG: "\x1b[32m",
}
)
type FileOutput struct {
handle *os.File
last time.Time
}
type ULog struct {
file, console, syslog bool
fileOutputs map[string]*FileOutput
filePath string
fileTime int
fileLast time.Time
fileSeverity bool
consoleHandle io.Writer
consoleTime int
consoleSeverity bool
consoleColors bool
syslogHandle *syslog.Writer
syslogRemote string
syslogName string
syslogFacility syslog.Priority
optionUTC bool
level syslog.Priority
sync.Mutex
}
func New(target string) *ULog {
if cores == 0 {
cores = runtime.NumCPU()
}
log := &ULog{
fileOutputs: map[string]*FileOutput{},
syslogHandle: nil,
}
return log.Load(target)
}
func (this *ULog) Load(target string) *ULog {
this.Close()
if cores > 1 {
this.Lock()
defer this.Unlock()
}
this.file = false
this.filePath = ""
this.fileTime = TIME_DATETIME
this.fileSeverity = true
this.console = false
this.consoleTime = TIME_DATETIME
this.consoleSeverity = true
this.consoleColors = true
this.consoleHandle = os.Stderr
this.syslog = false
this.syslogRemote = ""
this.syslogName = filepath.Base(os.Args[0])
this.syslogFacility = syslog.LOG_DAEMON
this.optionUTC = false
this.level = syslog.LOG_INFO
for _, target := range regexp.MustCompile("(file|console|syslog|option)\\s*\\(([^\\)]*)\\)").FindAllStringSubmatch(target, -1) {
switch strings.ToLower(target[1]) {
case "file":
this.file = true
for _, option := range regexp.MustCompile("([^:=,\\s]+)\\s*[:=]\\s*([^,\\s]+)").FindAllStringSubmatch(target[2], -1) {
switch strings.ToLower(option[1]) {
case "path":
this.filePath = option[2]
case "time":
option[2] = strings.ToLower(option[2])
switch {
case option[2] == "datetime":
this.fileTime = TIME_DATETIME
case option[2] == "msdatetime":
this.fileTime = TIME_MSDATETIME
case option[2] == "stamp" || option[2] == "timestamp":
this.fileTime = TIME_TIMESTAMP
case option[2] == "msstamp" || option[2] == "mstimestamp":
this.fileTime = TIME_MSTIMESTAMP
case option[2] != "1" && option[2] != "true" && option[2] != "on" && option[2] != "yes":
this.fileTime = TIME_NONE
}
case "severity":
option[2] = strings.ToLower(option[2])
if option[2] != "1" && option[2] != "true" && option[2] != "on" && option[2] != "yes" {
this.fileSeverity = false
}
}
}
if this.filePath == "" {
this.file = false
}
case "console":
this.console = true
for _, option := range regexp.MustCompile("([^:=,\\s]+)\\s*[:=]\\s*([^,\\s]+)").FindAllStringSubmatch(target[2], -1) {
option[2] = strings.ToLower(option[2])
switch strings.ToLower(option[1]) {
case "output":
if option[2] == "stdout" {
this.consoleHandle = os.Stdout
}
case "time":
switch {
case option[2] == "datetime":
this.consoleTime = TIME_DATETIME
case option[2] == "msdatetime":
this.consoleTime = TIME_MSDATETIME
case option[2] == "stamp" || option[2] == "timestamp":
this.consoleTime = TIME_TIMESTAMP
case option[2] == "msstamp" || option[2] == "mstimestamp":
this.consoleTime = TIME_MSTIMESTAMP
case option[2] != "1" && option[2] != "true" && option[2] != "on" && option[2] != "yes":
this.consoleTime = TIME_NONE
}
case "severity":
if option[2] != "1" && option[2] != "true" && option[2] != "on" && option[2] != "yes" {
this.consoleSeverity = false
}
case "colors":
if option[2] != "1" && option[2] != "true" && option[2] != "on" && option[2] != "yes" {
this.consoleColors = false
}
}
}
case "syslog":
this.syslog = true
for _, option := range regexp.MustCompile("([^:=,\\s]+)\\s*[:=]\\s*([^,\\s]+)").FindAllStringSubmatch(target[2], -1) {
switch strings.ToLower(option[1]) {
case "remote":
this.syslogRemote = option[2]
if !regexp.MustCompile(":\\d+$").MatchString(this.syslogRemote) {
this.syslogRemote += ":514"
}
case "name":
this.syslogName = option[2]
case "facility":
this.syslogFacility = facilities[strings.ToLower(option[2])]
}
}
case "option":
for _, option := range regexp.MustCompile("([^:=,\\s]+)\\s*[:=]\\s*([^,\\s]+)").FindAllStringSubmatch(target[2], -1) {
option[2] = strings.ToLower(option[2])
switch strings.ToLower(option[1]) {
case "utc":
if option[2] == "1" || option[2] == "true" || option[2] == "on" || option[2] == "yes" {
this.optionUTC = true
}
case "level":
this.level = severities[strings.ToLower(option[2])]
}
}
}
}
var info syscall.Termios
if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, this.consoleHandle.(*os.File).Fd(), syscall.TCGETS, uintptr(unsafe.Pointer(&info)), 0, 0, 0); err != 0 {
this.consoleColors = false
}
return this
}
func (this *ULog) Close() {
if cores > 1 {
this.Lock()
defer this.Unlock()
}
if this.syslogHandle != nil {
this.syslogHandle.Close()
this.syslogHandle = nil
}
for path, output := range this.fileOutputs {
if output.handle != nil {
output.handle.Close()
}
delete(this.fileOutputs, path)
}
}
func (this *ULog) SetLevel(level string) {
level = strings.ToLower(level)
switch level {
case "error":
this.level = syslog.LOG_ERR
case "warning":
this.level = syslog.LOG_WARNING
case "info":
this.level = syslog.LOG_INFO
case "debug":
this.level = syslog.LOG_DEBUG
}
}
func strftime(layout string, base time.Time) string {
var output []string
length := len(layout)
for index := 0; index < length; index++ {
switch layout[index] {
case '%':
if index < length-1 {
switch layout[index+1] {
case 'a':
output = append(output, base.Format("Mon"))
case 'A':
output = append(output, base.Format("Monday"))
case 'b':
output = append(output, base.Format("Jan"))
case 'B':
output = append(output, base.Format("January"))
case 'c':
output = append(output, base.Format("Mon Jan 2 15:04:05 2006"))
case 'C':
output = append(output, fmt.Sprintf("%02d", base.Year()/100))
case 'd':
output = append(output, fmt.Sprintf("%02d", base.Day()))
case 'D':
output = append(output, fmt.Sprintf("%02d/%02d/%02d", base.Month(), base.Day(), base.Year()%100))
case 'e':
output = append(output, fmt.Sprintf("%2d", base.Day()))
case 'f':
output = append(output, fmt.Sprintf("%06d", base.Nanosecond()/1000))
case 'F':
output = append(output, fmt.Sprintf("%04d-%02d-%02d", base.Year(), base.Month(), base.Day()))
case 'g':
year, _ := base.ISOWeek()
output = append(output, fmt.Sprintf("%02d", year%100))
case 'G':
year, _ := base.ISOWeek()
output = append(output, fmt.Sprintf("%04d", year))
case 'h':
output = append(output, base.Format("Jan"))
case 'H':
output = append(output, fmt.Sprintf("%02d", base.Hour()))
case 'I':
if base.Hour() == 0 || base.Hour() == 12 {
output = append(output, "12")
} else {
output = append(output, fmt.Sprintf("%02d", base.Hour()%12))
}
case 'j':
output = append(output, fmt.Sprintf("%03d", base.YearDay()))
case 'k':
output = append(output, fmt.Sprintf("%2d", base.Hour()))
case 'l':
if base.Hour() == 0 || base.Hour() == 12 {
output = append(output, "12")
} else {
output = append(output, fmt.Sprintf("%2d", base.Hour()%12))
}
case 'm':
output = append(output, fmt.Sprintf("%02d", base.Month()))
case 'M':
output = append(output, fmt.Sprintf("%02d", base.Minute()))
case 'n':
output = append(output, "\n")
case 'p':
if base.Hour() < 12 {
output = append(output, "AM")
} else {
output = append(output, "PM")
}
case 'P':
if base.Hour() < 12 {
output = append(output, "am")
} else {
output = append(output, "pm")
}
case 'r':
if base.Hour() == 0 || base.Hour() == 12 {
output = append(output, "12")
} else {
output = append(output, fmt.Sprintf("%02d", base.Hour()%12))
}
output = append(output, fmt.Sprintf(":%02d:%02d", base.Minute(), base.Second()))
if base.Hour() < 12 {
output = append(output, " AM")
} else {
output = append(output, " PM")
}
case 'R':
output = append(output, fmt.Sprintf("%02d:%02d", base.Hour(), base.Minute()))
case 's':
output = append(output, fmt.Sprintf("%d", base.Unix()))
case 'S':
output = append(output, fmt.Sprintf("%02d", base.Second()))
case 't':
output = append(output, "\t")
case 'T':
output = append(output, fmt.Sprintf("%02d:%02d:%02d", base.Hour(), base.Minute(), base.Second()))
case 'u':
day := base.Weekday()
if day == 0 {
day = 7
}
output = append(output, fmt.Sprintf("%d", day))
case 'U':
output = append(output, fmt.Sprintf("%d", (base.YearDay()+6-int(base.Weekday()))/7))
case 'V':
_, week := base.ISOWeek()
output = append(output, fmt.Sprintf("%02d", week))
case 'w':
output = append(output, fmt.Sprintf("%d", base.Weekday()))
case 'W':
day := int(base.Weekday())
if day == 0 {
day = 6
} else {
day -= 1
}
output = append(output, fmt.Sprintf("%d", (base.YearDay()+6-day)/7))
case 'x':
output = append(output, fmt.Sprintf("%02d/%02d/%02d", base.Month(), base.Day(), base.Year()%100))
case 'X':
output = append(output, fmt.Sprintf("%02d:%02d:%02d", base.Hour(), base.Minute(), base.Second()))
case 'y':
output = append(output, fmt.Sprintf("%02d", base.Year()%100))
case 'Y':
output = append(output, fmt.Sprintf("%04d", base.Year()))
case 'z':
output = append(output, base.Format("-0700"))
case 'Z':
output = append(output, base.Format("MST"))
case '%':
output = append(output, "%")
}
index++
}
default:
output = append(output, string(layout[index]))
}
}
return strings.Join(output, "")
}
func (this *ULog) log(now time.Time, severity syslog.Priority, xlayout interface{}, a ...interface{}) {
var err error
if this.level < severity || (!this.syslog && !this.file && !this.console) {
return
}
layout := ""
switch reflect.TypeOf(xlayout).Kind() {
case reflect.Map:
var buffer bytes.Buffer
encoder := json.NewEncoder(&buffer)
encoder.SetEscapeHTML(false)
if err := encoder.Encode(xlayout); err == nil {
layout = "%s"
a = []interface{}{bytes.TrimSpace(buffer.Bytes())}
}
case reflect.String:
layout = xlayout.(string)
}
layout = strings.TrimSpace(layout)
if this.syslog {
if this.syslogHandle == nil {
if cores > 1 {
this.Lock()
}
if this.syslogHandle == nil {
protocol := ""
if this.syslogRemote != "" {
protocol = "udp"
}
if this.syslogHandle, err = syslog.Dial(protocol, this.syslogRemote, this.syslogFacility, this.syslogName); err != nil {
this.syslogHandle = nil
}
}
if cores > 1 {
this.Unlock()
}
}
if this.syslogHandle != nil {
switch severity {
case syslog.LOG_ERR:
this.syslogHandle.Err(fmt.Sprintf(layout, a...))
case syslog.LOG_WARNING:
this.syslogHandle.Warning(fmt.Sprintf(layout, a...))
case syslog.LOG_INFO:
this.syslogHandle.Info(fmt.Sprintf(layout, a...))
case syslog.LOG_DEBUG:
this.syslogHandle.Debug(fmt.Sprintf(layout, a...))
}
}
}
if this.optionUTC {
now = now.UTC()
} else {
now = now.Local()
}
if this.file {
path := strftime(this.filePath, now)
if cores > 1 {
this.Lock()
}
if this.fileOutputs[path] == nil {
os.MkdirAll(filepath.Dir(path), 0755)
if handle, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644); err == nil {
this.fileOutputs[path] = &FileOutput{handle: handle}
}
}
if this.fileOutputs[path] != nil && this.fileOutputs[path].handle != nil {
prefix := ""
switch this.fileTime {
case TIME_DATETIME:
prefix = fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d ", now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second())
case TIME_MSDATETIME:
prefix = fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d.%03d ", now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second(), now.Nanosecond()/int(time.Millisecond))
case TIME_TIMESTAMP:
prefix = fmt.Sprintf("%d ", now.Unix())
case TIME_MSTIMESTAMP:
prefix = fmt.Sprintf("%d ", now.UnixNano()/int64(time.Millisecond))
}
if this.fileSeverity {
prefix += severityLabels[severity]
}
this.fileOutputs[path].handle.WriteString(fmt.Sprintf(prefix+layout+"\n", a...))
this.fileOutputs[path].last = now
}
if now.Sub(this.fileLast) >= 5*time.Second {
this.fileLast = now
for path, output := range this.fileOutputs {
if now.Sub(output.last) >= 5*time.Second {
output.handle.Close()
delete(this.fileOutputs, path)
}
}
}
if cores > 1 {
this.Unlock()
}
}
if this.console {
prefix := ""
switch this.consoleTime {
case TIME_DATETIME:
prefix = fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d ", now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second())
case TIME_MSDATETIME:
prefix = fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d.%03d ", now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second(), now.Nanosecond()/int(time.Millisecond))
case TIME_TIMESTAMP:
prefix = fmt.Sprintf("%d ", now.Unix())
case TIME_MSTIMESTAMP:
prefix = fmt.Sprintf("%d ", now.UnixNano()/int64(time.Millisecond))
}
if this.consoleSeverity {
if this.consoleColors {
prefix += fmt.Sprintf("%s%s\x1b[0m", severityColors[severity], severityLabels[severity])
} else {
prefix += severityLabels[severity]
}
}
if cores > 1 {
this.Lock()
}
fmt.Fprintf(this.consoleHandle, prefix+layout+"\n", a...)
if cores > 1 {
this.Unlock()
}
}
}
func (this *ULog) Error(layout interface{}, a ...interface{}) {
this.log(time.Now(), syslog.LOG_ERR, layout, a...)
}
func (this *ULog) Warn(layout interface{}, a ...interface{}) {
this.log(time.Now(), syslog.LOG_WARNING, layout, a...)
}
func (this *ULog) Info(layout interface{}, a ...interface{}) {
this.log(time.Now(), syslog.LOG_INFO, layout, a...)
}
func (this *ULog) Debug(layout interface{}, a ...interface{}) {
this.log(time.Now(), syslog.LOG_DEBUG, layout, a...)
}
func (this *ULog) ErrorTime(now time.Time, layout interface{}, a ...interface{}) {
this.log(now, syslog.LOG_ERR, layout, a...)
}
func (this *ULog) WarnTime(now time.Time, layout interface{}, a ...interface{}) {
this.log(now, syslog.LOG_WARNING, layout, a...)
}
func (this *ULog) InfoTime(now time.Time, layout interface{}, a ...interface{}) {
this.log(now, syslog.LOG_INFO, layout, a...)
}
func (this *ULog) DebugTime(now time.Time, layout interface{}, a ...interface{}) {
this.log(now, syslog.LOG_DEBUG, layout, a...)
}

0
vendor/github.com/pyke369/golang-support/uuid/README.md generated vendored Normal file
View File

30
vendor/github.com/pyke369/golang-support/uuid/uuid.go generated vendored Normal file
View File

@ -0,0 +1,30 @@
package uuid
import (
"fmt"
"math/rand"
"os"
"time"
)
var initialized bool
func init() {
if !initialized {
rand.Seed(time.Now().UnixNano() + int64(os.Getpid()))
initialized = true
}
}
func UUID() string {
var entropy = make([]byte, 16)
if !initialized {
rand.Seed(time.Now().UnixNano() + int64(os.Getpid()))
initialized = true
}
rand.Read(entropy)
entropy[6] = (entropy[6] & 0x0f) | 0x40
entropy[8] = (entropy[8] & 0x3f) | 0x80
return fmt.Sprintf("%08x-%04x-%04x-%04x-%12x", entropy[0:4], entropy[4:6], entropy[6:8], entropy[8:10], entropy[10:16])
}

0
vendor/github.com/pyke369/golang-support/whohas/README.md generated vendored Normal file
View File

244
vendor/github.com/pyke369/golang-support/whohas/whohas.go generated vendored Normal file
View File

@ -0,0 +1,244 @@
package whohas
import (
"context"
"mime"
"net/http"
"path/filepath"
"runtime"
"strconv"
"strings"
"sync"
"time"
)
type BACKEND struct {
Host string
Secure bool
Path string
Headers map[string]string
Penalty time.Duration
}
type CACHE struct {
TTL time.Duration
last time.Time
items map[string]*LOOKUP
sync.RWMutex
}
type LOOKUP struct {
index int
deadline time.Time
Protocol string
Host string
Headers map[string]string
Size int64
Mime string
Ranges bool
Date time.Time
Modified time.Time
Expires time.Time
}
var cores int
func Lookup(path string, backends []BACKEND, timeout time.Duration, cache *CACHE, ckey string) (lookup *LOOKUP) {
if cores == 0 {
cores = runtime.NumCPU()
}
if path == "" || backends == nil || len(backends) < 1 || timeout < 100*time.Millisecond {
return
}
cpath := path
if index := strings.Index(path, "?"); index >= 0 {
cpath = path[:index]
}
cbackends := backends
if cache != nil && cache.items != nil {
now := time.Now()
if cores > 1 {
cache.RLock()
}
if cache.items[cpath] != nil && now.Sub(cache.items[cpath].deadline) < 0 {
lookup = cache.items[cpath]
if cache.items[cpath].Host != "" {
cbackends = []BACKEND{}
for _, backend := range backends {
if backend.Host == cache.items[cpath].Host {
cbackends = append(cbackends, backend)
break
}
}
if len(cbackends) < 1 {
cbackends = backends
}
}
}
if len(cbackends) == len(backends) && ckey != "" && cache.items["k"+ckey] != nil && now.Sub(cache.items["k"+ckey].deadline) < 0 {
cbackends = []BACKEND{}
for _, backend := range backends {
if backend.Host == cache.items["k"+ckey].Host {
cbackends = append(cbackends, backend)
break
}
}
if len(cbackends) < 1 {
cbackends = backends
}
}
if cores > 1 {
cache.RUnlock()
}
}
if lookup == nil {
inflight := len(cbackends)
sink := make(chan LOOKUP, inflight+1)
cancels := make([]context.CancelFunc, inflight)
for index, backend := range cbackends {
var ctx context.Context
ctx, cancels[index] = context.WithCancel(context.Background())
go func(index int, backend BACKEND, ctx context.Context) {
lookup := LOOKUP{index: index}
if backend.Penalty != 0 && len(cbackends) > 1 {
select {
case <-time.After(backend.Penalty):
case <-ctx.Done():
sink <- lookup
return
}
}
lookup.Protocol = "http"
if backend.Secure {
lookup.Protocol = "https"
}
rpath := path
if backend.Path != "" {
rpath = backend.Path
}
if request, err := http.NewRequest(http.MethodHead, lookup.Protocol+"://"+backend.Host+rpath, nil); err == nil {
request = request.WithContext(ctx)
request.Header.Set("User-Agent", "whohas")
if backend.Headers != nil {
lookup.Headers = map[string]string{}
for name, value := range backend.Headers {
lookup.Headers[name] = value
request.Header.Set(name, value)
}
}
if response, err := http.DefaultClient.Do(request); err == nil {
if response.StatusCode == 200 {
lookup.Host = backend.Host
lookup.Size, _ = strconv.ParseInt(response.Header.Get("Content-Length"), 10, 64)
lookup.Mime = response.Header.Get("Content-Type")
if lookup.Mime == "" || lookup.Mime == "application/octet-stream" || lookup.Mime == "text/plain" {
if extension := filepath.Ext(path); extension != "" {
lookup.Mime = mime.TypeByExtension(extension)
}
}
if response.Header.Get("Accept-Ranges") != "" {
lookup.Ranges = true
}
if header := response.Header.Get("Date"); header != "" {
lookup.Date, _ = http.ParseTime(header)
} else {
lookup.Date = time.Now()
}
if header := response.Header.Get("Last-Modified"); header != "" {
lookup.Modified, _ = http.ParseTime(header)
}
if header := response.Header.Get("Expires"); header != "" {
lookup.Expires, _ = http.ParseTime(header)
} else {
lookup.Expires = lookup.Date.Add(time.Hour)
}
if lookup.Expires.Sub(lookup.Date) < 2*time.Second {
lookup.Expires = lookup.Date.Add(2 * time.Second)
}
}
response.Body.Close()
}
}
sink <- lookup
}(index, backend, ctx)
}
for inflight > 0 {
select {
case result := <-sink:
inflight--
cancels[result.index] = nil
if result.Host != "" {
lookup = &result
for index, cancel := range cancels {
if cancels[index] != nil && index != result.index {
cancel()
cancels[index] = nil
}
}
}
case <-time.After(timeout):
for index, cancel := range cancels {
if cancels[index] != nil {
cancel()
}
}
}
}
close(sink)
}
if cache != nil {
now := time.Now()
if cores > 1 {
cache.Lock()
}
if cache.items == nil {
cache.items = map[string]*LOOKUP{}
}
if now.Sub(cache.last) >= 5*time.Second {
cache.last = now
for key, item := range cache.items {
if now.Sub(item.deadline) >= 0 {
delete(cache.items, key)
}
}
}
if lookup == nil || lookup.Host == "" {
if ckey != "" {
delete(cache.items, "k"+ckey)
}
if cache.items[cpath] == nil {
cache.items[cpath] = &LOOKUP{deadline: now.Add(5 * time.Second)}
}
lookup = nil
} else {
if cache.TTL < 2*time.Second {
cache.TTL = 2 * time.Second
}
if ckey != "" {
cache.items["k"+ckey] = &LOOKUP{Host: lookup.Host, deadline: now.Add(cache.TTL)}
}
if cache.items[cpath] == nil {
ttl := lookup.Expires.Sub(lookup.Date)
if ttl < 2*time.Second {
ttl = 2 * time.Second
}
if ttl > cache.TTL {
ttl = cache.TTL
}
if ttl > 10*time.Minute {
ttl = 10 * time.Minute
}
lookup.deadline = now.Add(ttl)
cache.items[cpath] = lookup
}
}
if cores > 1 {
cache.Unlock()
}
}
return
}