read configuration file
This commit is contained in:
Xavier Henner
parent
9dc7d19811
commit
dd38706b0b
49
httpd.go
49
httpd.go
|
|
@ -1,15 +1,24 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
)
|
||||
|
||||
type HttpServer struct {
|
||||
Port string
|
||||
ovpn *OpenVpnMgt
|
||||
key string
|
||||
cert string
|
||||
certPool *x509.CertPool
|
||||
}
|
||||
|
||||
func (h *HttpServer) handler(w http.ResponseWriter, r *http.Request) {
|
||||
|
|
@ -33,19 +42,51 @@ func (h *HttpServer) helpHandler(w http.ResponseWriter, r *http.Request) {
|
|||
err, message := h.ovpn.Help()
|
||||
if err != nil {
|
||||
fmt.Fprintf(w, "Error : %s", err)
|
||||
} else {
|
||||
fmt.Fprintf(w, "%s", message)
|
||||
}
|
||||
|
||||
jsonStr, err := json.Marshal(message)
|
||||
if err != nil {
|
||||
fmt.Fprintf(w, "Error : %s", err)
|
||||
}
|
||||
fmt.Fprintf(w, "%s", jsonStr)
|
||||
}
|
||||
|
||||
func NewHTTPServer(port string, s *OpenVpnMgt) {
|
||||
func NewHTTPServer(port, key, cert, ca string, s *OpenVpnMgt) {
|
||||
h := &HttpServer{
|
||||
Port: port,
|
||||
ovpn: s,
|
||||
key: key,
|
||||
cert: cert,
|
||||
}
|
||||
|
||||
http.HandleFunc("/help", h.helpHandler)
|
||||
http.HandleFunc("/version", h.versionHandler)
|
||||
http.HandleFunc("/", h.handler)
|
||||
|
||||
switch {
|
||||
case key == "" || cert == "":
|
||||
log.Fatal(http.ListenAndServe(port, nil))
|
||||
case ca != "":
|
||||
h.certPool = x509.NewCertPool()
|
||||
fi, err := os.Open(ca)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer fi.Close()
|
||||
buf := new(bytes.Buffer)
|
||||
reader := bufio.NewReader(fi)
|
||||
io.Copy(buf, reader)
|
||||
if ok := h.certPool.AppendCertsFromPEM(buf.Bytes()); !ok {
|
||||
log.Fatal("Failed to append PEM.")
|
||||
}
|
||||
server := &http.Server{
|
||||
Addr: port,
|
||||
TLSConfig: &tls.Config{
|
||||
ClientAuth: tls.RequestClientCert,
|
||||
ClientCAs: h.certPool,
|
||||
},
|
||||
}
|
||||
log.Fatal(server.ListenAndServeTLS(cert, key))
|
||||
default:
|
||||
log.Fatal(http.ListenAndServeTLS(port, cert, key, nil))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
54
main.go
54
main.go
|
|
@ -1,7 +1,57 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"log"
|
||||
"log/syslog"
|
||||
"os"
|
||||
|
||||
"github.com/pyke369/golang-support/uconfig"
|
||||
)
|
||||
|
||||
var config *uconfig.UConfig
|
||||
|
||||
func main() {
|
||||
server := NewVPNServer(":4000")
|
||||
var err error
|
||||
// default configuration file is ./openvpn-dm-mgt-server.conf
|
||||
configFile := flag.String("config", "openvpn-dm-mgt-server.conf", "configuration file")
|
||||
logToSyslog := flag.Bool("syslog", false, "Log to syslog")
|
||||
flag.Parse()
|
||||
|
||||
// parseconfig
|
||||
if config, err = uconfig.New(*configFile); err != nil {
|
||||
log.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
server := NewVPNServer(config.GetString("config.openvpnPort", "127.0.0.01:5000"))
|
||||
server.vpnlogUrl = config.GetString("config.vpnLogUrl", "")
|
||||
server.mailRelay = config.GetString("config.mailRelay", "")
|
||||
server.MailFrom = config.GetString("config.mailFrom", "")
|
||||
server.CcPwnPassword = config.GetString("config.ccPwnPassword", "")
|
||||
server.pwnTemplate = config.GetString("config.pwnTemplate", "")
|
||||
server.newAsTemplate = config.GetString("config.newAsTemplate", "")
|
||||
server.slackTemplate = config.GetString("config.slackTemplate", "")
|
||||
server.slackTemplate2 = config.GetString("config.slackTemplate2", "")
|
||||
server.cacheDir = config.GetString("config.cacheDir", "")
|
||||
server.authCa = config.GetString("config.authCa", "")
|
||||
|
||||
server.syslog = false
|
||||
if *logToSyslog {
|
||||
log.SetFlags(0)
|
||||
server.syslog = true
|
||||
logWriter, e := syslog.New(syslog.LOG_NOTICE, "")
|
||||
if e == nil {
|
||||
log.SetOutput(logWriter)
|
||||
defer logWriter.Close()
|
||||
}
|
||||
}
|
||||
|
||||
go server.Run()
|
||||
NewHTTPServer(":8080", server)
|
||||
NewHTTPServer(
|
||||
config.GetString("config.httpPort", "127.0.0.01:8080"),
|
||||
config.GetString("config.httpKey", ""),
|
||||
config.GetString("config.httpCert", ""),
|
||||
config.GetString("config.httpCa", ""),
|
||||
server)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,10 +3,10 @@ package main
|
|||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
|
@ -18,12 +18,16 @@ type OpenVpnMgt struct {
|
|||
connected bool
|
||||
m sync.RWMutex
|
||||
ret chan []string
|
||||
authCa string
|
||||
vpnlogUrl string
|
||||
mailRelay string
|
||||
MailFrom string
|
||||
CcPwnPassword string
|
||||
pwnTemplate string
|
||||
newAsTemplate string
|
||||
slackTemplate string
|
||||
slackTemplate2 string
|
||||
cacheDir string
|
||||
syslog bool
|
||||
}
|
||||
|
||||
|
|
@ -40,12 +44,14 @@ func (s *OpenVpnMgt) Run() {
|
|||
// Resolve the passed port into an address
|
||||
addrs, err := net.ResolveTCPAddr("tcp", s.Port)
|
||||
if err != nil {
|
||||
return
|
||||
log.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
// start listening to client connections
|
||||
listener, err := net.ListenTCP("tcp", addrs)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
log.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
// Infinite loop since we dont want the server to shut down
|
||||
for {
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
package bslab
|
||||
|
||||
import "sync/atomic"
|
||||
|
||||
type slab struct {
|
||||
queue chan []byte
|
||||
get, put, alloc, lost int64
|
||||
}
|
||||
|
||||
var (
|
||||
slabs = map[int]*slab{}
|
||||
)
|
||||
|
||||
func init() {
|
||||
slabs[0] = &slab{}
|
||||
for size := uint(9); size <= 26; size++ {
|
||||
slabs[1<<size] = &slab{queue: make(chan []byte, 1024)}
|
||||
}
|
||||
}
|
||||
|
||||
func Stats() (info map[int][5]int64) {
|
||||
info = map[int][5]int64{}
|
||||
for size, slab := range slabs {
|
||||
info[size] = [5]int64{slab.get, slab.put, slab.alloc, slab.lost, int64(len(slab.queue))}
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
func Get(size int, item []byte) []byte {
|
||||
if size <= 0 {
|
||||
return nil
|
||||
}
|
||||
if item != nil {
|
||||
if cap(item) >= size {
|
||||
return item[:0]
|
||||
}
|
||||
Put(item)
|
||||
}
|
||||
bits, power := uint(0), uint(0)
|
||||
if size&(size-1) == 0 {
|
||||
power = 1
|
||||
}
|
||||
for size != 0 {
|
||||
size >>= 1
|
||||
bits++
|
||||
}
|
||||
size = 1 << (bits - power)
|
||||
if slab, ok := slabs[size]; ok {
|
||||
atomic.AddInt64(&(slab.get), 1)
|
||||
select {
|
||||
case item := <-slab.queue:
|
||||
return item[:0]
|
||||
default:
|
||||
atomic.AddInt64(&(slab.alloc), 1)
|
||||
return make([]byte, 0, size)
|
||||
}
|
||||
}
|
||||
atomic.AddInt64(&(slabs[0].get), 1)
|
||||
atomic.AddInt64(&(slabs[0].alloc), int64(size))
|
||||
return make([]byte, 0, size)
|
||||
}
|
||||
|
||||
func Put(item []byte) {
|
||||
if item == nil || cap(item) <= 0 {
|
||||
return
|
||||
}
|
||||
size, bits := cap(item), uint(0)
|
||||
for size != 0 {
|
||||
size >>= 1
|
||||
bits++
|
||||
}
|
||||
size = 1 << (bits - 1)
|
||||
if size > 0 && float64(cap(item))/float64(size) <= 1.2 {
|
||||
if slab, ok := slabs[size]; ok {
|
||||
atomic.AddInt64(&(slab.put), 1)
|
||||
select {
|
||||
case slab.queue <- item:
|
||||
default:
|
||||
atomic.AddInt64(&(slab.lost), 1)
|
||||
}
|
||||
} else {
|
||||
atomic.AddInt64(&(slabs[0].put), 1)
|
||||
atomic.AddInt64(&(slabs[0].lost), int64(cap(item)))
|
||||
}
|
||||
} else {
|
||||
atomic.AddInt64(&(slabs[0].put), 1)
|
||||
atomic.AddInt64(&(slabs[0].lost), int64(cap(item)))
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,356 @@
|
|||
package chash
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"os"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
CHASH_MAGIC uint32 = 0x48414843
|
||||
CHASH_REPLICAS = 128
|
||||
)
|
||||
|
||||
type item struct {
|
||||
hash uint32
|
||||
target uint16
|
||||
}
|
||||
|
||||
type CHash struct {
|
||||
targets map[string]uint8
|
||||
names []string
|
||||
ring []item
|
||||
ringSize uint32
|
||||
replicas uint8
|
||||
frozen bool
|
||||
sync.RWMutex
|
||||
}
|
||||
|
||||
var cores int
|
||||
|
||||
func init() {
|
||||
rand.Seed(time.Now().UnixNano() + int64(os.Getpid()))
|
||||
}
|
||||
|
||||
func mmhash2(key []byte, keySize int) uint32 {
|
||||
var magic, hash, current, value uint32 = 0x5bd1e995, uint32(0x4d4d4832 ^ keySize), 0, 0
|
||||
|
||||
if keySize < 0 {
|
||||
keySize = len(key)
|
||||
}
|
||||
for keySize >= 4 {
|
||||
value = uint32(key[current]) | uint32(key[current+1])<<8 |
|
||||
uint32(key[current+2])<<16 | uint32(key[current+3])<<24
|
||||
value *= magic
|
||||
value ^= value >> 24
|
||||
value *= magic
|
||||
hash *= magic
|
||||
hash ^= value
|
||||
current += 4
|
||||
keySize -= 4
|
||||
}
|
||||
if keySize >= 3 {
|
||||
hash ^= uint32(key[current+2]) << 16
|
||||
}
|
||||
if keySize >= 2 {
|
||||
hash ^= uint32(key[current+1]) << 8
|
||||
}
|
||||
if keySize >= 1 {
|
||||
hash ^= uint32(key[current])
|
||||
}
|
||||
if keySize != 0 {
|
||||
hash *= magic
|
||||
}
|
||||
hash ^= hash >> 13
|
||||
hash *= magic
|
||||
hash ^= hash >> 15
|
||||
return hash
|
||||
}
|
||||
|
||||
type ByHash []item
|
||||
|
||||
func (a ByHash) Len() int { return len(a) }
|
||||
func (a ByHash) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
||||
func (a ByHash) Less(i, j int) bool { return a[i].hash < a[j].hash }
|
||||
|
||||
func (this *CHash) freeze() {
|
||||
if cores > 1 {
|
||||
this.Lock()
|
||||
defer this.Unlock()
|
||||
}
|
||||
if this.frozen {
|
||||
return
|
||||
}
|
||||
this.ringSize = 0
|
||||
for _, tweight := range this.targets {
|
||||
this.ringSize += uint32(tweight) * uint32(this.replicas)
|
||||
}
|
||||
if this.ringSize == 0 {
|
||||
this.frozen = true
|
||||
return
|
||||
}
|
||||
|
||||
var (
|
||||
target uint16 = 0
|
||||
offset uint32 = 0
|
||||
key []byte = make([]byte, 128)
|
||||
)
|
||||
this.names = make([]string, len(this.targets))
|
||||
this.ring = make([]item, this.ringSize)
|
||||
for tname, tweight := range this.targets {
|
||||
this.names[target] = tname
|
||||
for weight := uint8(0); weight < tweight; weight++ {
|
||||
for replica := uint8(0); replica < this.replicas; replica++ {
|
||||
key = append(key[:0], tname...)
|
||||
key = strconv.AppendInt(key, int64(weight), 10)
|
||||
key = strconv.AppendInt(key, int64(replica), 10)
|
||||
this.ring[offset] = item{mmhash2(key, -1), target}
|
||||
offset++
|
||||
}
|
||||
}
|
||||
target++
|
||||
}
|
||||
sort.Sort(ByHash(this.ring))
|
||||
this.frozen = true
|
||||
}
|
||||
|
||||
func New(replicas ...uint8) *CHash {
|
||||
if cores == 0 {
|
||||
cores = runtime.NumCPU()
|
||||
}
|
||||
chash := &CHash{
|
||||
targets: make(map[string]uint8),
|
||||
names: nil,
|
||||
ring: nil,
|
||||
ringSize: 0,
|
||||
replicas: CHASH_REPLICAS,
|
||||
frozen: false,
|
||||
}
|
||||
if len(replicas) > 0 {
|
||||
chash.replicas = replicas[0]
|
||||
}
|
||||
if chash.replicas < 1 {
|
||||
chash.replicas = 1
|
||||
}
|
||||
if chash.replicas > CHASH_REPLICAS {
|
||||
chash.replicas = CHASH_REPLICAS
|
||||
}
|
||||
return chash
|
||||
}
|
||||
|
||||
func (this *CHash) AddTarget(name string, weight uint8) bool {
|
||||
if weight > 0 && weight <= 100 && len(name) <= 128 && this.targets[name] != weight {
|
||||
if cores > 1 {
|
||||
this.Lock()
|
||||
defer this.Unlock()
|
||||
}
|
||||
this.targets[name] = weight
|
||||
this.frozen = false
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
func (this *CHash) RemoveTarget(name string) bool {
|
||||
if cores > 1 {
|
||||
this.Lock()
|
||||
defer this.Unlock()
|
||||
}
|
||||
delete(this.targets, name)
|
||||
this.frozen = false
|
||||
return true
|
||||
}
|
||||
func (this *CHash) ClearTargets() bool {
|
||||
if cores > 1 {
|
||||
this.Lock()
|
||||
defer this.Unlock()
|
||||
}
|
||||
this.targets = make(map[string]uint8)
|
||||
this.frozen = false
|
||||
return true
|
||||
}
|
||||
|
||||
func (this *CHash) Serialize() []byte {
|
||||
this.freeze()
|
||||
if cores > 1 {
|
||||
this.RLock()
|
||||
defer this.RUnlock()
|
||||
}
|
||||
size := uint32(4) + 4 + 1 + 2 + 4
|
||||
for _, name := range this.names {
|
||||
size += 1 + 1 + uint32(len(name))
|
||||
}
|
||||
size += (this.ringSize * 6)
|
||||
serialized := make([]byte, size)
|
||||
offset := uint32(0)
|
||||
serialized[offset] = byte(CHASH_MAGIC & 0xff)
|
||||
serialized[offset+1] = byte((CHASH_MAGIC >> 8) & 0xff)
|
||||
serialized[offset+2] = byte((CHASH_MAGIC >> 16) & 0xff)
|
||||
serialized[offset+3] = byte((CHASH_MAGIC >> 24) & 0xff)
|
||||
serialized[offset+4] = byte(size & 0xff)
|
||||
serialized[offset+5] = byte((size >> 8) & 0xff)
|
||||
serialized[offset+6] = byte((size >> 16) & 0xff)
|
||||
serialized[offset+7] = byte((size >> 24) & 0xff)
|
||||
serialized[offset+8] = this.replicas
|
||||
serialized[offset+9] = byte(uint16(len(this.names)) & 0xff)
|
||||
serialized[offset+10] = byte(((uint16(len(this.names))) >> 8) & 0xff)
|
||||
serialized[offset+11] = byte(this.ringSize & 0xff)
|
||||
serialized[offset+12] = byte((this.ringSize >> 8) & 0xff)
|
||||
serialized[offset+13] = byte((this.ringSize >> 16) & 0xff)
|
||||
serialized[offset+14] = byte((this.ringSize >> 24) & 0xff)
|
||||
offset += 15
|
||||
for _, name := range this.names {
|
||||
serialized[offset] = this.targets[name]
|
||||
serialized[offset+1] = byte(len(name) & 0xff)
|
||||
copy(serialized[offset+2:offset+2+uint32(serialized[offset+1])], []byte(name))
|
||||
offset += 2 + uint32(serialized[offset+1])
|
||||
}
|
||||
for _, item := range this.ring {
|
||||
serialized[offset] = byte(item.hash & 0xff)
|
||||
serialized[offset+1] = byte((item.hash >> 8) & 0xff)
|
||||
serialized[offset+2] = byte((item.hash >> 16) & 0xff)
|
||||
serialized[offset+3] = byte((item.hash >> 24) & 0xff)
|
||||
serialized[offset+4] = byte(item.target & 0xff)
|
||||
serialized[offset+5] = byte((item.target >> 8) & 0xff)
|
||||
offset += 6
|
||||
}
|
||||
return serialized
|
||||
}
|
||||
func (this *CHash) FileSerialize(path string) bool {
|
||||
handle, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer handle.Close()
|
||||
if _, err := handle.Write(this.Serialize()); err != nil {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (this *CHash) Unserialize(serialized []byte) bool {
|
||||
if cores > 1 {
|
||||
this.Lock()
|
||||
defer this.Unlock()
|
||||
}
|
||||
if len(serialized) < 15 {
|
||||
return false
|
||||
}
|
||||
magic := uint32(serialized[0]) + (uint32(serialized[1]) << 8) + (uint32(serialized[2]) << 16) + (uint32(serialized[3]) << 24)
|
||||
size := uint32(serialized[4]) + (uint32(serialized[5]) << 8) + (uint32(serialized[6]) << 16) + (uint32(serialized[7]) << 24)
|
||||
replicas := serialized[8]
|
||||
names := uint16(serialized[9]) + (uint16(serialized[10]) << 8)
|
||||
ringSize := uint32(serialized[11]) + (uint32(serialized[12]) << 8) + (uint32(serialized[13]) << 16) + (uint32(serialized[14]) << 24)
|
||||
if magic != CHASH_MAGIC || size != uint32(len(serialized)) {
|
||||
return false
|
||||
}
|
||||
this.targets = make(map[string]uint8)
|
||||
this.names = make([]string, names)
|
||||
this.ring = make([]item, ringSize)
|
||||
this.ringSize = ringSize
|
||||
this.replicas = replicas
|
||||
offset := uint32(15)
|
||||
for index := uint16(0); index < names && offset < size; index++ {
|
||||
len := uint32(serialized[offset+1])
|
||||
this.names[index] = string(serialized[offset+2 : offset+2+len])
|
||||
this.targets[this.names[index]] = serialized[offset]
|
||||
offset += 2 + len
|
||||
}
|
||||
if offset > size {
|
||||
return false
|
||||
}
|
||||
for item := uint32(0); item < ringSize && offset < size; item++ {
|
||||
this.ring[item].hash = uint32(serialized[offset]) + (uint32(serialized[offset+1]) << 8) + (uint32(serialized[offset+2]) << 16) + (uint32(serialized[offset+3]) << 24)
|
||||
this.ring[item].target = uint16(serialized[offset+4]) + (uint16(serialized[offset+5]) << 8)
|
||||
offset += 6
|
||||
}
|
||||
if offset != size {
|
||||
return false
|
||||
}
|
||||
this.frozen = true
|
||||
return true
|
||||
}
|
||||
func (this *CHash) FileUnserialize(path string) bool {
|
||||
handle, err := os.OpenFile(path, os.O_RDONLY, 0)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer handle.Close()
|
||||
info, err := handle.Stat()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if info.Size() > 128*1024*1024 {
|
||||
return false
|
||||
}
|
||||
serialized := make([]byte, info.Size())
|
||||
read, err := handle.Read(serialized)
|
||||
if int64(read) != info.Size() || err != nil {
|
||||
return false
|
||||
}
|
||||
return this.Unserialize(serialized)
|
||||
}
|
||||
|
||||
func (this *CHash) Lookup(candidate string, count int) []string {
|
||||
var start uint32 = 0
|
||||
|
||||
this.freeze()
|
||||
if cores > 1 {
|
||||
this.RLock()
|
||||
defer this.RUnlock()
|
||||
}
|
||||
if count > len(this.targets) {
|
||||
count = len(this.targets)
|
||||
}
|
||||
if this.ringSize == 0 || count < 1 {
|
||||
return []string{}
|
||||
}
|
||||
hash := mmhash2([]byte(candidate), -1)
|
||||
if hash > this.ring[0].hash && hash <= this.ring[this.ringSize-1].hash {
|
||||
start = this.ringSize / 2
|
||||
span := start / 2
|
||||
for {
|
||||
if hash > this.ring[start].hash && hash <= this.ring[start+1].hash {
|
||||
break
|
||||
}
|
||||
if hash > this.ring[start].hash {
|
||||
start += span
|
||||
} else {
|
||||
start -= span
|
||||
}
|
||||
span /= 2
|
||||
if span < 1 {
|
||||
span = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
result := make([]string, count)
|
||||
rank := 0
|
||||
for rank < count {
|
||||
index := 0
|
||||
for index = 0; index < rank; index++ {
|
||||
if result[index] == this.names[this.ring[start].target] {
|
||||
break
|
||||
}
|
||||
}
|
||||
if index >= rank {
|
||||
result[rank] = this.names[this.ring[start].target]
|
||||
rank++
|
||||
}
|
||||
start++
|
||||
if start >= this.ringSize {
|
||||
start = 0
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
func (this *CHash) LookupBalance(candidate string, count int) string {
|
||||
result := this.Lookup(candidate, count)
|
||||
if len(result) > 0 {
|
||||
return result[rand.Intn(len(result))]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
package dynacert
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"os"
|
||||
"runtime"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type DYNACERT struct {
|
||||
Public, Key string
|
||||
Certificate *tls.Certificate
|
||||
Last, Modified time.Time
|
||||
sync.RWMutex
|
||||
}
|
||||
|
||||
var cores int
|
||||
|
||||
func (this *DYNACERT) GetCertificate(*tls.ClientHelloInfo) (cert *tls.Certificate, err error) {
|
||||
var info os.FileInfo
|
||||
|
||||
if cores == 0 {
|
||||
cores = runtime.NumCPU()
|
||||
}
|
||||
if this.Certificate == nil || time.Now().Sub(this.Last) >= 10*time.Second {
|
||||
this.Last = time.Now()
|
||||
if info, err = os.Stat(this.Public); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err = os.Stat(this.Key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if this.Certificate == nil || info.ModTime().Sub(this.Modified) != 0 {
|
||||
if certificate, err := tls.LoadX509KeyPair(this.Public, this.Key); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
if cores > 1 {
|
||||
this.Lock()
|
||||
}
|
||||
this.Modified = info.ModTime()
|
||||
this.Certificate = &certificate
|
||||
if cores > 1 {
|
||||
this.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if cores > 1 {
|
||||
this.RLock()
|
||||
defer this.RUnlock()
|
||||
}
|
||||
return this.Certificate, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
package fqdn
|
||||
|
||||
import (
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func FQDN() (string, string) {
|
||||
if hostname, err := os.Hostname(); err == nil {
|
||||
if addresses, err := net.LookupHost(hostname); err != nil {
|
||||
return hostname, "*"
|
||||
} else {
|
||||
for _, address := range addresses {
|
||||
if hostnames, err := net.LookupAddr(address); err == nil && len(hostnames) > 0 {
|
||||
for _, hostname := range hostnames {
|
||||
if strings.Count(hostname, ".") > 1 {
|
||||
hostname = strings.TrimSuffix(hostname, ".")
|
||||
addresses, _ = net.LookupHost(hostname)
|
||||
return hostname, addresses[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return "unknown", "*"
|
||||
}
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
package listener
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"syscall"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
type TCPListener struct {
|
||||
*net.TCPListener
|
||||
ReadBufferSize int
|
||||
WriteBufferSize int
|
||||
}
|
||||
|
||||
func (this *TCPListener) Accept() (net.Conn, error) {
|
||||
connection, err := this.AcceptTCP()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if rconnection, err := connection.SyscallConn(); err == nil {
|
||||
rconnection.Control(
|
||||
func(handle uintptr) {
|
||||
syscall.SetsockoptInt(int(handle), syscall.SOL_SOCKET, syscall.SO_KEEPALIVE, 1)
|
||||
syscall.SetsockoptInt(int(handle), syscall.IPPROTO_TCP, syscall.TCP_KEEPIDLE, 60)
|
||||
syscall.SetsockoptInt(int(handle), syscall.IPPROTO_TCP, syscall.TCP_KEEPINTVL, 10)
|
||||
syscall.SetsockoptInt(int(handle), syscall.IPPROTO_TCP, syscall.TCP_KEEPCNT, 3)
|
||||
})
|
||||
}
|
||||
if this.ReadBufferSize > 0 {
|
||||
connection.SetReadBuffer(this.ReadBufferSize)
|
||||
}
|
||||
if this.WriteBufferSize > 0 {
|
||||
connection.SetWriteBuffer(this.WriteBufferSize)
|
||||
}
|
||||
return connection, nil
|
||||
}
|
||||
|
||||
func NewTCPListener(network, address string, reuseport bool, read, write int) (listener *TCPListener, err error) {
|
||||
config := net.ListenConfig{
|
||||
Control: func(network, address string, connection syscall.RawConn) error {
|
||||
connection.Control(func(handle uintptr) {
|
||||
if err := syscall.SetsockoptInt(int(handle), syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1); err != nil {
|
||||
return
|
||||
}
|
||||
if reuseport {
|
||||
if err := syscall.SetsockoptInt(int(handle), syscall.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
return nil
|
||||
}}
|
||||
if clistener, err := config.Listen(context.Background(), network, address); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
return &TCPListener{clistener.(*net.TCPListener), read, write}, nil
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
PFDB binary layout
|
||||
------------------
|
||||
"PFDB"
|
||||
VVVV (4)
|
||||
HHHHHHHHHHHHHHHH (16)
|
||||
"DESC"
|
||||
"DDDDDDDDDDDDDDDDDDDD" (20)
|
||||
"STRS"
|
||||
W (1)
|
||||
SSSS (4)
|
||||
CCCC (4)
|
||||
...
|
||||
"NUMS"
|
||||
SSSS (4)
|
||||
CCCC (4)
|
||||
...
|
||||
"PAIR"
|
||||
SSSS (4)
|
||||
CCCC (4)
|
||||
...
|
||||
"CLUS"
|
||||
W (1)
|
||||
SSSS (4)
|
||||
CCCC (4)
|
||||
...
|
||||
"MAPS"
|
||||
SSSS (4)
|
||||
CCCC (4)
|
||||
...
|
||||
"NODE"
|
||||
W (1)
|
||||
SSSS (4)
|
||||
CCCC (4)
|
||||
...
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
prefixdb: prefixdb.go
|
||||
@go build prefixdb.go && strip prefixdb
|
||||
|
||||
run: prefixdb
|
||||
@#./prefixdb city city.pfdb@'MMCITY 20190402' GeoIP2-City-Locations-en.csv GeoIP2-City-Blocks-IPv4.csv GeoIP2-City-Blocks-IPv6.csv
|
||||
@#./prefixdb asn asn.pfdb@'MMASN 20190402' GeoLite2-ASN-Blocks-IPv4.csv GeoLite2-ASN-Blocks-IPv6.csv
|
||||
@./prefixdb lookup city.pfdb asn.pfdb 78.193.67.63 188.65.124.26
|
||||
@#./prefixdb server *:8000 city.pfdb asn.pfdb
|
||||
|
||||
clean:
|
||||
@rm -f prefixdb *.pfdb
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
module .
|
||||
|
||||
go 1.12
|
||||
|
||||
require github.com/pyke369/golang-support v0.0.0-20190428173758-fae1fcd33c43 // indirect
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
github.com/pyke369/golang-support v0.0.0-20190428173758-fae1fcd33c43 h1:638A4GSCbTc/Z8N1TyymmC8iWQOE3BWnWJv9fzZeHJc=
|
||||
github.com/pyke369/golang-support v0.0.0-20190428173758-fae1fcd33c43/go.mod h1:0XGrzgrEp0fa/+JSV8XZePUwyjnU6C3bMc7Xz2bHHKI=
|
||||
367
vendor/github.com/pyke369/golang-support/prefixdb/cmd/prefixdb.go
generated
vendored
Normal file
367
vendor/github.com/pyke369/golang-support/prefixdb/cmd/prefixdb.go
generated
vendored
Normal file
|
|
@ -0,0 +1,367 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pyke369/golang-support/prefixdb"
|
||||
)
|
||||
|
||||
type LOCATION struct {
|
||||
ContinentCode string
|
||||
ContinentName string
|
||||
CountryCode string
|
||||
CountryName string
|
||||
RegionCode string
|
||||
RegionName string
|
||||
StateCode string
|
||||
StateName string
|
||||
CityName string
|
||||
TimeZone string
|
||||
InEurope bool
|
||||
}
|
||||
|
||||
var (
|
||||
csvMatcher = regexp.MustCompile(`(?:,|\n|^)("(?:(?:"")*[^"]*)*"|[^",\n]*|(?:\n|$))`)
|
||||
jsonMatcher = regexp.MustCompile(`^(\S+)(?:\s(\{.+?\}))?$`)
|
||||
pfdb = prefixdb.New()
|
||||
)
|
||||
|
||||
func size(input int) string {
|
||||
if input < 1024*1024 {
|
||||
return fmt.Sprintf("%.1fkB", float64(input)/1024)
|
||||
}
|
||||
return fmt.Sprintf("%.1fMB", float64(input)/(1024*1024))
|
||||
}
|
||||
|
||||
func mkjson() {
|
||||
for index := 3; index < len(os.Args); index++ {
|
||||
count := 0
|
||||
if handle, err := os.Open(os.Args[index]); err == nil {
|
||||
reader := bufio.NewReader(handle)
|
||||
last := time.Now()
|
||||
start := last
|
||||
for {
|
||||
if line, err := reader.ReadString('\n'); err != nil {
|
||||
break
|
||||
} else {
|
||||
if fields := jsonMatcher.FindStringSubmatch(strings.TrimSpace(line)); fields != nil {
|
||||
if _, prefix, err := net.ParseCIDR(fields[1]); err == nil {
|
||||
data := map[string]interface{}{}
|
||||
json.Unmarshal([]byte(fields[2]), &data)
|
||||
pfdb.Add(*prefix, data, [][]string{[]string{"key1", "key2"}})
|
||||
count++
|
||||
}
|
||||
}
|
||||
}
|
||||
if now := time.Now(); now.Sub(last) >= 250*time.Millisecond {
|
||||
last = now
|
||||
fmt.Fprintf(os.Stderr, "\r- adding prefixes [%s] %d", os.Args[index], count)
|
||||
}
|
||||
}
|
||||
handle.Close()
|
||||
fmt.Fprintf(os.Stderr, "\r- added prefixes [%s] (%.3fs - %d entries)\n", os.Args[index], float64(time.Now().Sub(start))/float64(time.Second), count)
|
||||
}
|
||||
}
|
||||
start := time.Now()
|
||||
description := ""
|
||||
if index := strings.Index(os.Args[2], "@"); index > 0 {
|
||||
description = os.Args[2][index+1:]
|
||||
os.Args[2] = os.Args[2][:index]
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "\r- saving database [%s]... ", os.Args[2])
|
||||
if _, err := pfdb.Save(os.Args[2], description); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "\r- saving database [%s] failed (%v)\n", os.Args[2], err)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "\r- saved database [%s] (%.3fs - total[%s] strings[%s] numbers[%s] pairs[%s] clusters[%s] maps[%s] nodes[%s])\n",
|
||||
os.Args[2], float64(time.Now().Sub(start))/float64(time.Second), size(pfdb.Total), size(pfdb.Strings[0]),
|
||||
size(pfdb.Numbers[0]), size(pfdb.Pairs[0]), size(pfdb.Clusters[0]), size(pfdb.Maps[0]), size(pfdb.Nodes[0]),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func mkcity() {
|
||||
locations := map[int]*LOCATION{}
|
||||
if handle, err := os.Open(os.Args[3]); err == nil {
|
||||
reader := bufio.NewReader(handle)
|
||||
last := time.Now()
|
||||
start := last
|
||||
for {
|
||||
if line, err := reader.ReadString('\n'); err != nil {
|
||||
break
|
||||
} else {
|
||||
if fields := csvMatcher.FindAllStringSubmatch(strings.TrimSpace(line), -1); len(fields) == 14 {
|
||||
for index := 0; index < len(fields); index++ {
|
||||
fields[index][1] = strings.Trim(fields[index][1], `"`)
|
||||
}
|
||||
if id, err := strconv.Atoi(fields[0][1]); err == nil {
|
||||
locations[id] = &LOCATION{
|
||||
ContinentCode: fields[2][1],
|
||||
ContinentName: fields[3][1],
|
||||
CountryCode: fields[4][1],
|
||||
CountryName: fields[5][1],
|
||||
RegionCode: fields[6][1],
|
||||
RegionName: fields[7][1],
|
||||
StateCode: fields[8][1],
|
||||
StateName: fields[9][1],
|
||||
CityName: fields[10][1],
|
||||
TimeZone: fields[12][1],
|
||||
InEurope: fields[13][1] == "1",
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if now := time.Now(); now.Sub(last) >= 250*time.Millisecond {
|
||||
last = now
|
||||
fmt.Fprintf(os.Stderr, "\r- loading locations [%s] %d", os.Args[3], len(locations))
|
||||
}
|
||||
}
|
||||
handle.Close()
|
||||
fmt.Fprintf(os.Stderr, "\r- loaded locations [%s] (%.3fs - %d entries)\n", os.Args[3], float64(time.Now().Sub(start))/float64(time.Second), len(locations))
|
||||
}
|
||||
|
||||
clusters := [][]string{
|
||||
[]string{"continent_code", "continent_name", "country_code", "country_name", "region_code", "region_name", "state_code", "state_name", "timezone", "in_europe"},
|
||||
[]string{"city_name", "postal_code", "latitude", "longitude"},
|
||||
}
|
||||
for index := 4; index < len(os.Args); index++ {
|
||||
count := 0
|
||||
if handle, err := os.Open(os.Args[index]); err == nil {
|
||||
reader := bufio.NewReader(handle)
|
||||
last := time.Now()
|
||||
start := last
|
||||
for {
|
||||
if line, err := reader.ReadString('\n'); err != nil {
|
||||
break
|
||||
} else {
|
||||
if fields := strings.Split(strings.TrimSpace(line), ","); len(fields) == 10 {
|
||||
id := 0
|
||||
if id, _ = strconv.Atoi(fields[1]); id == 0 {
|
||||
id, _ = strconv.Atoi(fields[2])
|
||||
}
|
||||
if id != 0 && locations[id] != nil {
|
||||
if _, prefix, err := net.ParseCIDR(fields[0]); err == nil {
|
||||
latitude, _ := strconv.ParseFloat(fields[7], 64)
|
||||
longitude, _ := strconv.ParseFloat(fields[8], 64)
|
||||
pfdb.Add(*prefix, map[string]interface{}{
|
||||
"continent_code": locations[id].ContinentCode,
|
||||
"continent_name": locations[id].ContinentName,
|
||||
"country_code": locations[id].CountryCode,
|
||||
"country_name": locations[id].CountryName,
|
||||
"region_code": locations[id].RegionCode,
|
||||
"region_name": locations[id].RegionName,
|
||||
"state_code": locations[id].StateCode,
|
||||
"state_name": locations[id].StateName,
|
||||
"city_name": locations[id].CityName,
|
||||
"in_europe": locations[id].InEurope,
|
||||
"timezone": locations[id].TimeZone,
|
||||
"postal_code": fields[6],
|
||||
"latitude": latitude,
|
||||
"longitude": longitude,
|
||||
}, clusters)
|
||||
count++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if now := time.Now(); now.Sub(last) >= 250*time.Millisecond {
|
||||
last = now
|
||||
fmt.Fprintf(os.Stderr, "\r- adding prefixes [%s] %d", os.Args[index], count)
|
||||
}
|
||||
}
|
||||
handle.Close()
|
||||
fmt.Fprintf(os.Stderr, "\r- added prefixes [%s] (%.3fs - %d entries)\n", os.Args[index], float64(time.Now().Sub(start))/float64(time.Second), count)
|
||||
}
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
description := ""
|
||||
if index := strings.Index(os.Args[2], "@"); index > 0 {
|
||||
description = os.Args[2][index+1:]
|
||||
os.Args[2] = os.Args[2][:index]
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "\r- saving database [%s]... ", os.Args[2])
|
||||
if _, err := pfdb.Save(os.Args[2], description); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "\r- saving database [%s] failed (%v)\n", os.Args[2], err)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "\r- saved database [%s] (%.3fs - total[%s] strings[%s] numbers[%s] pairs[%s] clusters[%s] maps[%s] nodes[%s])\n",
|
||||
os.Args[2], float64(time.Now().Sub(start))/float64(time.Second), size(pfdb.Total), size(pfdb.Strings[0]),
|
||||
size(pfdb.Numbers[0]), size(pfdb.Pairs[0]), size(pfdb.Clusters[0]), size(pfdb.Maps[0]), size(pfdb.Nodes[0]),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func mkasn() {
|
||||
for index := 3; index < len(os.Args); index++ {
|
||||
count := 0
|
||||
if handle, err := os.Open(os.Args[index]); err == nil {
|
||||
reader := bufio.NewReader(handle)
|
||||
last := time.Now()
|
||||
start := last
|
||||
for {
|
||||
if line, err := reader.ReadString('\n'); err != nil {
|
||||
break
|
||||
} else {
|
||||
if fields := csvMatcher.FindAllStringSubmatch(strings.TrimSpace(line), -1); len(fields) == 3 {
|
||||
for index := 0; index < len(fields); index++ {
|
||||
fields[index][1] = strings.Trim(fields[index][1], `"`)
|
||||
}
|
||||
if asnum, _ := strconv.Atoi(fields[1][1]); asnum != 0 {
|
||||
if _, prefix, err := net.ParseCIDR(fields[0][1]); err == nil {
|
||||
pfdb.Add(*prefix, map[string]interface{}{
|
||||
"as_number": fmt.Sprintf("AS%d", asnum),
|
||||
"as_name": fields[2][1],
|
||||
}, nil)
|
||||
count++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if now := time.Now(); now.Sub(last) >= 250*time.Millisecond {
|
||||
last = now
|
||||
fmt.Fprintf(os.Stderr, "\r- adding prefixes [%s] %d", os.Args[index], count)
|
||||
}
|
||||
}
|
||||
handle.Close()
|
||||
fmt.Fprintf(os.Stderr, "\r- added prefixes [%s] (%.3fs - %d entries)\n", os.Args[index], float64(time.Now().Sub(start))/float64(time.Second), count)
|
||||
}
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
description := ""
|
||||
if index := strings.Index(os.Args[2], "@"); index > 0 {
|
||||
description = os.Args[2][index+1:]
|
||||
os.Args[2] = os.Args[2][:index]
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "\r- saving database [%s]... ", os.Args[2])
|
||||
if _, err := pfdb.Save(os.Args[2], description); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "\r- saving database [%s] failed (%v)\n", os.Args[2], err)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "\r- saved database [%s] (%.3fs - total[%s] strings[%s] numbers[%s] pairs[%s] clusters[%s] maps[%s] nodes[%s])\n",
|
||||
os.Args[2], float64(time.Now().Sub(start))/float64(time.Second), size(pfdb.Total), size(pfdb.Strings[0]),
|
||||
size(pfdb.Numbers[0]), size(pfdb.Pairs[0]), size(pfdb.Clusters[0]), size(pfdb.Maps[0]), size(pfdb.Nodes[0]),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func lookup() {
|
||||
databases := []*prefixdb.PrefixDB{}
|
||||
for index := 2; index < len(os.Args); index++ {
|
||||
if strings.HasSuffix(os.Args[index], `.pfdb`) {
|
||||
fmt.Fprintf(os.Stderr, "\r- loading database [%s]...", os.Args[index])
|
||||
database := prefixdb.New()
|
||||
if err := database.Load(os.Args[index]); err == nil {
|
||||
fmt.Fprintf(os.Stderr, "\r- loaded database [%s] (total[%s] version[%d.%d.%d] description[%s])\n",
|
||||
os.Args[index], size(database.Total), (database.Version>>16)&0xff, (database.Version>>8)&0xff, database.Version&0xff, database.Description)
|
||||
databases = append(databases, database)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "\r- loading database [%s] failed (%v)\n", os.Args[index], err)
|
||||
}
|
||||
} else {
|
||||
if ip := net.ParseIP(os.Args[index]); ip == nil {
|
||||
fmt.Fprintf(os.Stderr, "- lookup [%s] failed (not a valid IP address)", os.Args[index])
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "- lookup [%s] ", os.Args[index])
|
||||
lookup := map[string]interface{}{}
|
||||
for _, database := range databases {
|
||||
lookup, _ = database.Lookup(ip, lookup)
|
||||
}
|
||||
data, _ := json.Marshal(lookup)
|
||||
fmt.Printf("%s\n", data)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func server() {
|
||||
databases := []*prefixdb.PrefixDB{}
|
||||
for index := 3; index < len(os.Args); index++ {
|
||||
fmt.Fprintf(os.Stderr, "\r- loading database [%s]...", os.Args[index])
|
||||
database := prefixdb.New()
|
||||
if err := database.Load(os.Args[index]); err == nil {
|
||||
fmt.Fprintf(os.Stderr, "\r- loaded database [%s] (total[%s] version[%d.%d.%d] description[%s])\n",
|
||||
os.Args[index], size(database.Total), (database.Version>>16)&0xff, (database.Version>>8)&0xff, database.Version&0xff, database.Description)
|
||||
databases = append(databases, database)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "\r- loading database [%s] failed (%v)\n", os.Args[index], err)
|
||||
}
|
||||
}
|
||||
http.HandleFunc("/", func(response http.ResponseWriter, request *http.Request) {
|
||||
response.Header().Set("Content-Type", "application/json")
|
||||
remote, _, _ := net.SplitHostPort(request.RemoteAddr)
|
||||
if value := request.Header.Get("X-Forwarded-For"); value != "" {
|
||||
remote = strings.Split(value, ",")[0]
|
||||
}
|
||||
parameters := request.URL.Query()
|
||||
if value := parameters.Get("remote"); value != "" {
|
||||
remote = value
|
||||
}
|
||||
lookup := map[string]interface{}{}
|
||||
if ip := net.ParseIP(remote); ip != nil {
|
||||
lookup["ip"] = fmt.Sprintf("%s", ip)
|
||||
for _, database := range databases {
|
||||
lookup, _ = database.Lookup(ip, lookup)
|
||||
}
|
||||
data, _ := json.Marshal(lookup)
|
||||
response.Write(data)
|
||||
}
|
||||
})
|
||||
parts := strings.Split(os.Args[2], ",")
|
||||
server := &http.Server{Addr: strings.TrimLeft(parts[0], "*"), ReadTimeout: 10 * time.Second, WriteTimeout: 10 * time.Second}
|
||||
fmt.Fprintf(os.Stderr, "\r- listening to [%s]\n", os.Args[2])
|
||||
if len(parts) > 1 {
|
||||
server.ListenAndServeTLS(parts[1], parts[2])
|
||||
} else {
|
||||
server.ListenAndServe()
|
||||
}
|
||||
}
|
||||
|
||||
func usage(status int) {
|
||||
fmt.Fprintf(os.Stderr, "usage: prefixdb <action> [parameters...]\n\n"+
|
||||
"help show this help screen\n"+
|
||||
"json <database[@<description>]> <JSON prefixes>... build database from generic JSON-formatted prefixes lists\n"+
|
||||
"city <database[@<description>]> <CSV locations> <CSV prefixes>... build database from MaxMind GeoIP2 cities lists\n"+
|
||||
"asn <database[@<description>]> <CSV prefixes>... build database from MaxMind GeoLite2 asnums lists\n"+
|
||||
"lookup <database>... <address>... lookup entries in databases\n"+
|
||||
"server <bind address> <database>... spawn an HTTP(S) server for entries lookup\n")
|
||||
os.Exit(status)
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 2 {
|
||||
usage(1)
|
||||
}
|
||||
switch os.Args[1] {
|
||||
case "help":
|
||||
usage(0)
|
||||
case "json":
|
||||
if len(os.Args) < 3 {
|
||||
usage(1)
|
||||
}
|
||||
mkjson()
|
||||
case "city":
|
||||
if len(os.Args) < 5 {
|
||||
usage(1)
|
||||
}
|
||||
mkcity()
|
||||
case "asn":
|
||||
if len(os.Args) < 3 {
|
||||
usage(1)
|
||||
}
|
||||
mkasn()
|
||||
case "lookup":
|
||||
lookup()
|
||||
case "server":
|
||||
server()
|
||||
default:
|
||||
usage(2)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,827 @@
|
|||
package prefixdb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"net"
|
||||
"os"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const VERSION = 0x00010000
|
||||
|
||||
type fame struct {
|
||||
fame int
|
||||
value interface{}
|
||||
}
|
||||
type byfame []*fame
|
||||
|
||||
func (a byfame) Len() int { return len(a) }
|
||||
func (a byfame) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
||||
func (a byfame) Less(i, j int) bool { return a[i].fame > a[j].fame }
|
||||
|
||||
type node struct {
|
||||
down [2]*node
|
||||
up *node
|
||||
data []uint64
|
||||
offset int
|
||||
explored [4]bool
|
||||
emitted bool
|
||||
id int
|
||||
}
|
||||
type cluster struct {
|
||||
values [3]int // fame / initial index / final index
|
||||
pairs []uint64 // cluster pairs
|
||||
data []byte // reduced cluster pairs
|
||||
}
|
||||
type PrefixDB struct {
|
||||
sync.RWMutex
|
||||
tree node
|
||||
strings map[string]*[3]int // fame / initial index / final index
|
||||
numbers map[float64]*[3]int // fame / initial index / final index
|
||||
pairs map[uint64]*[3]int // fame / initial index / final index
|
||||
clusters map[[16]byte]*cluster
|
||||
data []byte
|
||||
Total int
|
||||
Version uint32
|
||||
Description string
|
||||
Strings [4]int // size / count / offset / strings index width (bytes)
|
||||
Numbers [3]int // size / count / offset
|
||||
Pairs [3]int // size / count / offset
|
||||
Clusters [4]int // size / count / offset / clusters index width (bytes)
|
||||
Maps [3]int // size / count / offset
|
||||
Nodes [4]int // size / count / offset / nodes width (bits)
|
||||
}
|
||||
|
||||
var cores int
|
||||
|
||||
func New() *PrefixDB {
|
||||
if cores == 0 {
|
||||
cores = runtime.NumCPU()
|
||||
}
|
||||
return &PrefixDB{strings: map[string]*[3]int{}, numbers: map[float64]*[3]int{}, pairs: map[uint64]*[3]int{}, clusters: map[[16]byte]*cluster{}}
|
||||
}
|
||||
|
||||
func (this *PrefixDB) Add(prefix net.IPNet, data map[string]interface{}, clusters [][]string) {
|
||||
prefix.IP = prefix.IP.To16()
|
||||
ones, bits := prefix.Mask.Size()
|
||||
if bits == 32 {
|
||||
ones += 96
|
||||
prefix.Mask = net.CIDRMask(ones, bits+96)
|
||||
}
|
||||
if cores > 1 {
|
||||
this.Lock()
|
||||
}
|
||||
pnode := &this.tree
|
||||
for bit := 0; bit < ones; bit++ {
|
||||
down := 0
|
||||
if (prefix.IP[bit/8] & (1 << (7 - (byte(bit) % 8)))) != 0 {
|
||||
down = 1
|
||||
}
|
||||
if pnode.down[down] == nil {
|
||||
pnode.down[down] = &node{}
|
||||
pnode.down[down].up = pnode
|
||||
}
|
||||
if len(pnode.data) != 0 {
|
||||
pnode.data = []uint64{}
|
||||
}
|
||||
pnode = pnode.down[down]
|
||||
}
|
||||
|
||||
skeys, ckeys, lkeys := "", [][]string{}, []string{}
|
||||
for _, cluster := range clusters {
|
||||
skeys += strings.Join(cluster, ` `) + ` `
|
||||
ckeys = append(ckeys, cluster)
|
||||
}
|
||||
for key, _ := range data {
|
||||
if strings.Index(skeys, key) < 0 {
|
||||
lkeys = append(lkeys, key)
|
||||
}
|
||||
}
|
||||
ckeys = append(ckeys, lkeys)
|
||||
for cindex, keys := range ckeys {
|
||||
cpairs := []uint64{}
|
||||
for _, key := range keys {
|
||||
if len(key) > 255 {
|
||||
continue
|
||||
}
|
||||
if value, ok := data[key]; ok {
|
||||
index := 0
|
||||
if _, ok := this.strings[key]; !ok {
|
||||
index = len(this.strings)
|
||||
this.strings[key] = &[3]int{1, index}
|
||||
} else {
|
||||
index = this.strings[key][1]
|
||||
this.strings[key][0]++
|
||||
}
|
||||
pair := uint64((uint32(index)&0x0fffffff)|0x10000000) << 32
|
||||
if tvalue, ok := value.(string); ok {
|
||||
if len(tvalue) <= 255 {
|
||||
index = 0
|
||||
if _, ok := this.strings[tvalue]; !ok {
|
||||
index = len(this.strings)
|
||||
this.strings[tvalue] = &[3]int{1, index}
|
||||
} else {
|
||||
index = this.strings[tvalue][1]
|
||||
this.strings[tvalue][0]++
|
||||
}
|
||||
pair |= uint64((uint32(index) & 0x0fffffff) | 0x10000000)
|
||||
} else {
|
||||
pair |= uint64(0x50000000)
|
||||
}
|
||||
} else if tvalue, ok := value.(float64); ok {
|
||||
index = 0
|
||||
if _, ok := this.numbers[tvalue]; !ok {
|
||||
index = len(this.numbers)
|
||||
this.numbers[tvalue] = &[3]int{1, index}
|
||||
} else {
|
||||
index = this.numbers[tvalue][1]
|
||||
this.numbers[tvalue][0]++
|
||||
}
|
||||
pair |= uint64((uint32(index) & 0x0fffffff) | 0x20000000)
|
||||
} else if tvalue, ok := value.(bool); ok {
|
||||
if tvalue {
|
||||
pair |= uint64(0x30000000)
|
||||
} else {
|
||||
pair |= uint64(0x40000000)
|
||||
}
|
||||
} else {
|
||||
pair |= uint64(0x50000000)
|
||||
}
|
||||
if _, ok := this.pairs[pair]; !ok {
|
||||
index = len(this.pairs)
|
||||
this.pairs[pair] = &[3]int{1, index}
|
||||
} else {
|
||||
this.pairs[pair][0]++
|
||||
}
|
||||
if cindex < len(ckeys)-1 {
|
||||
cpairs = append(cpairs, pair)
|
||||
} else {
|
||||
pnode.data = append(pnode.data, pair)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(cpairs) != 0 {
|
||||
buffer := make([]byte, len(cpairs)*8)
|
||||
for index, value := range cpairs {
|
||||
binary.BigEndian.PutUint64(buffer[index*8:], value)
|
||||
}
|
||||
key := md5.Sum(buffer)
|
||||
index := 0
|
||||
if _, ok := this.clusters[key]; !ok {
|
||||
index = len(this.clusters)
|
||||
this.clusters[key] = &cluster{pairs: cpairs, values: [3]int{1, index}}
|
||||
} else {
|
||||
index = this.clusters[key].values[1]
|
||||
this.clusters[key].values[0]++
|
||||
}
|
||||
pnode.data = append(pnode.data, 0x7000000000000000|((uint64(index)<<32)&0x0fffffff00000000))
|
||||
}
|
||||
}
|
||||
if cores > 1 {
|
||||
this.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func wbytes(bytes, value int, data []byte) {
|
||||
if len(data) >= bytes {
|
||||
for index := bytes - 1; index >= 0; index-- {
|
||||
data[bytes-index-1] = byte(value >> (uint(index * 8)))
|
||||
}
|
||||
}
|
||||
}
|
||||
func wpbits(prefix byte, value int) []byte {
|
||||
if value <= 7 {
|
||||
return []byte{prefix | (byte(value) & 0x07)}
|
||||
}
|
||||
bytes := int(math.Ceil(math.Ceil(math.Log2(float64(value+1))) / 8))
|
||||
data := []byte{prefix | 0x08 | byte(bytes)}
|
||||
for nibble := bytes - 1; nibble >= 0; nibble-- {
|
||||
data = append(data, byte(value>>(uint(nibble*8))))
|
||||
}
|
||||
return data
|
||||
}
|
||||
func wnbits(bits, value0, value1 int, data []byte) {
|
||||
if bits >= 8 && bits <= 32 && bits%4 == 0 && len(data) >= bits/4 {
|
||||
switch bits {
|
||||
case 8:
|
||||
data[0], data[1] = byte(value0), byte(value1)
|
||||
case 12:
|
||||
data[0], data[1], data[2] = byte(value0>>4), byte(value0<<4)|(byte(value1>>8)&0x0f), byte(value1)
|
||||
case 16:
|
||||
binary.BigEndian.PutUint16(data[0:], uint16(value0))
|
||||
binary.BigEndian.PutUint16(data[2:], uint16(value1))
|
||||
case 20:
|
||||
data[0], data[1] = byte(value0>>12), byte(value0>>4)
|
||||
data[2] = byte(value0<<4) | (byte(value1>>16) & 0x0f)
|
||||
data[3], data[4] = byte(value1>>8), byte(value1)
|
||||
case 24:
|
||||
data[0], data[1], data[2] = byte(value0>>16), byte(value0>>8), byte(value0)
|
||||
data[3], data[4], data[5] = byte(value1>>16), byte(value1>>8), byte(value1)
|
||||
case 28:
|
||||
data[0], data[1], data[2] = byte(value0>>20), byte(value0>>12), byte(value0>>4)
|
||||
data[3] = byte(value0<<4) | (byte(value1>>24) & 0x0f)
|
||||
data[4], data[5], data[6] = byte(value1>>16), byte(value1>>8), byte(value1)
|
||||
case 32:
|
||||
binary.BigEndian.PutUint32(data[0:], uint32(value0))
|
||||
binary.BigEndian.PutUint32(data[4:], uint32(value1))
|
||||
}
|
||||
}
|
||||
}
|
||||
func (this *PrefixDB) Save(path, description string) (content []byte, err error) {
|
||||
// layout header + signature placeholder + description
|
||||
if cores > 1 {
|
||||
this.Lock()
|
||||
}
|
||||
this.data = []byte{'P', 'F', 'D', 'B', 0, (VERSION >> 16) & 0xff, (VERSION >> 8) & 0xff, (VERSION & 0xff),
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 'D', 'E', 'S', 'C', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
|
||||
if description == "" {
|
||||
description = time.Now().Format(`20060102150405`)
|
||||
}
|
||||
this.Description = description
|
||||
copy(this.data[28:47], []byte(description))
|
||||
|
||||
// layout strings dictionary (ordered by fame)
|
||||
this.Strings[0] = 0
|
||||
for key, _ := range this.strings {
|
||||
this.Strings[0] += len(key)
|
||||
}
|
||||
this.Strings[3] = int(math.Ceil(math.Ceil(math.Log2(float64(this.Strings[0]+1))) / 8))
|
||||
this.data = append(this.data, []byte{'S', 'T', 'R', 'S', byte(this.Strings[3]), 0, 0, 0, 0, 0, 0, 0, 0}...)
|
||||
this.Strings[2] = len(this.data)
|
||||
this.Strings[1] = len(this.strings)
|
||||
this.Strings[0] += this.Strings[1] * this.Strings[3]
|
||||
flist := make([]*fame, this.Strings[1])
|
||||
for key, values := range this.strings {
|
||||
flist[values[1]] = &fame{values[0], key}
|
||||
}
|
||||
sort.Sort(byfame(flist))
|
||||
this.data = append(this.data, make([]byte, this.Strings[1]*this.Strings[3])...)
|
||||
offset := 0
|
||||
for index, item := range flist {
|
||||
this.strings[item.value.(string)][2] = index
|
||||
this.data = append(this.data, []byte(item.value.(string))...)
|
||||
wbytes(this.Strings[3], offset, this.data[this.Strings[2]+(index*this.Strings[3]):])
|
||||
offset += len(item.value.(string))
|
||||
}
|
||||
binary.BigEndian.PutUint32(this.data[this.Strings[2]-8:], uint32(this.Strings[0]))
|
||||
binary.BigEndian.PutUint32(this.data[this.Strings[2]-4:], uint32(this.Strings[1]))
|
||||
strings := make([]*fame, this.Strings[1])
|
||||
for key, values := range this.strings {
|
||||
strings[values[1]] = &fame{values[0], key}
|
||||
}
|
||||
|
||||
// layout numbers dictionary (ordered by fame)
|
||||
this.data = append(this.data, []byte{'N', 'U', 'M', 'S', 0, 0, 0, 0, 0, 0, 0, 0}...)
|
||||
this.Numbers[2] = len(this.data)
|
||||
this.Numbers[1] = len(this.numbers)
|
||||
this.Numbers[0] = this.Numbers[1] * 8
|
||||
flist = make([]*fame, this.Numbers[1])
|
||||
for key, values := range this.numbers {
|
||||
flist[values[1]] = &fame{values[0], key}
|
||||
}
|
||||
sort.Sort(byfame(flist))
|
||||
this.data = append(this.data, make([]byte, this.Numbers[1]*8)...)
|
||||
for index, item := range flist {
|
||||
this.numbers[item.value.(float64)][2] = index
|
||||
binary.BigEndian.PutUint64(this.data[this.Numbers[2]+(index*8):], math.Float64bits(item.value.(float64)))
|
||||
}
|
||||
binary.BigEndian.PutUint32(this.data[this.Numbers[2]-8:], uint32(this.Numbers[0]))
|
||||
binary.BigEndian.PutUint32(this.data[this.Numbers[2]-4:], uint32(this.Numbers[1]))
|
||||
numbers := make([]*fame, this.Numbers[1])
|
||||
for key, values := range this.numbers {
|
||||
numbers[values[1]] = &fame{values[0], key}
|
||||
}
|
||||
|
||||
// layout pairs dictionary (ordered by fame)
|
||||
this.data = append(this.data, []byte{'P', 'A', 'I', 'R', 0, 0, 0, 0, 0, 0, 0, 0}...)
|
||||
this.Pairs[2] = len(this.data)
|
||||
flist = make([]*fame, len(this.pairs))
|
||||
for key, values := range this.pairs {
|
||||
flist[values[1]] = &fame{values[0], key}
|
||||
}
|
||||
sort.Sort(byfame(flist))
|
||||
for index, item := range flist {
|
||||
if item.fame > 1 {
|
||||
this.pairs[item.value.(uint64)][2] = index
|
||||
} else {
|
||||
delete(this.pairs, item.value.(uint64))
|
||||
}
|
||||
}
|
||||
this.Pairs[1] = len(this.pairs)
|
||||
this.Pairs[0] = this.Pairs[1] * 8
|
||||
this.data = append(this.data, make([]byte, this.Pairs[0])...)
|
||||
for index, item := range flist {
|
||||
if item.fame <= 1 {
|
||||
break
|
||||
}
|
||||
pair := 0x1000000000000000 | (uint64(this.strings[strings[(item.value.(uint64)>>32)&0x0fffffff].value.(string)][2]) << 32)
|
||||
switch (item.value.(uint64) & 0xf0000000) >> 28 {
|
||||
case 1:
|
||||
pair |= 0x10000000 | uint64(this.strings[strings[item.value.(uint64)&0x0fffffff].value.(string)][2])
|
||||
case 2:
|
||||
pair |= 0x20000000 | uint64(this.numbers[numbers[item.value.(uint64)&0x0fffffff].value.(float64)][2])
|
||||
default:
|
||||
pair |= item.value.(uint64) & 0xf0000000
|
||||
}
|
||||
binary.BigEndian.PutUint64(this.data[this.Pairs[2]+(index*8):], pair)
|
||||
}
|
||||
binary.BigEndian.PutUint32(this.data[this.Pairs[2]-8:], uint32(this.Pairs[0]))
|
||||
binary.BigEndian.PutUint32(this.data[this.Pairs[2]-4:], uint32(this.Pairs[1]))
|
||||
|
||||
// layout clusters dictionary (ordered by fame, and reduced for strings, numbers and pairs)
|
||||
this.Clusters[0] = 0
|
||||
for _, cluster := range this.clusters {
|
||||
for _, pair := range cluster.pairs {
|
||||
if _, ok := this.pairs[pair]; ok {
|
||||
cluster.data = append(cluster.data, wpbits(0x60, this.pairs[pair][2])...)
|
||||
} else {
|
||||
cluster.data = append(cluster.data, wpbits(0x10, this.strings[strings[(pair>>32)&0x0fffffff].value.(string)][2])...)
|
||||
switch (pair & 0xf0000000) >> 28 {
|
||||
case 1:
|
||||
cluster.data = append(cluster.data, wpbits(0x10, this.strings[strings[pair&0x0fffffff].value.(string)][2])...)
|
||||
case 2:
|
||||
cluster.data = append(cluster.data, wpbits(0x20, this.numbers[numbers[pair&0x0fffffff].value.(float64)][2])...)
|
||||
default:
|
||||
cluster.data = append(cluster.data, byte((pair&0xf0000000)>>24))
|
||||
}
|
||||
}
|
||||
}
|
||||
this.Clusters[0] += len(cluster.data)
|
||||
}
|
||||
this.Clusters[3] = int(math.Ceil(math.Ceil(math.Log2(float64(this.Clusters[0]+1))) / 8))
|
||||
this.data = append(this.data, []byte{'C', 'L', 'U', 'S', byte(this.Clusters[3]), 0, 0, 0, 0, 0, 0, 0, 0}...)
|
||||
this.Clusters[2] = len(this.data)
|
||||
this.Clusters[1] = len(this.clusters)
|
||||
this.Clusters[0] += this.Clusters[1] * this.Clusters[3]
|
||||
flist = make([]*fame, this.Clusters[1])
|
||||
for key, cluster := range this.clusters {
|
||||
flist[cluster.values[1]] = &fame{cluster.values[0], key}
|
||||
}
|
||||
sort.Sort(byfame(flist))
|
||||
this.data = append(this.data, make([]byte, this.Clusters[1]*this.Clusters[3])...)
|
||||
offset = 0
|
||||
for index, item := range flist {
|
||||
this.clusters[item.value.([16]byte)].values[2] = index
|
||||
this.data = append(this.data, this.clusters[item.value.([16]byte)].data...)
|
||||
wbytes(this.Clusters[3], offset, this.data[this.Clusters[2]+(index*this.Clusters[3]):])
|
||||
offset += len(this.clusters[item.value.([16]byte)].data)
|
||||
}
|
||||
binary.BigEndian.PutUint32(this.data[this.Clusters[2]-8:], uint32(this.Clusters[0]))
|
||||
binary.BigEndian.PutUint32(this.data[this.Clusters[2]-4:], uint32(this.Clusters[1]))
|
||||
clusters := make([]*fame, this.Clusters[1])
|
||||
for key, cluster := range this.clusters {
|
||||
clusters[cluster.values[1]] = &fame{cluster.values[0], key}
|
||||
}
|
||||
|
||||
// layout maps dictionary (reduced for strings, numbers, pairs and clusters)
|
||||
this.Nodes[1] = 1
|
||||
this.data = append(this.data, []byte{'M', 'A', 'P', 'S', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x80}...)
|
||||
pnode := &this.tree
|
||||
this.Maps[2] = len(this.data) - 2
|
||||
this.Maps[1] = 1
|
||||
this.Maps[0] = 2
|
||||
for {
|
||||
if pnode.down[0] != nil && !pnode.explored[0] {
|
||||
pnode.explored[0] = true
|
||||
pnode = pnode.down[0]
|
||||
} else if pnode.down[1] != nil && !pnode.explored[1] {
|
||||
pnode.explored[1] = true
|
||||
pnode = pnode.down[1]
|
||||
} else if pnode.up != nil {
|
||||
pnode = pnode.up
|
||||
}
|
||||
if pnode.up == nil {
|
||||
break
|
||||
}
|
||||
if pnode.down[0] == nil && pnode.down[1] == nil {
|
||||
if len(pnode.data) == 0 {
|
||||
pnode.offset = 1
|
||||
} else {
|
||||
data := []byte{}
|
||||
for index := 0; index < len(pnode.data); index++ {
|
||||
last := byte(0x00)
|
||||
if index == len(pnode.data)-1 {
|
||||
last = 0x80
|
||||
}
|
||||
if ((pnode.data[index]>>32)&0xf0000000)>>28 == 7 {
|
||||
data = append(data, wpbits(last|0x70, this.clusters[clusters[(pnode.data[index]>>32)&0x0fffffff].value.([16]byte)].values[2])...)
|
||||
} else {
|
||||
if _, ok := this.pairs[pnode.data[index]]; ok {
|
||||
data = append(data, wpbits(last|0x60, this.pairs[pnode.data[index]][2])...)
|
||||
} else {
|
||||
data = append(data, wpbits(0x10, this.strings[strings[(pnode.data[index]>>32)&0x0fffffff].value.(string)][2])...)
|
||||
switch (pnode.data[index] & 0xf0000000) >> 28 {
|
||||
case 1:
|
||||
data = append(data, wpbits(last|0x10, this.strings[strings[pnode.data[index]&0x0fffffff].value.(string)][2])...)
|
||||
case 2:
|
||||
data = append(data, wpbits(last|0x20, this.numbers[numbers[pnode.data[index]&0x0fffffff].value.(float64)][2])...)
|
||||
default:
|
||||
data = append(data, last|byte((pnode.data[index]&0xf0000000)>>24))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
this.data = append(this.data, data...)
|
||||
pnode.offset = this.Maps[0]
|
||||
this.Maps[0] += len(data)
|
||||
this.Maps[1]++
|
||||
}
|
||||
} else if pnode.id == 0 {
|
||||
pnode.id = this.Nodes[1]
|
||||
this.Nodes[1]++
|
||||
}
|
||||
}
|
||||
binary.BigEndian.PutUint32(this.data[this.Maps[2]-8:], uint32(this.Maps[0]))
|
||||
binary.BigEndian.PutUint32(this.data[this.Maps[2]-4:], uint32(this.Maps[1]))
|
||||
|
||||
// layout nodes tree
|
||||
this.Nodes[3] = int(math.Ceil(math.Ceil(math.Log2(float64(this.Nodes[1]+this.Maps[0]+1)))/4) * 4)
|
||||
this.data = append(this.data, []byte{'N', 'O', 'D', 'E', byte(this.Nodes[3]), 0, 0, 0, 0, 0, 0, 0, 0}...)
|
||||
this.Nodes[2] = len(this.data)
|
||||
this.Nodes[0] = this.Nodes[1] * ((2 * this.Nodes[3]) / 8)
|
||||
this.data = append(this.data, make([]byte, this.Nodes[0])...)
|
||||
pnode = &this.tree
|
||||
next := [2]int{}
|
||||
for {
|
||||
if (pnode == &this.tree || pnode.id != 0) && !pnode.emitted {
|
||||
pnode.emitted = true
|
||||
for index := 0; index <= 1; index++ {
|
||||
next[index] = this.Nodes[1]
|
||||
if pnode.down[index] != nil {
|
||||
if pnode.down[index].id != 0 {
|
||||
next[index] = pnode.down[index].id
|
||||
} else {
|
||||
next[index] += pnode.down[index].offset
|
||||
}
|
||||
}
|
||||
}
|
||||
wnbits(this.Nodes[3], next[0], next[1], this.data[this.Nodes[2]+(pnode.id*((2*this.Nodes[3])/8)):])
|
||||
}
|
||||
if pnode.down[0] != nil && !pnode.explored[2] {
|
||||
pnode.explored[2] = true
|
||||
pnode = pnode.down[0]
|
||||
} else if pnode.down[1] != nil && !pnode.explored[3] {
|
||||
pnode.explored[3] = true
|
||||
pnode = pnode.down[1]
|
||||
} else if pnode.up != nil {
|
||||
pnode = pnode.up
|
||||
}
|
||||
if pnode.up == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
binary.BigEndian.PutUint32(this.data[this.Nodes[2]-8:], uint32(this.Nodes[0]))
|
||||
binary.BigEndian.PutUint32(this.data[this.Nodes[2]-4:], uint32(this.Nodes[1]))
|
||||
|
||||
// finalize header
|
||||
this.tree, this.strings, this.numbers, this.pairs, this.clusters = node{}, map[string]*[3]int{}, map[float64]*[3]int{}, map[uint64]*[3]int{}, map[[16]byte]*cluster{}
|
||||
hash := md5.Sum(this.data[24:])
|
||||
copy(this.data[8:], hash[:])
|
||||
this.Total = len(this.data)
|
||||
|
||||
// save database
|
||||
if path != "" {
|
||||
if path == "-" {
|
||||
_, err = os.Stdout.Write(this.data)
|
||||
} else {
|
||||
err = ioutil.WriteFile(path, this.data, 0644)
|
||||
}
|
||||
}
|
||||
|
||||
if cores > 1 {
|
||||
this.Unlock()
|
||||
}
|
||||
return this.data, err
|
||||
}
|
||||
|
||||
func (this *PrefixDB) Load(path string) error {
|
||||
if data, err := ioutil.ReadFile(path); err != nil {
|
||||
return err
|
||||
} else {
|
||||
if len(data) < 8 || string(data[0:4]) != "PFDB" {
|
||||
return errors.New(`invalid preamble`)
|
||||
}
|
||||
if version := (uint32(data[5]) << 16) + (uint32(data[6]) << 8) + uint32(data[7]); (version & 0xff0000) > (VERSION & 0xff0000) {
|
||||
return errors.New(fmt.Sprintf(`library major version %d is incompatible with database major version %d`, (VERSION&0xff0000)>>16, (version&0xff0000)>>16))
|
||||
} else {
|
||||
if len(data) < 24 || fmt.Sprintf("%x", md5.Sum(data[24:])) != fmt.Sprintf("%x", data[8:24]) {
|
||||
return errors.New(`database checksum is invalid`)
|
||||
}
|
||||
if cores > 1 {
|
||||
this.Lock()
|
||||
}
|
||||
this.data = data
|
||||
this.Total = len(data)
|
||||
this.Version = version
|
||||
offset := 24
|
||||
if this.Total >= offset+4 && string(data[offset:offset+4]) == "DESC" {
|
||||
offset += 4
|
||||
if this.Total >= offset+20 {
|
||||
index := 0
|
||||
if index = bytes.Index(data[offset:offset+20], []byte{0}); index < 0 {
|
||||
index = 20
|
||||
}
|
||||
this.Description = fmt.Sprintf("%s", data[offset:offset+index])
|
||||
offset += 20
|
||||
if this.Total >= offset+4 && string(data[offset:offset+4]) == "STRS" {
|
||||
offset += 4
|
||||
if this.Total >= offset+9 {
|
||||
this.Strings[3] = int(data[offset])
|
||||
this.Strings[2] = offset + 9
|
||||
this.Strings[1] = int(binary.BigEndian.Uint32(this.data[offset+5:]))
|
||||
this.Strings[0] = int(binary.BigEndian.Uint32(this.data[offset+1:]))
|
||||
offset += 9 + this.Strings[0]
|
||||
if this.Total >= offset+4 && string(data[offset:offset+4]) == "NUMS" {
|
||||
offset += 4
|
||||
if this.Total >= offset+8 {
|
||||
this.Numbers[2] = offset + 8
|
||||
this.Numbers[1] = int(binary.BigEndian.Uint32(this.data[offset+4:]))
|
||||
this.Numbers[0] = int(binary.BigEndian.Uint32(this.data[offset:]))
|
||||
offset += 8 + this.Numbers[0]
|
||||
if this.Total >= offset+4 && string(data[offset:offset+4]) == "PAIR" {
|
||||
offset += 4
|
||||
if this.Total >= offset+8 {
|
||||
this.Pairs[2] = offset + 8
|
||||
this.Pairs[1] = int(binary.BigEndian.Uint32(this.data[offset+4:]))
|
||||
this.Pairs[0] = int(binary.BigEndian.Uint32(this.data[offset:]))
|
||||
offset += 8 + this.Pairs[0]
|
||||
if this.Total >= offset+4 && string(data[offset:offset+4]) == "CLUS" {
|
||||
offset += 4
|
||||
this.Clusters[3] = int(data[offset])
|
||||
this.Clusters[2] = offset + 9
|
||||
this.Clusters[1] = int(binary.BigEndian.Uint32(this.data[offset+5:]))
|
||||
this.Clusters[0] = int(binary.BigEndian.Uint32(this.data[offset+1:]))
|
||||
offset += 9 + this.Clusters[0]
|
||||
if this.Total >= offset+4 && string(data[offset:offset+4]) == "MAPS" {
|
||||
offset += 4
|
||||
if this.Total >= offset+8 {
|
||||
this.Maps[2] = offset + 8
|
||||
this.Maps[1] = int(binary.BigEndian.Uint32(this.data[offset+4:]))
|
||||
this.Maps[0] = int(binary.BigEndian.Uint32(this.data[offset:]))
|
||||
offset += 8 + this.Maps[0]
|
||||
if this.Total >= offset+9 && string(data[offset:offset+4]) == "NODE" {
|
||||
offset += 4
|
||||
this.Nodes[3] = int(data[offset])
|
||||
this.Nodes[2] = offset + 9
|
||||
this.Nodes[1] = int(binary.BigEndian.Uint32(this.data[offset+5:]))
|
||||
this.Nodes[0] = int(binary.BigEndian.Uint32(this.data[offset+1:]))
|
||||
if offset+9+this.Nodes[0] != this.Total {
|
||||
this.Nodes[2] = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if cores > 1 {
|
||||
this.Unlock()
|
||||
}
|
||||
if this.Strings[2] == 0 || this.Numbers[2] == 0 || this.Pairs[2] == 0 || this.Clusters[2] == 0 || this.Maps[2] == 0 || this.Nodes[2] == 0 {
|
||||
return errors.New(`database structure is invalid`)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func rpbits(data []byte) (section, index, size int, last bool) {
|
||||
section = int((data[0] & 0x70) >> 4)
|
||||
if data[0]&0x80 != 0 || section == 0 {
|
||||
last = true
|
||||
}
|
||||
if section == 1 || section == 2 || section == 6 || section == 7 {
|
||||
if data[0]&0x08 != 0 {
|
||||
size = int(data[0] & 0x07)
|
||||
for nibble := 1; nibble <= size; nibble++ {
|
||||
index |= int(data[nibble]) << (uint(size-nibble) * 8)
|
||||
}
|
||||
} else {
|
||||
index = int(data[0] & 0x07)
|
||||
}
|
||||
}
|
||||
size++
|
||||
return section, index, size, last
|
||||
}
|
||||
func rnbits(bits, index, down int, data []byte) int {
|
||||
if bits >= 8 && bits <= 32 && bits%4 == 0 && (down == 0 || down == 1) && len(data) >= (index+1)*(bits/4) {
|
||||
offset := index * (bits / 4)
|
||||
switch bits {
|
||||
case 8:
|
||||
return int(data[offset+down])
|
||||
case 12:
|
||||
if down == 0 {
|
||||
return (int(data[offset]) << 4) | ((int(data[offset+1]) >> 4) & 0x0f)
|
||||
} else {
|
||||
return ((int(data[offset+1]) & 0x0f) << 8) | int(data[offset+2])
|
||||
}
|
||||
case 16:
|
||||
return int(binary.BigEndian.Uint16(data[offset+(down*2):]))
|
||||
case 20:
|
||||
if down == 0 {
|
||||
return (int(data[offset]) << 12) | (int(data[offset+1]) << 4) | ((int(data[offset+2]) >> 4) & 0x0f)
|
||||
} else {
|
||||
return ((int(data[offset+2]) & 0x0f) << 16) | (int(data[offset+3]) << 8) | int(data[offset+4])
|
||||
}
|
||||
case 24:
|
||||
if down == 0 {
|
||||
return (int(data[offset]) << 16) | (int(data[offset+1]) << 8) | int(data[offset+2])
|
||||
} else {
|
||||
return (int(data[offset+3]) << 16) | (int(data[offset+4]) << 8) | int(data[offset+5])
|
||||
}
|
||||
case 28:
|
||||
if down == 0 {
|
||||
return (int(data[offset]) << 20) | (int(data[offset+1]) << 12) | (int(data[offset+2]) << 4) | ((int(data[offset+3]) >> 4) & 0x0f)
|
||||
} else {
|
||||
return ((int(data[offset+3]) & 0x0f) << 24) | (int(data[offset+4]) << 16) | (int(data[offset+5]) << 8) | int(data[offset+6])
|
||||
}
|
||||
case 32:
|
||||
return int(binary.BigEndian.Uint32(data[offset+(down*4):]))
|
||||
}
|
||||
}
|
||||
return index
|
||||
}
|
||||
func rbytes(width int, data []byte) (value int) {
|
||||
for index := 0; index < width; index++ {
|
||||
value |= int(data[index]) << (uint(width-1-index) * 8)
|
||||
}
|
||||
return value
|
||||
}
|
||||
func (this *PrefixDB) rstring(index int) string {
|
||||
count, offset, width := this.Strings[1], this.Strings[2], this.Strings[3]
|
||||
if index >= count {
|
||||
return ""
|
||||
}
|
||||
start, end := rbytes(width, this.data[offset+(index*width):]), 0
|
||||
if index < count-1 {
|
||||
end = rbytes(width, this.data[offset+(index+1)*width:])
|
||||
} else {
|
||||
end = this.Strings[0] - (count * width)
|
||||
}
|
||||
return string(this.data[offset+(count*width)+start : offset+(count*width)+end])
|
||||
}
|
||||
func (this *PrefixDB) rnumber(index int) float64 {
|
||||
if index >= this.Numbers[1] {
|
||||
return 0.0
|
||||
}
|
||||
return math.Float64frombits(binary.BigEndian.Uint64(this.data[this.Numbers[2]+(index*8):]))
|
||||
}
|
||||
func (this *PrefixDB) rpair(index int, pairs map[string]interface{}) {
|
||||
if index < this.Pairs[1] {
|
||||
pair := binary.BigEndian.Uint64(this.data[this.Pairs[2]+(index*8):])
|
||||
if key := this.rstring(int((pair >> 32) & 0x0fffffff)); key != "" {
|
||||
switch (pair & 0xf0000000) >> 28 {
|
||||
case 1:
|
||||
pairs[key] = this.rstring(int(pair & 0x0fffffff))
|
||||
case 2:
|
||||
pairs[key] = this.rnumber(int(pair & 0x0fffffff))
|
||||
case 3:
|
||||
pairs[key] = true
|
||||
case 4:
|
||||
pairs[key] = false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
func (this *PrefixDB) rcluster(index int, pairs map[string]interface{}) {
|
||||
count, offset, width := this.Clusters[1], this.Clusters[2], this.Clusters[3]
|
||||
if index < count {
|
||||
start, end := rbytes(width, this.data[offset+(index*width):]), 0
|
||||
if index < count-1 {
|
||||
end = rbytes(width, this.data[offset+(index+1)*width:])
|
||||
} else {
|
||||
end = this.Clusters[0] - (count * width)
|
||||
}
|
||||
start += offset + (count * width)
|
||||
end += offset + (count * width)
|
||||
key := ""
|
||||
for start < end {
|
||||
section, index, size, _ := rpbits(this.data[start:])
|
||||
switch section {
|
||||
case 1:
|
||||
if key != "" {
|
||||
pairs[key] = this.rstring(index)
|
||||
key = ""
|
||||
} else {
|
||||
key = this.rstring(index)
|
||||
}
|
||||
case 2:
|
||||
if key != "" {
|
||||
pairs[key] = this.rnumber(index)
|
||||
key = ""
|
||||
}
|
||||
case 3:
|
||||
if key != "" {
|
||||
pairs[key] = true
|
||||
key = ""
|
||||
}
|
||||
case 4:
|
||||
if key != "" {
|
||||
pairs[key] = false
|
||||
key = ""
|
||||
}
|
||||
case 5:
|
||||
if key != "" {
|
||||
pairs[key] = nil
|
||||
key = ""
|
||||
}
|
||||
case 6:
|
||||
this.rpair(index, pairs)
|
||||
}
|
||||
start += size
|
||||
}
|
||||
}
|
||||
}
|
||||
func (this *PrefixDB) Lookup(address net.IP, input map[string]interface{}) (output map[string]interface{}, err error) {
|
||||
output = input
|
||||
if this.data == nil || this.Total == 0 || this.Version == 0 || this.Strings[2] == 0 || this.Numbers[2] == 0 ||
|
||||
this.Pairs[2] == 0 || this.Clusters[2] == 0 || this.Maps[2] == 0 || this.Nodes[2] == 0 || address == nil {
|
||||
err = errors.New("record not found")
|
||||
} else {
|
||||
address = address.To16()
|
||||
offset := 0
|
||||
if cores > 1 {
|
||||
this.RLock()
|
||||
}
|
||||
for bit := 0; bit < 128; bit++ {
|
||||
down := 0
|
||||
if (address[bit/8] & (1 << (7 - (byte(bit) % 8)))) != 0 {
|
||||
down = 1
|
||||
}
|
||||
offset = rnbits(this.Nodes[3], offset, down, this.data[this.Nodes[2]:])
|
||||
if offset == this.Nodes[1] || offset == 0 {
|
||||
break
|
||||
}
|
||||
if output == nil {
|
||||
output = map[string]interface{}{}
|
||||
}
|
||||
if offset > this.Nodes[1] {
|
||||
offset -= this.Nodes[1]
|
||||
if offset < this.Maps[0] {
|
||||
offset += this.Maps[2]
|
||||
key := ""
|
||||
for offset < this.Maps[2]+this.Maps[0] {
|
||||
section, index, size, last := rpbits(this.data[offset:])
|
||||
switch section {
|
||||
case 1:
|
||||
if key != "" {
|
||||
output[key] = this.rstring(index)
|
||||
key = ""
|
||||
} else {
|
||||
key = this.rstring(index)
|
||||
}
|
||||
case 2:
|
||||
if key != "" {
|
||||
output[key] = this.rnumber(index)
|
||||
key = ""
|
||||
}
|
||||
case 3:
|
||||
if key != "" {
|
||||
output[key] = true
|
||||
key = ""
|
||||
}
|
||||
case 4:
|
||||
if key != "" {
|
||||
output[key] = false
|
||||
key = ""
|
||||
}
|
||||
case 5:
|
||||
if key != "" {
|
||||
output[key] = nil
|
||||
key = ""
|
||||
}
|
||||
case 6:
|
||||
this.rpair(index, output)
|
||||
case 7:
|
||||
this.rcluster(index, output)
|
||||
}
|
||||
if last {
|
||||
break
|
||||
}
|
||||
offset += size
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
if cores > 1 {
|
||||
this.RUnlock()
|
||||
}
|
||||
}
|
||||
return output, err
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
package rcache
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"sync"
|
||||
)
|
||||
|
||||
var (
|
||||
cores int
|
||||
cache map[[16]byte]*regexp.Regexp = map[[16]byte]*regexp.Regexp{}
|
||||
lock sync.RWMutex
|
||||
)
|
||||
|
||||
func Get(expression string) *regexp.Regexp {
|
||||
if cores == 0 {
|
||||
cores = runtime.NumCPU()
|
||||
}
|
||||
key := md5.Sum([]byte(expression))
|
||||
if cores > 1 {
|
||||
lock.RLock()
|
||||
}
|
||||
if cache[key] != nil {
|
||||
if cores > 1 {
|
||||
defer lock.RUnlock()
|
||||
}
|
||||
return cache[key].Copy()
|
||||
}
|
||||
if cores > 1 {
|
||||
lock.RUnlock()
|
||||
}
|
||||
if regex, err := regexp.Compile(expression); err == nil {
|
||||
if cores > 1 {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
}
|
||||
cache[key] = regex
|
||||
return cache[key].Copy()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/pyke369/golang-support/rpack"
|
||||
)
|
||||
|
||||
func main() {
|
||||
var (
|
||||
options flag.FlagSet
|
||||
usemain bool
|
||||
)
|
||||
|
||||
options = flag.FlagSet{Usage: func() {
|
||||
fmt.Fprintf(os.Stderr, "usage: %s [options] rootdir\n\noptions are:\n\n", filepath.Base(os.Args[0]))
|
||||
options.PrintDefaults()
|
||||
},
|
||||
}
|
||||
options.String("output", "static.go", "the generated output file path")
|
||||
options.String("pkgname", "main", "the package name to use in the generated output")
|
||||
options.String("funcname", "resources", "the function name to use in the generated output")
|
||||
options.Bool("main", false, "whether to generate a main func or not")
|
||||
if err := options.Parse(os.Args[1:]); err != nil {
|
||||
os.Exit(1)
|
||||
}
|
||||
if options.NArg() == 0 {
|
||||
options.Usage()
|
||||
os.Exit(1)
|
||||
}
|
||||
if options.Lookup("main").Value.String() == "true" {
|
||||
usemain = true
|
||||
}
|
||||
rpack.Pack(options.Arg(0), options.Lookup("output").Value.String(), options.Lookup("pkgname").Value.String(), options.Lookup("funcname").Value.String(), usemain)
|
||||
}
|
||||
|
|
@ -0,0 +1,155 @@
|
|||
package rpack
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"mime"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type RPACK struct {
|
||||
Compressed bool
|
||||
Modified int64
|
||||
Mime string
|
||||
Content string
|
||||
raw []byte
|
||||
}
|
||||
|
||||
var guzpool = sync.Pool{
|
||||
New: func() interface{} {
|
||||
return &gzip.Reader{}
|
||||
}}
|
||||
|
||||
func Pack(root, output, pkgname, funcname string, main bool) {
|
||||
root = strings.TrimSuffix(root, "/")
|
||||
if root == "" || output == "" {
|
||||
return
|
||||
}
|
||||
if pkgname == "" || main {
|
||||
pkgname = "main"
|
||||
}
|
||||
if funcname == "" {
|
||||
funcname = "resources"
|
||||
}
|
||||
funcname = strings.ToUpper(funcname[:1]) + funcname[1:]
|
||||
entries := map[string]*RPACK{}
|
||||
compressor, _ := gzip.NewWriterLevel(nil, gzip.BestCompression)
|
||||
count := 0
|
||||
size := int64(0)
|
||||
start := time.Now()
|
||||
filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
rpath := strings.TrimPrefix(path, root+"/")
|
||||
if info.Mode()&os.ModeType == 0 {
|
||||
for _, part := range strings.Split(rpath, "/") {
|
||||
if len(part) > 0 && part[0] == '.' {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
pack := &RPACK{Modified: info.ModTime().Unix(), Mime: "text/plain"}
|
||||
if mime := mime.TypeByExtension(filepath.Ext(rpath)); mime != "" {
|
||||
pack.Mime = mime
|
||||
}
|
||||
content, _ := ioutil.ReadFile(path)
|
||||
compressed := bytes.Buffer{}
|
||||
compressor.Reset(&compressed)
|
||||
compressor.Write(content)
|
||||
compressor.Close()
|
||||
if compressed.Len() < len(content) {
|
||||
pack.Content = base64.StdEncoding.EncodeToString(compressed.Bytes())
|
||||
pack.Compressed = true
|
||||
} else {
|
||||
pack.Content = base64.StdEncoding.EncodeToString(content)
|
||||
}
|
||||
entries[rpath] = pack
|
||||
fmt.Fprintf(os.Stderr, "\r%-120.120s ", rpath)
|
||||
count++
|
||||
size += info.Size()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
fmt.Fprintf(os.Stderr, "\r%-120.120s\rpacked %d file(s) %d byte(s) in %v\n", "", count, size, time.Now().Sub(start))
|
||||
if handle, err := os.OpenFile(output, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644); err == nil {
|
||||
random := make([]byte, 64)
|
||||
rand.Seed(time.Now().UnixNano() + int64(os.Getpid()))
|
||||
rand.Read(random)
|
||||
uid := fmt.Sprintf("rpack%8.8x", md5.Sum(random))
|
||||
fmt.Fprintf(handle,
|
||||
`package %s
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"github.com/pyke369/golang-support/rpack"
|
||||
)
|
||||
|
||||
var %s map[string]*rpack.RPACK = map[string]*rpack.RPACK {
|
||||
`,
|
||||
pkgname, uid)
|
||||
for path, entry := range entries {
|
||||
fmt.Fprintf(handle,
|
||||
` "%s": &rpack.RPACK{Compressed:%t, Modified:%d, Mime:"%s", Content:"%s"},
|
||||
`, path, entry.Compressed, entry.Modified, entry.Mime, entry.Content)
|
||||
}
|
||||
fmt.Fprintf(handle,
|
||||
`}
|
||||
|
||||
func %s() http.Handler {
|
||||
return rpack.Serve(%s)
|
||||
}
|
||||
`, funcname, uid)
|
||||
if main {
|
||||
fmt.Fprintf(handle,
|
||||
`
|
||||
func main() {
|
||||
http.Handle("/resources/", http.StripPrefix("/resources/", %s()))
|
||||
http.ListenAndServe(":8000", nil)
|
||||
}
|
||||
`, funcname)
|
||||
}
|
||||
handle.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func Serve(pack map[string]*RPACK) http.Handler {
|
||||
return http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
var err error
|
||||
if pack == nil || pack[request.URL.Path] == nil {
|
||||
response.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if pack[request.URL.Path].raw == nil {
|
||||
if pack[request.URL.Path].raw, err = base64.StdEncoding.DecodeString(pack[request.URL.Path].Content); err != nil {
|
||||
response.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
}
|
||||
resource := pack[request.URL.Path]
|
||||
response.Header().Set("Content-Type", resource.Mime)
|
||||
if strings.Index(request.Header.Get("Accept-Encoding"), "gzip") >= 0 && request.Header.Get("Range") == "" && resource.Compressed {
|
||||
response.Header().Set("Content-Encoding", "gzip")
|
||||
response.Header().Set("Content-Length", fmt.Sprintf("%d", len(resource.raw)))
|
||||
http.ServeContent(response, request, request.URL.Path, time.Unix(resource.Modified, 0), bytes.NewReader(resource.raw))
|
||||
} else {
|
||||
if resource.Compressed {
|
||||
decompressor := guzpool.Get().(*gzip.Reader)
|
||||
decompressor.Reset(bytes.NewReader(resource.raw))
|
||||
if raw, err := ioutil.ReadAll(decompressor); err == nil {
|
||||
http.ServeContent(response, request, request.URL.Path, time.Unix(resource.Modified, 0), bytes.NewReader(raw))
|
||||
}
|
||||
decompressor.Close()
|
||||
guzpool.Put(decompressor)
|
||||
} else {
|
||||
http.ServeContent(response, request, request.URL.Path, time.Unix(resource.Modified, 0), bytes.NewReader(resource.raw))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
@ -0,0 +1,612 @@
|
|||
package uconfig
|
||||
|
||||
import (
|
||||
"crypto/sha1"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type UConfig struct {
|
||||
config interface{}
|
||||
hash string
|
||||
cache map[string]interface{}
|
||||
cacheLock sync.RWMutex
|
||||
sync.RWMutex
|
||||
}
|
||||
|
||||
type replacer struct {
|
||||
search *regexp.Regexp
|
||||
replace string
|
||||
loop bool
|
||||
}
|
||||
|
||||
var (
|
||||
cores int
|
||||
escaped string
|
||||
unescaper *regexp.Regexp
|
||||
requoter *regexp.Regexp
|
||||
expander *regexp.Regexp
|
||||
sizer *regexp.Regexp
|
||||
duration1 *regexp.Regexp
|
||||
duration2 *regexp.Regexp
|
||||
replacers [16]replacer
|
||||
)
|
||||
|
||||
func init() {
|
||||
escaped = "{}[],#/*;:= " // match characters within quotes to escape
|
||||
unescaper = regexp.MustCompile("@\\d+@") // match escaped characters (to reverse previous escaping)
|
||||
expander = regexp.MustCompile("{{([<|@&!\\-\\+])\\s*([^{}]*?)\\s*}}") // match external content macros
|
||||
sizer = regexp.MustCompile("^(\\d+(?:\\.\\d*)?)\\s*([KMGTP]?)(B?)$") // match size value
|
||||
duration1 = regexp.MustCompile("(\\d+)(Y|MO|D|H|MN|S|MS|US)?") // match duration value form1 (free)
|
||||
duration2 = regexp.MustCompile("^(?:(\\d+):)?(\\d{2}):(\\d{2})(?:\\.(\\d{1,3}))?$") // match duration value form2 (timecode)
|
||||
replacers[0] = replacer{regexp.MustCompile("(?m)^(.*?)(?:#|//).*?$"), "$1", false} // remove # and // commented portions
|
||||
replacers[1] = replacer{regexp.MustCompile("/\\*[^\\*]*\\*/"), "", true} // remove /* */ commented portions
|
||||
replacers[2] = replacer{regexp.MustCompile("(?m)^\\s+"), "", false} // trim leading spaces
|
||||
replacers[3] = replacer{regexp.MustCompile("(?m)\\s+$"), "", false} // trim trailing spaces
|
||||
replacers[4] = replacer{regexp.MustCompile("(?s)(^|[\r\n]+)\\[([^\\]\r\n]+?)\\](.+?)((?:[\r\n]+\\[)|$)"), "$1$2\n{$3\n}$4", true} // convert INI sections into JSON objects
|
||||
replacers[5] = replacer{regexp.MustCompile("(?m)^(\\S+)\\s+([^{}\\[\\],;:=]+);$"), "$1 = $2;", false} // add missing key-value separators
|
||||
replacers[6] = replacer{regexp.MustCompile("(?m);$"), ",", false} // replace ; line terminators by ,
|
||||
replacers[7] = replacer{regexp.MustCompile("(\\S+?)\\s*[:=]"), "$1:", false} // replace = key-value separators by :
|
||||
replacers[8] = replacer{regexp.MustCompile("([}\\]])(\\s*)([^,}\\]\\s])"), "$1,$2$3", false} // add missing objects/arrays , separators
|
||||
replacers[9] = replacer{regexp.MustCompile("(?m)(^[^:]+:.+?[^,])$"), "$1,", false} // add missing values trailing , seperators
|
||||
replacers[10] = replacer{regexp.MustCompile("(^|[,{\\[]+\\s*)([^:{\\[]+?)(\\s*[{\\[])"), "$1$2:$3", true} // add missing key-(object/array-)value separator
|
||||
replacers[11] = replacer{regexp.MustCompile("(?m)^([^\":{}\\[\\]]+)"), "\"$1\"", false} // add missing quotes around keys
|
||||
replacers[12] = replacer{regexp.MustCompile("([:,\\[\\s]+)([^\",\\[\\]{}\n\r]+?)(\\s*[,\\]}])"), "$1\"$2\"$3", false} // add missing quotes around values
|
||||
replacers[13] = replacer{regexp.MustCompile("\"[\r\n]"), "\",\n", false} // add still issing objects/arrays , separators
|
||||
replacers[14] = replacer{regexp.MustCompile("\"\\s*(.+?)\\s*\""), "\"$1\"", false} // trim leading and trailing spaces in quoted strings
|
||||
replacers[15] = replacer{regexp.MustCompile(",+(\\s*[}\\]])"), "$1", false} // remove objets/arrays last element extra ,
|
||||
}
|
||||
|
||||
func escape(input string) string {
|
||||
var output []byte
|
||||
|
||||
instring := false
|
||||
for index := 0; index < len(input); index++ {
|
||||
if input[index:index+1] == "\"" && (index == 0 || input[index-1:index] != "\\") {
|
||||
instring = !instring
|
||||
}
|
||||
if instring == true {
|
||||
offset := strings.IndexAny(escaped, input[index:index+1])
|
||||
if offset >= 0 {
|
||||
output = append(output, []byte(fmt.Sprintf("@%02d@", offset))...)
|
||||
} else {
|
||||
output = append(output, input[index:index+1]...)
|
||||
}
|
||||
} else {
|
||||
output = append(output, input[index:index+1]...)
|
||||
}
|
||||
}
|
||||
return string(output)
|
||||
}
|
||||
|
||||
func unescape(input string) string {
|
||||
return unescaper.ReplaceAllStringFunc(input, func(a string) string {
|
||||
offset, _ := strconv.Atoi(a[1:3])
|
||||
if offset < len(escaped) {
|
||||
return escaped[offset : offset+1]
|
||||
}
|
||||
return ""
|
||||
})
|
||||
}
|
||||
|
||||
func reduce(input interface{}) {
|
||||
if input != nil {
|
||||
switch reflect.TypeOf(input).Kind() {
|
||||
case reflect.Map:
|
||||
for key, _ := range input.(map[string]interface{}) {
|
||||
var parts []string
|
||||
for _, value := range strings.Split(key, " ") {
|
||||
if value != "" {
|
||||
parts = append(parts, value)
|
||||
}
|
||||
}
|
||||
if len(parts) > 1 {
|
||||
if input.(map[string]interface{})[parts[0]] == nil || reflect.TypeOf(input.(map[string]interface{})[parts[0]]).Kind() != reflect.Map {
|
||||
input.(map[string]interface{})[parts[0]] = make(map[string]interface{})
|
||||
}
|
||||
input.(map[string]interface{})[parts[0]].(map[string]interface{})[parts[1]] = input.(map[string]interface{})[key]
|
||||
delete(input.(map[string]interface{}), key)
|
||||
}
|
||||
}
|
||||
for _, value := range input.(map[string]interface{}) {
|
||||
reduce(value)
|
||||
}
|
||||
case reflect.Slice:
|
||||
for index := 0; index < len(input.([]interface{})); index++ {
|
||||
reduce(input.([]interface{})[index])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func New(input string, inline ...bool) (*UConfig, error) {
|
||||
if cores == 0 {
|
||||
cores = runtime.NumCPU()
|
||||
}
|
||||
config := &UConfig{
|
||||
config: nil,
|
||||
}
|
||||
return config, config.Load(input, inline...)
|
||||
}
|
||||
|
||||
func (this *UConfig) Load(input string, inline ...bool) error {
|
||||
if cores > 1 {
|
||||
this.Lock()
|
||||
defer this.Unlock()
|
||||
}
|
||||
|
||||
this.cache = map[string]interface{}{}
|
||||
base, _ := os.Getwd()
|
||||
content := fmt.Sprintf("/*base:%s*/\n", base)
|
||||
if len(inline) > 0 && inline[0] {
|
||||
content += input
|
||||
} else {
|
||||
content += fmt.Sprintf("{{<%s}}", input)
|
||||
}
|
||||
|
||||
for {
|
||||
indexes := expander.FindStringSubmatchIndex(content)
|
||||
if indexes == nil {
|
||||
content = escape(content)
|
||||
for index := 0; index < len(replacers); index++ {
|
||||
mcontent := ""
|
||||
for mcontent != content {
|
||||
mcontent = content
|
||||
content = replacers[index].search.ReplaceAllString(content, replacers[index].replace)
|
||||
if !replacers[index].loop {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
content = unescape(strings.Trim(content, " \n,"))
|
||||
break
|
||||
}
|
||||
|
||||
expanded := ""
|
||||
arguments := strings.Split(content[indexes[4]:indexes[5]], " ")
|
||||
if start := strings.LastIndex(content[0:indexes[2]], "/*base:"); start >= 0 {
|
||||
if end := strings.Index(content[start+7:indexes[2]], "*/"); end > 0 {
|
||||
base = content[start+7 : start+7+end]
|
||||
}
|
||||
}
|
||||
switch content[indexes[2]:indexes[3]] {
|
||||
case "<":
|
||||
if arguments[0][0:1] != "/" {
|
||||
arguments[0] = fmt.Sprintf("%s/%s", base, arguments[0])
|
||||
}
|
||||
base = ""
|
||||
if elements, err := filepath.Glob(arguments[0]); err == nil {
|
||||
for _, element := range elements {
|
||||
if mcontent, err := ioutil.ReadFile(element); err == nil {
|
||||
base = filepath.Dir(element)
|
||||
expanded += string(mcontent)
|
||||
}
|
||||
}
|
||||
}
|
||||
if base != "" && strings.Index(expanded, "\n") >= 0 {
|
||||
expanded = fmt.Sprintf("/*base:%s*/\n%s\n", base, expanded)
|
||||
}
|
||||
case "|":
|
||||
if arguments[0][0:1] != "/" {
|
||||
arguments[0] = fmt.Sprintf("%s/%s", base, arguments[0])
|
||||
}
|
||||
base = ""
|
||||
if elements, err := filepath.Glob(arguments[0]); err == nil {
|
||||
for _, element := range elements {
|
||||
if element[0:1] != "/" {
|
||||
element = fmt.Sprintf("%s/%s", base, element)
|
||||
}
|
||||
if mcontent, err := exec.Command(element, strings.Join(arguments[1:], " ")).Output(); err == nil {
|
||||
base = filepath.Dir(element)
|
||||
expanded += string(mcontent)
|
||||
}
|
||||
}
|
||||
}
|
||||
if base != "" && strings.Index(expanded, "\n") >= 0 {
|
||||
expanded = fmt.Sprintf("/*base:%s*/\n%s\n", base, expanded)
|
||||
}
|
||||
case "@":
|
||||
requester := http.Client{
|
||||
Timeout: time.Duration(5 * time.Second),
|
||||
}
|
||||
if response, err := requester.Get(arguments[0]); err == nil {
|
||||
if (response.StatusCode / 100) == 2 {
|
||||
if mcontent, err := ioutil.ReadAll(response.Body); err == nil {
|
||||
expanded += string(mcontent)
|
||||
}
|
||||
}
|
||||
}
|
||||
case "&":
|
||||
expanded += os.Getenv(arguments[0])
|
||||
case "!":
|
||||
arguments[0] = strings.ToLower(arguments[0])
|
||||
for index := 1; index < len(os.Args); index++ {
|
||||
option := strings.ToLower(os.Args[index])
|
||||
if option == "--"+arguments[0] {
|
||||
if index == len(os.Args)-1 || strings.HasPrefix(os.Args[index+1], "--") {
|
||||
expanded = "true"
|
||||
} else {
|
||||
expanded = os.Args[index+1]
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
case "-":
|
||||
if index := strings.Index(os.Args[0], "-"); index >= 0 {
|
||||
expanded = strings.ToLower(os.Args[0][index+1:])
|
||||
}
|
||||
case "+":
|
||||
if arguments[0][0:1] != "/" {
|
||||
arguments[0] = fmt.Sprintf("%s/%s", base, arguments[0])
|
||||
}
|
||||
if elements, err := filepath.Glob(arguments[0]); err == nil {
|
||||
for _, element := range elements {
|
||||
element = path.Base(element)
|
||||
expanded += fmt.Sprintf("%s ", strings.TrimSuffix(element, filepath.Ext(element)))
|
||||
}
|
||||
}
|
||||
expanded = strings.TrimSpace(expanded)
|
||||
}
|
||||
content = fmt.Sprintf("%s%s%s", content[0:indexes[0]], expanded, content[indexes[1]:len(content)])
|
||||
}
|
||||
|
||||
this.hash = fmt.Sprintf("%x", sha1.Sum([]byte(content)))
|
||||
if err := json.Unmarshal([]byte(content), &this.config); err != nil {
|
||||
if err := json.Unmarshal([]byte("{"+content+"}"), &this.config); err != nil {
|
||||
this.config = nil
|
||||
if syntax, ok := err.(*json.SyntaxError); ok && syntax.Offset < int64(len(content)) {
|
||||
if start := strings.LastIndex(content[:syntax.Offset], "\n") + 1; start >= 0 {
|
||||
line := strings.Count(content[:start], "\n") + 1
|
||||
return errors.New(fmt.Sprintf("%s at line %d near %s", syntax, line, content[start:syntax.Offset]))
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
reduce(this.config)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *UConfig) Loaded() bool {
|
||||
if cores > 1 {
|
||||
this.RLock()
|
||||
defer this.RUnlock()
|
||||
}
|
||||
return !(this.config == nil)
|
||||
}
|
||||
|
||||
func (this *UConfig) Hash() string {
|
||||
if cores > 1 {
|
||||
this.RLock()
|
||||
defer this.RUnlock()
|
||||
}
|
||||
return this.hash
|
||||
}
|
||||
|
||||
func (this *UConfig) String() string {
|
||||
if config, err := json.MarshalIndent(this.config, " ", " "); this.config != nil && err == nil {
|
||||
return string(config)
|
||||
}
|
||||
return "{}"
|
||||
}
|
||||
|
||||
func (this *UConfig) GetPaths(path string) []string {
|
||||
var (
|
||||
current interface{} = this.config
|
||||
paths []string = []string{}
|
||||
)
|
||||
|
||||
if cores > 1 {
|
||||
this.RLock()
|
||||
defer this.RUnlock()
|
||||
}
|
||||
prefix := ""
|
||||
if current == nil || path == "" {
|
||||
return paths
|
||||
}
|
||||
if cores > 1 {
|
||||
this.cacheLock.RLock()
|
||||
}
|
||||
if this.cache[path] != nil {
|
||||
if paths, ok := this.cache[path].([]string); ok {
|
||||
if cores > 1 {
|
||||
this.cacheLock.RUnlock()
|
||||
}
|
||||
return paths
|
||||
}
|
||||
}
|
||||
if cores > 1 {
|
||||
this.cacheLock.RUnlock()
|
||||
}
|
||||
if path != "" {
|
||||
prefix = "."
|
||||
for _, part := range strings.Split(path, ".") {
|
||||
kind := reflect.TypeOf(current).Kind()
|
||||
index, err := strconv.Atoi(part)
|
||||
if (kind == reflect.Slice && (err != nil || index < 0 || index >= len(current.([]interface{})))) || (kind != reflect.Slice && kind != reflect.Map) {
|
||||
if cores > 1 {
|
||||
this.cacheLock.Lock()
|
||||
}
|
||||
this.cache[path] = paths
|
||||
if cores > 1 {
|
||||
this.cacheLock.Unlock()
|
||||
}
|
||||
return paths
|
||||
}
|
||||
if kind == reflect.Slice {
|
||||
current = current.([]interface{})[index]
|
||||
} else {
|
||||
if current = current.(map[string]interface{})[strings.TrimSpace(part)]; current == nil {
|
||||
if cores > 1 {
|
||||
this.cacheLock.Lock()
|
||||
}
|
||||
this.cache[path] = paths
|
||||
if cores > 1 {
|
||||
this.cacheLock.Unlock()
|
||||
}
|
||||
return paths
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
switch reflect.TypeOf(current).Kind() {
|
||||
case reflect.Slice:
|
||||
for index := 0; index < len(current.([]interface{})); index++ {
|
||||
paths = append(paths, fmt.Sprintf("%s%s%d", path, prefix, index))
|
||||
}
|
||||
case reflect.Map:
|
||||
for key, _ := range current.(map[string]interface{}) {
|
||||
paths = append(paths, fmt.Sprintf("%s%s%s", path, prefix, key))
|
||||
}
|
||||
}
|
||||
if cores > 1 {
|
||||
this.cacheLock.Lock()
|
||||
}
|
||||
this.cache[path] = paths
|
||||
if cores > 1 {
|
||||
this.cacheLock.Unlock()
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func (this *UConfig) value(path string) (string, error) {
|
||||
var current interface{} = this.config
|
||||
|
||||
if cores > 1 {
|
||||
this.RLock()
|
||||
defer this.RUnlock()
|
||||
}
|
||||
if current == nil || path == "" {
|
||||
return "", fmt.Errorf("invalid parameter")
|
||||
}
|
||||
if cores > 1 {
|
||||
this.cacheLock.RLock()
|
||||
}
|
||||
if this.cache[path] != nil {
|
||||
if current, ok := this.cache[path].(bool); ok && !current {
|
||||
if cores > 1 {
|
||||
this.cacheLock.RUnlock()
|
||||
}
|
||||
return "", fmt.Errorf("invalid path")
|
||||
}
|
||||
if current, ok := this.cache[path].(string); ok {
|
||||
if cores > 1 {
|
||||
this.cacheLock.RUnlock()
|
||||
}
|
||||
return current, nil
|
||||
}
|
||||
}
|
||||
if cores > 1 {
|
||||
this.cacheLock.RUnlock()
|
||||
}
|
||||
for _, part := range strings.Split(path, ".") {
|
||||
kind := reflect.TypeOf(current).Kind()
|
||||
index, err := strconv.Atoi(part)
|
||||
if (kind == reflect.Slice && (err != nil || index < 0 || index >= len(current.([]interface{})))) || (kind != reflect.Slice && kind != reflect.Map) {
|
||||
if cores > 1 {
|
||||
this.cacheLock.Lock()
|
||||
}
|
||||
this.cache[path] = false
|
||||
if cores > 1 {
|
||||
this.cacheLock.Unlock()
|
||||
}
|
||||
return "", fmt.Errorf("invalid path")
|
||||
}
|
||||
if kind == reflect.Slice {
|
||||
current = current.([]interface{})[index]
|
||||
} else {
|
||||
if current = current.(map[string]interface{})[strings.TrimSpace(part)]; current == nil {
|
||||
if cores > 1 {
|
||||
this.cacheLock.Lock()
|
||||
}
|
||||
this.cache[path] = false
|
||||
if cores > 1 {
|
||||
this.cacheLock.Unlock()
|
||||
}
|
||||
return "", fmt.Errorf("invalid path")
|
||||
}
|
||||
}
|
||||
}
|
||||
if reflect.TypeOf(current).Kind() == reflect.String {
|
||||
if cores > 1 {
|
||||
this.cacheLock.Lock()
|
||||
}
|
||||
this.cache[path] = current.(string)
|
||||
if cores > 1 {
|
||||
this.cacheLock.Unlock()
|
||||
}
|
||||
return current.(string), nil
|
||||
}
|
||||
if cores > 1 {
|
||||
this.cacheLock.Lock()
|
||||
}
|
||||
this.cache[path] = false
|
||||
if cores > 1 {
|
||||
this.cacheLock.Unlock()
|
||||
}
|
||||
return "", fmt.Errorf("invalid path")
|
||||
}
|
||||
|
||||
func (this *UConfig) GetBoolean(path string, fallback bool) bool {
|
||||
value, err := this.value(path)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
if value = strings.ToLower(strings.TrimSpace(value)); value == "1" || value == "on" || value == "yes" || value == "true" {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (this *UConfig) GetString(path string, fallback string) string {
|
||||
return this.GetStringMatch(path, fallback, "")
|
||||
}
|
||||
func (this *UConfig) GetStringMatch(path string, fallback, match string) string {
|
||||
return this.GetStringMatchCaptures(path, fallback, match)[0]
|
||||
}
|
||||
func (this *UConfig) GetStringMatchCaptures(path string, fallback, match string) []string {
|
||||
value, err := this.value(path)
|
||||
if err != nil {
|
||||
return []string{fallback}
|
||||
}
|
||||
if match != "" {
|
||||
if matcher, err := regexp.Compile(match); err == nil {
|
||||
if matches := matcher.FindStringSubmatch(value); matches != nil {
|
||||
return matches
|
||||
} else {
|
||||
return []string{fallback}
|
||||
}
|
||||
} else {
|
||||
return []string{fallback}
|
||||
}
|
||||
}
|
||||
return []string{value}
|
||||
}
|
||||
|
||||
func (this *UConfig) GetInteger(path string, fallback int64) int64 {
|
||||
return this.GetIntegerBounds(path, fallback, math.MinInt64, math.MaxInt64)
|
||||
}
|
||||
func (this *UConfig) GetIntegerBounds(path string, fallback, min, max int64) int64 {
|
||||
value, err := this.value(path)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
nvalue, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
if nvalue < min {
|
||||
nvalue = min
|
||||
}
|
||||
if nvalue > max {
|
||||
nvalue = max
|
||||
}
|
||||
return nvalue
|
||||
}
|
||||
|
||||
func (this *UConfig) GetFloat(path string, fallback float64) float64 {
|
||||
return this.GetFloatBounds(path, fallback, -math.MaxFloat64, math.MaxFloat64)
|
||||
}
|
||||
func (this *UConfig) GetFloatBounds(path string, fallback, min, max float64) float64 {
|
||||
value, err := this.value(path)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
nvalue, err := strconv.ParseFloat(strings.TrimSpace(value), 64)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
return math.Max(math.Min(nvalue, max), min)
|
||||
}
|
||||
|
||||
func (this *UConfig) GetSize(path string, fallback int64) int64 {
|
||||
return this.GetSizeBounds(path, fallback, math.MinInt64, math.MaxInt64)
|
||||
}
|
||||
func (this *UConfig) GetSizeBounds(path string, fallback, min, max int64) int64 {
|
||||
value, err := this.value(path)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
nvalue := int64(0)
|
||||
if matches := sizer.FindStringSubmatch(strings.TrimSpace(strings.ToUpper(value))); matches != nil {
|
||||
fvalue, err := strconv.ParseFloat(matches[1], 64)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
scale := float64(1000)
|
||||
if matches[3] == "B" {
|
||||
scale = float64(1024)
|
||||
}
|
||||
nvalue = int64(fvalue * math.Pow(scale, float64(strings.Index("_KMGTP", matches[2]))))
|
||||
} else {
|
||||
return fallback
|
||||
}
|
||||
if nvalue < min {
|
||||
nvalue = min
|
||||
}
|
||||
if nvalue > max {
|
||||
nvalue = max
|
||||
}
|
||||
return nvalue
|
||||
}
|
||||
|
||||
func (this *UConfig) GetDuration(path string, fallback float64) float64 {
|
||||
return this.GetDurationBounds(path, fallback, -math.MaxFloat64, math.MaxFloat64)
|
||||
}
|
||||
func (this *UConfig) GetDurationBounds(path string, fallback, min, max float64) float64 {
|
||||
value, err := this.value(path)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
nvalue := float64(0.0)
|
||||
if matches := duration1.FindAllStringSubmatch(strings.TrimSpace(strings.ToUpper(value)), -1); matches != nil {
|
||||
for index := 0; index < len(matches); index++ {
|
||||
if uvalue, err := strconv.ParseFloat(matches[index][1], 64); err == nil {
|
||||
switch matches[index][2] {
|
||||
case "Y":
|
||||
nvalue += uvalue * 86400 * 365.256
|
||||
case "MO":
|
||||
nvalue += uvalue * 86400 * 365.256 / 12
|
||||
case "D":
|
||||
nvalue += uvalue * 86400
|
||||
case "H":
|
||||
nvalue += uvalue * 3600
|
||||
case "MN":
|
||||
nvalue += uvalue * 60
|
||||
case "S":
|
||||
nvalue += uvalue
|
||||
case "":
|
||||
nvalue += uvalue
|
||||
case "MS":
|
||||
nvalue += uvalue / 1000
|
||||
case "US":
|
||||
nvalue += uvalue / 1000000
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if matches := duration2.FindStringSubmatch(strings.TrimSpace(value)); matches != nil {
|
||||
hours, _ := strconv.ParseFloat(matches[1], 64)
|
||||
minutes, _ := strconv.ParseFloat(matches[2], 64)
|
||||
seconds, _ := strconv.ParseFloat(matches[3], 64)
|
||||
milliseconds, _ := strconv.ParseFloat(matches[4], 64)
|
||||
nvalue = (hours * 3600) + (math.Min(minutes, 59) * 60) + math.Min(seconds, 59) + (milliseconds / 1000)
|
||||
}
|
||||
return math.Max(math.Min(nvalue, max), min)
|
||||
}
|
||||
|
|
@ -0,0 +1,545 @@
|
|||
package ulog
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/syslog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
const (
|
||||
TIME_NONE int = iota
|
||||
TIME_DATETIME
|
||||
TIME_MSDATETIME
|
||||
TIME_TIMESTAMP
|
||||
TIME_MSTIMESTAMP
|
||||
)
|
||||
|
||||
var (
|
||||
cores int
|
||||
facilities = map[string]syslog.Priority{
|
||||
"user": syslog.LOG_USER,
|
||||
"daemon": syslog.LOG_DAEMON,
|
||||
"local0": syslog.LOG_LOCAL0,
|
||||
"local1": syslog.LOG_LOCAL1,
|
||||
"local2": syslog.LOG_LOCAL2,
|
||||
"local3": syslog.LOG_LOCAL3,
|
||||
"local4": syslog.LOG_LOCAL4,
|
||||
"local5": syslog.LOG_LOCAL5,
|
||||
"local6": syslog.LOG_LOCAL6,
|
||||
"local7": syslog.LOG_LOCAL7,
|
||||
}
|
||||
severities = map[string]syslog.Priority{
|
||||
"error": syslog.LOG_ERR,
|
||||
"warning": syslog.LOG_WARNING,
|
||||
"info": syslog.LOG_INFO,
|
||||
"debug": syslog.LOG_DEBUG,
|
||||
}
|
||||
severityLabels = map[syslog.Priority]string{
|
||||
syslog.LOG_ERR: "ERRO ",
|
||||
syslog.LOG_WARNING: "WARN ",
|
||||
syslog.LOG_INFO: "INFO ",
|
||||
syslog.LOG_DEBUG: "DBUG ",
|
||||
}
|
||||
severityColors = map[syslog.Priority]string{
|
||||
syslog.LOG_ERR: "\x1b[31m",
|
||||
syslog.LOG_WARNING: "\x1b[33m",
|
||||
syslog.LOG_INFO: "\x1b[36m",
|
||||
syslog.LOG_DEBUG: "\x1b[32m",
|
||||
}
|
||||
)
|
||||
|
||||
type FileOutput struct {
|
||||
handle *os.File
|
||||
last time.Time
|
||||
}
|
||||
type ULog struct {
|
||||
file, console, syslog bool
|
||||
fileOutputs map[string]*FileOutput
|
||||
filePath string
|
||||
fileTime int
|
||||
fileLast time.Time
|
||||
fileSeverity bool
|
||||
consoleHandle io.Writer
|
||||
consoleTime int
|
||||
consoleSeverity bool
|
||||
consoleColors bool
|
||||
syslogHandle *syslog.Writer
|
||||
syslogRemote string
|
||||
syslogName string
|
||||
syslogFacility syslog.Priority
|
||||
optionUTC bool
|
||||
level syslog.Priority
|
||||
sync.Mutex
|
||||
}
|
||||
|
||||
func New(target string) *ULog {
|
||||
if cores == 0 {
|
||||
cores = runtime.NumCPU()
|
||||
}
|
||||
log := &ULog{
|
||||
fileOutputs: map[string]*FileOutput{},
|
||||
syslogHandle: nil,
|
||||
}
|
||||
return log.Load(target)
|
||||
}
|
||||
|
||||
func (this *ULog) Load(target string) *ULog {
|
||||
this.Close()
|
||||
if cores > 1 {
|
||||
this.Lock()
|
||||
defer this.Unlock()
|
||||
}
|
||||
|
||||
this.file = false
|
||||
this.filePath = ""
|
||||
this.fileTime = TIME_DATETIME
|
||||
this.fileSeverity = true
|
||||
this.console = false
|
||||
this.consoleTime = TIME_DATETIME
|
||||
this.consoleSeverity = true
|
||||
this.consoleColors = true
|
||||
this.consoleHandle = os.Stderr
|
||||
this.syslog = false
|
||||
this.syslogRemote = ""
|
||||
this.syslogName = filepath.Base(os.Args[0])
|
||||
this.syslogFacility = syslog.LOG_DAEMON
|
||||
this.optionUTC = false
|
||||
this.level = syslog.LOG_INFO
|
||||
for _, target := range regexp.MustCompile("(file|console|syslog|option)\\s*\\(([^\\)]*)\\)").FindAllStringSubmatch(target, -1) {
|
||||
switch strings.ToLower(target[1]) {
|
||||
case "file":
|
||||
this.file = true
|
||||
for _, option := range regexp.MustCompile("([^:=,\\s]+)\\s*[:=]\\s*([^,\\s]+)").FindAllStringSubmatch(target[2], -1) {
|
||||
switch strings.ToLower(option[1]) {
|
||||
case "path":
|
||||
this.filePath = option[2]
|
||||
case "time":
|
||||
option[2] = strings.ToLower(option[2])
|
||||
switch {
|
||||
case option[2] == "datetime":
|
||||
this.fileTime = TIME_DATETIME
|
||||
case option[2] == "msdatetime":
|
||||
this.fileTime = TIME_MSDATETIME
|
||||
case option[2] == "stamp" || option[2] == "timestamp":
|
||||
this.fileTime = TIME_TIMESTAMP
|
||||
case option[2] == "msstamp" || option[2] == "mstimestamp":
|
||||
this.fileTime = TIME_MSTIMESTAMP
|
||||
case option[2] != "1" && option[2] != "true" && option[2] != "on" && option[2] != "yes":
|
||||
this.fileTime = TIME_NONE
|
||||
}
|
||||
case "severity":
|
||||
option[2] = strings.ToLower(option[2])
|
||||
if option[2] != "1" && option[2] != "true" && option[2] != "on" && option[2] != "yes" {
|
||||
this.fileSeverity = false
|
||||
}
|
||||
}
|
||||
}
|
||||
if this.filePath == "" {
|
||||
this.file = false
|
||||
}
|
||||
case "console":
|
||||
this.console = true
|
||||
for _, option := range regexp.MustCompile("([^:=,\\s]+)\\s*[:=]\\s*([^,\\s]+)").FindAllStringSubmatch(target[2], -1) {
|
||||
option[2] = strings.ToLower(option[2])
|
||||
switch strings.ToLower(option[1]) {
|
||||
case "output":
|
||||
if option[2] == "stdout" {
|
||||
this.consoleHandle = os.Stdout
|
||||
}
|
||||
case "time":
|
||||
switch {
|
||||
case option[2] == "datetime":
|
||||
this.consoleTime = TIME_DATETIME
|
||||
case option[2] == "msdatetime":
|
||||
this.consoleTime = TIME_MSDATETIME
|
||||
case option[2] == "stamp" || option[2] == "timestamp":
|
||||
this.consoleTime = TIME_TIMESTAMP
|
||||
case option[2] == "msstamp" || option[2] == "mstimestamp":
|
||||
this.consoleTime = TIME_MSTIMESTAMP
|
||||
case option[2] != "1" && option[2] != "true" && option[2] != "on" && option[2] != "yes":
|
||||
this.consoleTime = TIME_NONE
|
||||
}
|
||||
case "severity":
|
||||
if option[2] != "1" && option[2] != "true" && option[2] != "on" && option[2] != "yes" {
|
||||
this.consoleSeverity = false
|
||||
}
|
||||
case "colors":
|
||||
if option[2] != "1" && option[2] != "true" && option[2] != "on" && option[2] != "yes" {
|
||||
this.consoleColors = false
|
||||
}
|
||||
}
|
||||
}
|
||||
case "syslog":
|
||||
this.syslog = true
|
||||
for _, option := range regexp.MustCompile("([^:=,\\s]+)\\s*[:=]\\s*([^,\\s]+)").FindAllStringSubmatch(target[2], -1) {
|
||||
switch strings.ToLower(option[1]) {
|
||||
case "remote":
|
||||
this.syslogRemote = option[2]
|
||||
if !regexp.MustCompile(":\\d+$").MatchString(this.syslogRemote) {
|
||||
this.syslogRemote += ":514"
|
||||
}
|
||||
case "name":
|
||||
this.syslogName = option[2]
|
||||
case "facility":
|
||||
this.syslogFacility = facilities[strings.ToLower(option[2])]
|
||||
}
|
||||
}
|
||||
case "option":
|
||||
for _, option := range regexp.MustCompile("([^:=,\\s]+)\\s*[:=]\\s*([^,\\s]+)").FindAllStringSubmatch(target[2], -1) {
|
||||
option[2] = strings.ToLower(option[2])
|
||||
switch strings.ToLower(option[1]) {
|
||||
case "utc":
|
||||
if option[2] == "1" || option[2] == "true" || option[2] == "on" || option[2] == "yes" {
|
||||
this.optionUTC = true
|
||||
}
|
||||
case "level":
|
||||
this.level = severities[strings.ToLower(option[2])]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var info syscall.Termios
|
||||
if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, this.consoleHandle.(*os.File).Fd(), syscall.TCGETS, uintptr(unsafe.Pointer(&info)), 0, 0, 0); err != 0 {
|
||||
this.consoleColors = false
|
||||
}
|
||||
return this
|
||||
}
|
||||
|
||||
func (this *ULog) Close() {
|
||||
if cores > 1 {
|
||||
this.Lock()
|
||||
defer this.Unlock()
|
||||
}
|
||||
if this.syslogHandle != nil {
|
||||
this.syslogHandle.Close()
|
||||
this.syslogHandle = nil
|
||||
}
|
||||
for path, output := range this.fileOutputs {
|
||||
if output.handle != nil {
|
||||
output.handle.Close()
|
||||
}
|
||||
delete(this.fileOutputs, path)
|
||||
}
|
||||
}
|
||||
|
||||
func (this *ULog) SetLevel(level string) {
|
||||
level = strings.ToLower(level)
|
||||
switch level {
|
||||
case "error":
|
||||
this.level = syslog.LOG_ERR
|
||||
case "warning":
|
||||
this.level = syslog.LOG_WARNING
|
||||
case "info":
|
||||
this.level = syslog.LOG_INFO
|
||||
case "debug":
|
||||
this.level = syslog.LOG_DEBUG
|
||||
}
|
||||
}
|
||||
|
||||
func strftime(layout string, base time.Time) string {
|
||||
var output []string
|
||||
|
||||
length := len(layout)
|
||||
for index := 0; index < length; index++ {
|
||||
switch layout[index] {
|
||||
case '%':
|
||||
if index < length-1 {
|
||||
switch layout[index+1] {
|
||||
case 'a':
|
||||
output = append(output, base.Format("Mon"))
|
||||
case 'A':
|
||||
output = append(output, base.Format("Monday"))
|
||||
case 'b':
|
||||
output = append(output, base.Format("Jan"))
|
||||
case 'B':
|
||||
output = append(output, base.Format("January"))
|
||||
case 'c':
|
||||
output = append(output, base.Format("Mon Jan 2 15:04:05 2006"))
|
||||
case 'C':
|
||||
output = append(output, fmt.Sprintf("%02d", base.Year()/100))
|
||||
case 'd':
|
||||
output = append(output, fmt.Sprintf("%02d", base.Day()))
|
||||
case 'D':
|
||||
output = append(output, fmt.Sprintf("%02d/%02d/%02d", base.Month(), base.Day(), base.Year()%100))
|
||||
case 'e':
|
||||
output = append(output, fmt.Sprintf("%2d", base.Day()))
|
||||
case 'f':
|
||||
output = append(output, fmt.Sprintf("%06d", base.Nanosecond()/1000))
|
||||
case 'F':
|
||||
output = append(output, fmt.Sprintf("%04d-%02d-%02d", base.Year(), base.Month(), base.Day()))
|
||||
case 'g':
|
||||
year, _ := base.ISOWeek()
|
||||
output = append(output, fmt.Sprintf("%02d", year%100))
|
||||
case 'G':
|
||||
year, _ := base.ISOWeek()
|
||||
output = append(output, fmt.Sprintf("%04d", year))
|
||||
case 'h':
|
||||
output = append(output, base.Format("Jan"))
|
||||
case 'H':
|
||||
output = append(output, fmt.Sprintf("%02d", base.Hour()))
|
||||
case 'I':
|
||||
if base.Hour() == 0 || base.Hour() == 12 {
|
||||
output = append(output, "12")
|
||||
} else {
|
||||
output = append(output, fmt.Sprintf("%02d", base.Hour()%12))
|
||||
}
|
||||
case 'j':
|
||||
output = append(output, fmt.Sprintf("%03d", base.YearDay()))
|
||||
case 'k':
|
||||
output = append(output, fmt.Sprintf("%2d", base.Hour()))
|
||||
case 'l':
|
||||
if base.Hour() == 0 || base.Hour() == 12 {
|
||||
output = append(output, "12")
|
||||
} else {
|
||||
output = append(output, fmt.Sprintf("%2d", base.Hour()%12))
|
||||
}
|
||||
case 'm':
|
||||
output = append(output, fmt.Sprintf("%02d", base.Month()))
|
||||
case 'M':
|
||||
output = append(output, fmt.Sprintf("%02d", base.Minute()))
|
||||
case 'n':
|
||||
output = append(output, "\n")
|
||||
case 'p':
|
||||
if base.Hour() < 12 {
|
||||
output = append(output, "AM")
|
||||
} else {
|
||||
output = append(output, "PM")
|
||||
}
|
||||
case 'P':
|
||||
if base.Hour() < 12 {
|
||||
output = append(output, "am")
|
||||
} else {
|
||||
output = append(output, "pm")
|
||||
}
|
||||
case 'r':
|
||||
if base.Hour() == 0 || base.Hour() == 12 {
|
||||
output = append(output, "12")
|
||||
} else {
|
||||
output = append(output, fmt.Sprintf("%02d", base.Hour()%12))
|
||||
}
|
||||
output = append(output, fmt.Sprintf(":%02d:%02d", base.Minute(), base.Second()))
|
||||
if base.Hour() < 12 {
|
||||
output = append(output, " AM")
|
||||
} else {
|
||||
output = append(output, " PM")
|
||||
}
|
||||
case 'R':
|
||||
output = append(output, fmt.Sprintf("%02d:%02d", base.Hour(), base.Minute()))
|
||||
case 's':
|
||||
output = append(output, fmt.Sprintf("%d", base.Unix()))
|
||||
case 'S':
|
||||
output = append(output, fmt.Sprintf("%02d", base.Second()))
|
||||
case 't':
|
||||
output = append(output, "\t")
|
||||
case 'T':
|
||||
output = append(output, fmt.Sprintf("%02d:%02d:%02d", base.Hour(), base.Minute(), base.Second()))
|
||||
case 'u':
|
||||
day := base.Weekday()
|
||||
if day == 0 {
|
||||
day = 7
|
||||
}
|
||||
output = append(output, fmt.Sprintf("%d", day))
|
||||
case 'U':
|
||||
output = append(output, fmt.Sprintf("%d", (base.YearDay()+6-int(base.Weekday()))/7))
|
||||
case 'V':
|
||||
_, week := base.ISOWeek()
|
||||
output = append(output, fmt.Sprintf("%02d", week))
|
||||
case 'w':
|
||||
output = append(output, fmt.Sprintf("%d", base.Weekday()))
|
||||
case 'W':
|
||||
day := int(base.Weekday())
|
||||
if day == 0 {
|
||||
day = 6
|
||||
} else {
|
||||
day -= 1
|
||||
}
|
||||
output = append(output, fmt.Sprintf("%d", (base.YearDay()+6-day)/7))
|
||||
case 'x':
|
||||
output = append(output, fmt.Sprintf("%02d/%02d/%02d", base.Month(), base.Day(), base.Year()%100))
|
||||
case 'X':
|
||||
output = append(output, fmt.Sprintf("%02d:%02d:%02d", base.Hour(), base.Minute(), base.Second()))
|
||||
case 'y':
|
||||
output = append(output, fmt.Sprintf("%02d", base.Year()%100))
|
||||
case 'Y':
|
||||
output = append(output, fmt.Sprintf("%04d", base.Year()))
|
||||
case 'z':
|
||||
output = append(output, base.Format("-0700"))
|
||||
case 'Z':
|
||||
output = append(output, base.Format("MST"))
|
||||
case '%':
|
||||
output = append(output, "%")
|
||||
}
|
||||
index++
|
||||
}
|
||||
default:
|
||||
output = append(output, string(layout[index]))
|
||||
}
|
||||
}
|
||||
return strings.Join(output, "")
|
||||
}
|
||||
|
||||
func (this *ULog) log(now time.Time, severity syslog.Priority, xlayout interface{}, a ...interface{}) {
|
||||
var err error
|
||||
if this.level < severity || (!this.syslog && !this.file && !this.console) {
|
||||
return
|
||||
}
|
||||
layout := ""
|
||||
switch reflect.TypeOf(xlayout).Kind() {
|
||||
case reflect.Map:
|
||||
var buffer bytes.Buffer
|
||||
|
||||
encoder := json.NewEncoder(&buffer)
|
||||
encoder.SetEscapeHTML(false)
|
||||
if err := encoder.Encode(xlayout); err == nil {
|
||||
layout = "%s"
|
||||
a = []interface{}{bytes.TrimSpace(buffer.Bytes())}
|
||||
}
|
||||
case reflect.String:
|
||||
layout = xlayout.(string)
|
||||
}
|
||||
layout = strings.TrimSpace(layout)
|
||||
if this.syslog {
|
||||
if this.syslogHandle == nil {
|
||||
if cores > 1 {
|
||||
this.Lock()
|
||||
}
|
||||
if this.syslogHandle == nil {
|
||||
protocol := ""
|
||||
if this.syslogRemote != "" {
|
||||
protocol = "udp"
|
||||
}
|
||||
if this.syslogHandle, err = syslog.Dial(protocol, this.syslogRemote, this.syslogFacility, this.syslogName); err != nil {
|
||||
this.syslogHandle = nil
|
||||
}
|
||||
}
|
||||
if cores > 1 {
|
||||
this.Unlock()
|
||||
}
|
||||
}
|
||||
if this.syslogHandle != nil {
|
||||
switch severity {
|
||||
case syslog.LOG_ERR:
|
||||
this.syslogHandle.Err(fmt.Sprintf(layout, a...))
|
||||
case syslog.LOG_WARNING:
|
||||
this.syslogHandle.Warning(fmt.Sprintf(layout, a...))
|
||||
case syslog.LOG_INFO:
|
||||
this.syslogHandle.Info(fmt.Sprintf(layout, a...))
|
||||
case syslog.LOG_DEBUG:
|
||||
this.syslogHandle.Debug(fmt.Sprintf(layout, a...))
|
||||
}
|
||||
}
|
||||
}
|
||||
if this.optionUTC {
|
||||
now = now.UTC()
|
||||
} else {
|
||||
now = now.Local()
|
||||
}
|
||||
if this.file {
|
||||
path := strftime(this.filePath, now)
|
||||
if cores > 1 {
|
||||
this.Lock()
|
||||
}
|
||||
if this.fileOutputs[path] == nil {
|
||||
os.MkdirAll(filepath.Dir(path), 0755)
|
||||
if handle, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644); err == nil {
|
||||
this.fileOutputs[path] = &FileOutput{handle: handle}
|
||||
}
|
||||
}
|
||||
if this.fileOutputs[path] != nil && this.fileOutputs[path].handle != nil {
|
||||
prefix := ""
|
||||
switch this.fileTime {
|
||||
case TIME_DATETIME:
|
||||
prefix = fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d ", now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second())
|
||||
case TIME_MSDATETIME:
|
||||
prefix = fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d.%03d ", now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second(), now.Nanosecond()/int(time.Millisecond))
|
||||
case TIME_TIMESTAMP:
|
||||
prefix = fmt.Sprintf("%d ", now.Unix())
|
||||
case TIME_MSTIMESTAMP:
|
||||
prefix = fmt.Sprintf("%d ", now.UnixNano()/int64(time.Millisecond))
|
||||
}
|
||||
if this.fileSeverity {
|
||||
prefix += severityLabels[severity]
|
||||
}
|
||||
this.fileOutputs[path].handle.WriteString(fmt.Sprintf(prefix+layout+"\n", a...))
|
||||
this.fileOutputs[path].last = now
|
||||
}
|
||||
if now.Sub(this.fileLast) >= 5*time.Second {
|
||||
this.fileLast = now
|
||||
for path, output := range this.fileOutputs {
|
||||
if now.Sub(output.last) >= 5*time.Second {
|
||||
output.handle.Close()
|
||||
delete(this.fileOutputs, path)
|
||||
}
|
||||
}
|
||||
}
|
||||
if cores > 1 {
|
||||
this.Unlock()
|
||||
}
|
||||
}
|
||||
if this.console {
|
||||
prefix := ""
|
||||
switch this.consoleTime {
|
||||
case TIME_DATETIME:
|
||||
prefix = fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d ", now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second())
|
||||
case TIME_MSDATETIME:
|
||||
prefix = fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d.%03d ", now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second(), now.Nanosecond()/int(time.Millisecond))
|
||||
case TIME_TIMESTAMP:
|
||||
prefix = fmt.Sprintf("%d ", now.Unix())
|
||||
case TIME_MSTIMESTAMP:
|
||||
prefix = fmt.Sprintf("%d ", now.UnixNano()/int64(time.Millisecond))
|
||||
}
|
||||
if this.consoleSeverity {
|
||||
if this.consoleColors {
|
||||
prefix += fmt.Sprintf("%s%s\x1b[0m", severityColors[severity], severityLabels[severity])
|
||||
} else {
|
||||
prefix += severityLabels[severity]
|
||||
}
|
||||
}
|
||||
if cores > 1 {
|
||||
this.Lock()
|
||||
}
|
||||
fmt.Fprintf(this.consoleHandle, prefix+layout+"\n", a...)
|
||||
if cores > 1 {
|
||||
this.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (this *ULog) Error(layout interface{}, a ...interface{}) {
|
||||
this.log(time.Now(), syslog.LOG_ERR, layout, a...)
|
||||
}
|
||||
func (this *ULog) Warn(layout interface{}, a ...interface{}) {
|
||||
this.log(time.Now(), syslog.LOG_WARNING, layout, a...)
|
||||
}
|
||||
func (this *ULog) Info(layout interface{}, a ...interface{}) {
|
||||
this.log(time.Now(), syslog.LOG_INFO, layout, a...)
|
||||
}
|
||||
func (this *ULog) Debug(layout interface{}, a ...interface{}) {
|
||||
this.log(time.Now(), syslog.LOG_DEBUG, layout, a...)
|
||||
}
|
||||
|
||||
func (this *ULog) ErrorTime(now time.Time, layout interface{}, a ...interface{}) {
|
||||
this.log(now, syslog.LOG_ERR, layout, a...)
|
||||
}
|
||||
func (this *ULog) WarnTime(now time.Time, layout interface{}, a ...interface{}) {
|
||||
this.log(now, syslog.LOG_WARNING, layout, a...)
|
||||
}
|
||||
func (this *ULog) InfoTime(now time.Time, layout interface{}, a ...interface{}) {
|
||||
this.log(now, syslog.LOG_INFO, layout, a...)
|
||||
}
|
||||
func (this *ULog) DebugTime(now time.Time, layout interface{}, a ...interface{}) {
|
||||
this.log(now, syslog.LOG_DEBUG, layout, a...)
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
package uuid
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
var initialized bool
|
||||
|
||||
func init() {
|
||||
if !initialized {
|
||||
rand.Seed(time.Now().UnixNano() + int64(os.Getpid()))
|
||||
initialized = true
|
||||
}
|
||||
}
|
||||
|
||||
func UUID() string {
|
||||
var entropy = make([]byte, 16)
|
||||
|
||||
if !initialized {
|
||||
rand.Seed(time.Now().UnixNano() + int64(os.Getpid()))
|
||||
initialized = true
|
||||
}
|
||||
rand.Read(entropy)
|
||||
entropy[6] = (entropy[6] & 0x0f) | 0x40
|
||||
entropy[8] = (entropy[8] & 0x3f) | 0x80
|
||||
return fmt.Sprintf("%08x-%04x-%04x-%04x-%12x", entropy[0:4], entropy[4:6], entropy[6:8], entropy[8:10], entropy[10:16])
|
||||
}
|
||||
|
|
@ -0,0 +1,244 @@
|
|||
package whohas
|
||||
|
||||
import (
|
||||
"context"
|
||||
"mime"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type BACKEND struct {
|
||||
Host string
|
||||
Secure bool
|
||||
Path string
|
||||
Headers map[string]string
|
||||
Penalty time.Duration
|
||||
}
|
||||
type CACHE struct {
|
||||
TTL time.Duration
|
||||
last time.Time
|
||||
items map[string]*LOOKUP
|
||||
sync.RWMutex
|
||||
}
|
||||
type LOOKUP struct {
|
||||
index int
|
||||
deadline time.Time
|
||||
Protocol string
|
||||
Host string
|
||||
Headers map[string]string
|
||||
Size int64
|
||||
Mime string
|
||||
Ranges bool
|
||||
Date time.Time
|
||||
Modified time.Time
|
||||
Expires time.Time
|
||||
}
|
||||
|
||||
var cores int
|
||||
|
||||
func Lookup(path string, backends []BACKEND, timeout time.Duration, cache *CACHE, ckey string) (lookup *LOOKUP) {
|
||||
if cores == 0 {
|
||||
cores = runtime.NumCPU()
|
||||
}
|
||||
if path == "" || backends == nil || len(backends) < 1 || timeout < 100*time.Millisecond {
|
||||
return
|
||||
}
|
||||
|
||||
cpath := path
|
||||
if index := strings.Index(path, "?"); index >= 0 {
|
||||
cpath = path[:index]
|
||||
}
|
||||
cbackends := backends
|
||||
if cache != nil && cache.items != nil {
|
||||
now := time.Now()
|
||||
if cores > 1 {
|
||||
cache.RLock()
|
||||
}
|
||||
if cache.items[cpath] != nil && now.Sub(cache.items[cpath].deadline) < 0 {
|
||||
lookup = cache.items[cpath]
|
||||
if cache.items[cpath].Host != "" {
|
||||
cbackends = []BACKEND{}
|
||||
for _, backend := range backends {
|
||||
if backend.Host == cache.items[cpath].Host {
|
||||
cbackends = append(cbackends, backend)
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(cbackends) < 1 {
|
||||
cbackends = backends
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(cbackends) == len(backends) && ckey != "" && cache.items["k"+ckey] != nil && now.Sub(cache.items["k"+ckey].deadline) < 0 {
|
||||
cbackends = []BACKEND{}
|
||||
for _, backend := range backends {
|
||||
if backend.Host == cache.items["k"+ckey].Host {
|
||||
cbackends = append(cbackends, backend)
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(cbackends) < 1 {
|
||||
cbackends = backends
|
||||
}
|
||||
}
|
||||
if cores > 1 {
|
||||
cache.RUnlock()
|
||||
}
|
||||
}
|
||||
|
||||
if lookup == nil {
|
||||
inflight := len(cbackends)
|
||||
sink := make(chan LOOKUP, inflight+1)
|
||||
cancels := make([]context.CancelFunc, inflight)
|
||||
for index, backend := range cbackends {
|
||||
var ctx context.Context
|
||||
|
||||
ctx, cancels[index] = context.WithCancel(context.Background())
|
||||
go func(index int, backend BACKEND, ctx context.Context) {
|
||||
lookup := LOOKUP{index: index}
|
||||
if backend.Penalty != 0 && len(cbackends) > 1 {
|
||||
select {
|
||||
case <-time.After(backend.Penalty):
|
||||
case <-ctx.Done():
|
||||
sink <- lookup
|
||||
return
|
||||
}
|
||||
}
|
||||
lookup.Protocol = "http"
|
||||
if backend.Secure {
|
||||
lookup.Protocol = "https"
|
||||
}
|
||||
rpath := path
|
||||
if backend.Path != "" {
|
||||
rpath = backend.Path
|
||||
}
|
||||
if request, err := http.NewRequest(http.MethodHead, lookup.Protocol+"://"+backend.Host+rpath, nil); err == nil {
|
||||
request = request.WithContext(ctx)
|
||||
request.Header.Set("User-Agent", "whohas")
|
||||
if backend.Headers != nil {
|
||||
lookup.Headers = map[string]string{}
|
||||
for name, value := range backend.Headers {
|
||||
lookup.Headers[name] = value
|
||||
request.Header.Set(name, value)
|
||||
}
|
||||
}
|
||||
if response, err := http.DefaultClient.Do(request); err == nil {
|
||||
if response.StatusCode == 200 {
|
||||
lookup.Host = backend.Host
|
||||
lookup.Size, _ = strconv.ParseInt(response.Header.Get("Content-Length"), 10, 64)
|
||||
lookup.Mime = response.Header.Get("Content-Type")
|
||||
if lookup.Mime == "" || lookup.Mime == "application/octet-stream" || lookup.Mime == "text/plain" {
|
||||
if extension := filepath.Ext(path); extension != "" {
|
||||
lookup.Mime = mime.TypeByExtension(extension)
|
||||
}
|
||||
}
|
||||
if response.Header.Get("Accept-Ranges") != "" {
|
||||
lookup.Ranges = true
|
||||
}
|
||||
if header := response.Header.Get("Date"); header != "" {
|
||||
lookup.Date, _ = http.ParseTime(header)
|
||||
} else {
|
||||
lookup.Date = time.Now()
|
||||
}
|
||||
if header := response.Header.Get("Last-Modified"); header != "" {
|
||||
lookup.Modified, _ = http.ParseTime(header)
|
||||
}
|
||||
if header := response.Header.Get("Expires"); header != "" {
|
||||
lookup.Expires, _ = http.ParseTime(header)
|
||||
} else {
|
||||
lookup.Expires = lookup.Date.Add(time.Hour)
|
||||
}
|
||||
if lookup.Expires.Sub(lookup.Date) < 2*time.Second {
|
||||
lookup.Expires = lookup.Date.Add(2 * time.Second)
|
||||
}
|
||||
}
|
||||
response.Body.Close()
|
||||
}
|
||||
}
|
||||
sink <- lookup
|
||||
}(index, backend, ctx)
|
||||
}
|
||||
|
||||
for inflight > 0 {
|
||||
select {
|
||||
case result := <-sink:
|
||||
inflight--
|
||||
cancels[result.index] = nil
|
||||
if result.Host != "" {
|
||||
lookup = &result
|
||||
for index, cancel := range cancels {
|
||||
if cancels[index] != nil && index != result.index {
|
||||
cancel()
|
||||
cancels[index] = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
case <-time.After(timeout):
|
||||
for index, cancel := range cancels {
|
||||
if cancels[index] != nil {
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
close(sink)
|
||||
}
|
||||
|
||||
if cache != nil {
|
||||
now := time.Now()
|
||||
if cores > 1 {
|
||||
cache.Lock()
|
||||
}
|
||||
if cache.items == nil {
|
||||
cache.items = map[string]*LOOKUP{}
|
||||
}
|
||||
if now.Sub(cache.last) >= 5*time.Second {
|
||||
cache.last = now
|
||||
for key, item := range cache.items {
|
||||
if now.Sub(item.deadline) >= 0 {
|
||||
delete(cache.items, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
if lookup == nil || lookup.Host == "" {
|
||||
if ckey != "" {
|
||||
delete(cache.items, "k"+ckey)
|
||||
}
|
||||
if cache.items[cpath] == nil {
|
||||
cache.items[cpath] = &LOOKUP{deadline: now.Add(5 * time.Second)}
|
||||
}
|
||||
lookup = nil
|
||||
} else {
|
||||
if cache.TTL < 2*time.Second {
|
||||
cache.TTL = 2 * time.Second
|
||||
}
|
||||
if ckey != "" {
|
||||
cache.items["k"+ckey] = &LOOKUP{Host: lookup.Host, deadline: now.Add(cache.TTL)}
|
||||
}
|
||||
if cache.items[cpath] == nil {
|
||||
ttl := lookup.Expires.Sub(lookup.Date)
|
||||
if ttl < 2*time.Second {
|
||||
ttl = 2 * time.Second
|
||||
}
|
||||
if ttl > cache.TTL {
|
||||
ttl = cache.TTL
|
||||
}
|
||||
if ttl > 10*time.Minute {
|
||||
ttl = 10 * time.Minute
|
||||
}
|
||||
lookup.deadline = now.Add(ttl)
|
||||
cache.items[cpath] = lookup
|
||||
}
|
||||
}
|
||||
if cores > 1 {
|
||||
cache.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
Loading…
Reference in New Issue