Merge pull request #9538 from hashicorp/f-nomad-provider

provider/nomad: Nomad provider for managing jobs
This commit is contained in:
Mitchell Hashimoto
2016-11-09 18:34:55 -08:00
committed by GitHub
956 changed files with 177005 additions and 475 deletions

56
vendor/github.com/hashicorp/consul/lib/cluster.go generated vendored Normal file
View File

@@ -0,0 +1,56 @@
package lib
import (
"math/rand"
"time"
)
// DurationMinusBuffer returns a duration, minus a buffer and jitter
// subtracted from the duration. This function is used primarily for
// servicing Consul TTL Checks in advance of the TTL.
func DurationMinusBuffer(intv time.Duration, buffer time.Duration, jitter int64) time.Duration {
d := intv - buffer
if jitter == 0 {
d -= RandomStagger(d)
} else {
d -= RandomStagger(time.Duration(int64(d) / jitter))
}
return d
}
// DurationMinusBufferDomain returns the domain of valid durations from a
// call to DurationMinusBuffer. This function is used to check user
// specified input values to DurationMinusBuffer.
func DurationMinusBufferDomain(intv time.Duration, buffer time.Duration, jitter int64) (min time.Duration, max time.Duration) {
max = intv - buffer
if jitter == 0 {
min = max
} else {
min = max - time.Duration(int64(max)/jitter)
}
return min, max
}
// Returns a random stagger interval between 0 and the duration
func RandomStagger(intv time.Duration) time.Duration {
if intv == 0 {
return 0
}
return time.Duration(uint64(rand.Int63()) % uint64(intv))
}
// RateScaledInterval is used to choose an interval to perform an action in
// order to target an aggregate number of actions per second across the whole
// cluster.
func RateScaledInterval(rate float64, min time.Duration, n int) time.Duration {
const minRate = 1 / 86400 // 1/(1 * time.Day)
if rate <= minRate {
return min
}
interval := time.Duration(float64(time.Second) * float64(n) / rate)
if interval < min {
return min
}
return interval
}

22
vendor/github.com/hashicorp/consul/lib/math.go generated vendored Normal file
View File

@@ -0,0 +1,22 @@
package lib
func AbsInt(a int) int {
if a > 0 {
return a
}
return a * -1
}
func MaxInt(a, b int) int {
if a > b {
return a
}
return b
}
func MinInt(a, b int) int {
if a > b {
return b
}
return a
}

34
vendor/github.com/hashicorp/consul/lib/rand.go generated vendored Normal file
View File

@@ -0,0 +1,34 @@
package lib
import (
crand "crypto/rand"
"math"
"math/big"
"math/rand"
"sync"
"time"
)
var (
once sync.Once
// SeededSecurely is set to true if a cryptographically secure seed
// was used to initialize rand. When false, the start time is used
// as a seed.
SeededSecurely bool
)
// SeedMathRand provides weak, but guaranteed seeding, which is better than
// running with Go's default seed of 1. A call to SeedMathRand() is expected
// to be called via init(), but never a second time.
func SeedMathRand() {
once.Do(func() {
n, err := crand.Int(crand.Reader, big.NewInt(math.MaxInt64))
if err != nil {
rand.Seed(time.Now().UTC().UnixNano())
return
}
rand.Seed(n.Int64())
SeededSecurely = true
})
}

11
vendor/github.com/hashicorp/consul/lib/string.go generated vendored Normal file
View File

@@ -0,0 +1,11 @@
package lib
// StrContains checks if a list contains a string
func StrContains(l []string, s string) bool {
for _, v := range l {
if v == s {
return true
}
}
return false
}

25
vendor/github.com/hashicorp/go-msgpack/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,25 @@
Copyright (c) 2012, 2013 Ugorji Nwoke.
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name of the author nor the names of its contributors may be used
to endorse or promote products derived from this software
without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

143
vendor/github.com/hashicorp/go-msgpack/codec/0doc.go generated vendored Normal file
View File

@@ -0,0 +1,143 @@
// Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved.
// Use of this source code is governed by a BSD-style license found in the LICENSE file.
/*
High Performance, Feature-Rich Idiomatic Go encoding library for msgpack and binc .
Supported Serialization formats are:
- msgpack: [https://github.com/msgpack/msgpack]
- binc: [http://github.com/ugorji/binc]
To install:
go get github.com/ugorji/go/codec
The idiomatic Go support is as seen in other encoding packages in
the standard library (ie json, xml, gob, etc).
Rich Feature Set includes:
- Simple but extremely powerful and feature-rich API
- Very High Performance.
Our extensive benchmarks show us outperforming Gob, Json and Bson by 2-4X.
This was achieved by taking extreme care on:
- managing allocation
- function frame size (important due to Go's use of split stacks),
- reflection use (and by-passing reflection for common types)
- recursion implications
- zero-copy mode (encoding/decoding to byte slice without using temp buffers)
- Correct.
Care was taken to precisely handle corner cases like:
overflows, nil maps and slices, nil value in stream, etc.
- Efficient zero-copying into temporary byte buffers
when encoding into or decoding from a byte slice.
- Standard field renaming via tags
- Encoding from any value
(struct, slice, map, primitives, pointers, interface{}, etc)
- Decoding into pointer to any non-nil typed value
(struct, slice, map, int, float32, bool, string, reflect.Value, etc)
- Supports extension functions to handle the encode/decode of custom types
- Support Go 1.2 encoding.BinaryMarshaler/BinaryUnmarshaler
- Schema-less decoding
(decode into a pointer to a nil interface{} as opposed to a typed non-nil value).
Includes Options to configure what specific map or slice type to use
when decoding an encoded list or map into a nil interface{}
- Provides a RPC Server and Client Codec for net/rpc communication protocol.
- Msgpack Specific:
- Provides extension functions to handle spec-defined extensions (binary, timestamp)
- Options to resolve ambiguities in handling raw bytes (as string or []byte)
during schema-less decoding (decoding into a nil interface{})
- RPC Server/Client Codec for msgpack-rpc protocol defined at:
https://github.com/msgpack-rpc/msgpack-rpc/blob/master/spec.md
- Fast Paths for some container types:
For some container types, we circumvent reflection and its associated overhead
and allocation costs, and encode/decode directly. These types are:
[]interface{}
[]int
[]string
map[interface{}]interface{}
map[int]interface{}
map[string]interface{}
Extension Support
Users can register a function to handle the encoding or decoding of
their custom types.
There are no restrictions on what the custom type can be. Some examples:
type BisSet []int
type BitSet64 uint64
type UUID string
type MyStructWithUnexportedFields struct { a int; b bool; c []int; }
type GifImage struct { ... }
As an illustration, MyStructWithUnexportedFields would normally be
encoded as an empty map because it has no exported fields, while UUID
would be encoded as a string. However, with extension support, you can
encode any of these however you like.
RPC
RPC Client and Server Codecs are implemented, so the codecs can be used
with the standard net/rpc package.
Usage
Typical usage model:
// create and configure Handle
var (
bh codec.BincHandle
mh codec.MsgpackHandle
)
mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
// configure extensions
// e.g. for msgpack, define functions and enable Time support for tag 1
// mh.AddExt(reflect.TypeOf(time.Time{}), 1, myMsgpackTimeEncodeExtFn, myMsgpackTimeDecodeExtFn)
// create and use decoder/encoder
var (
r io.Reader
w io.Writer
b []byte
h = &bh // or mh to use msgpack
)
dec = codec.NewDecoder(r, h)
dec = codec.NewDecoderBytes(b, h)
err = dec.Decode(&v)
enc = codec.NewEncoder(w, h)
enc = codec.NewEncoderBytes(&b, h)
err = enc.Encode(v)
//RPC Server
go func() {
for {
conn, err := listener.Accept()
rpcCodec := codec.GoRpc.ServerCodec(conn, h)
//OR rpcCodec := codec.MsgpackSpecRpc.ServerCodec(conn, h)
rpc.ServeCodec(rpcCodec)
}
}()
//RPC Communication (client side)
conn, err = net.Dial("tcp", "localhost:5555")
rpcCodec := codec.GoRpc.ClientCodec(conn, h)
//OR rpcCodec := codec.MsgpackSpecRpc.ClientCodec(conn, h)
client := rpc.NewClientWithCodec(rpcCodec)
Representative Benchmark Results
Run the benchmark suite using:
go test -bi -bench=. -benchmem
To run full benchmark suite (including against vmsgpack and bson),
see notes in ext_dep_test.go
*/
package codec

174
vendor/github.com/hashicorp/go-msgpack/codec/README.md generated vendored Normal file
View File

@@ -0,0 +1,174 @@
# Codec
High Performance and Feature-Rich Idiomatic Go Library providing
encode/decode support for different serialization formats.
Supported Serialization formats are:
- msgpack: [https://github.com/msgpack/msgpack]
- binc: [http://github.com/ugorji/binc]
To install:
go get github.com/ugorji/go/codec
Online documentation: [http://godoc.org/github.com/ugorji/go/codec]
The idiomatic Go support is as seen in other encoding packages in
the standard library (ie json, xml, gob, etc).
Rich Feature Set includes:
- Simple but extremely powerful and feature-rich API
- Very High Performance.
Our extensive benchmarks show us outperforming Gob, Json and Bson by 2-4X.
This was achieved by taking extreme care on:
- managing allocation
- function frame size (important due to Go's use of split stacks),
- reflection use (and by-passing reflection for common types)
- recursion implications
- zero-copy mode (encoding/decoding to byte slice without using temp buffers)
- Correct.
Care was taken to precisely handle corner cases like:
overflows, nil maps and slices, nil value in stream, etc.
- Efficient zero-copying into temporary byte buffers
when encoding into or decoding from a byte slice.
- Standard field renaming via tags
- Encoding from any value
(struct, slice, map, primitives, pointers, interface{}, etc)
- Decoding into pointer to any non-nil typed value
(struct, slice, map, int, float32, bool, string, reflect.Value, etc)
- Supports extension functions to handle the encode/decode of custom types
- Support Go 1.2 encoding.BinaryMarshaler/BinaryUnmarshaler
- Schema-less decoding
(decode into a pointer to a nil interface{} as opposed to a typed non-nil value).
Includes Options to configure what specific map or slice type to use
when decoding an encoded list or map into a nil interface{}
- Provides a RPC Server and Client Codec for net/rpc communication protocol.
- Msgpack Specific:
- Provides extension functions to handle spec-defined extensions (binary, timestamp)
- Options to resolve ambiguities in handling raw bytes (as string or []byte)
during schema-less decoding (decoding into a nil interface{})
- RPC Server/Client Codec for msgpack-rpc protocol defined at:
https://github.com/msgpack-rpc/msgpack-rpc/blob/master/spec.md
- Fast Paths for some container types:
For some container types, we circumvent reflection and its associated overhead
and allocation costs, and encode/decode directly. These types are:
[]interface{}
[]int
[]string
map[interface{}]interface{}
map[int]interface{}
map[string]interface{}
## Extension Support
Users can register a function to handle the encoding or decoding of
their custom types.
There are no restrictions on what the custom type can be. Some examples:
type BisSet []int
type BitSet64 uint64
type UUID string
type MyStructWithUnexportedFields struct { a int; b bool; c []int; }
type GifImage struct { ... }
As an illustration, MyStructWithUnexportedFields would normally be
encoded as an empty map because it has no exported fields, while UUID
would be encoded as a string. However, with extension support, you can
encode any of these however you like.
## RPC
RPC Client and Server Codecs are implemented, so the codecs can be used
with the standard net/rpc package.
## Usage
Typical usage model:
// create and configure Handle
var (
bh codec.BincHandle
mh codec.MsgpackHandle
)
mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
// configure extensions
// e.g. for msgpack, define functions and enable Time support for tag 1
// mh.AddExt(reflect.TypeOf(time.Time{}), 1, myMsgpackTimeEncodeExtFn, myMsgpackTimeDecodeExtFn)
// create and use decoder/encoder
var (
r io.Reader
w io.Writer
b []byte
h = &bh // or mh to use msgpack
)
dec = codec.NewDecoder(r, h)
dec = codec.NewDecoderBytes(b, h)
err = dec.Decode(&v)
enc = codec.NewEncoder(w, h)
enc = codec.NewEncoderBytes(&b, h)
err = enc.Encode(v)
//RPC Server
go func() {
for {
conn, err := listener.Accept()
rpcCodec := codec.GoRpc.ServerCodec(conn, h)
//OR rpcCodec := codec.MsgpackSpecRpc.ServerCodec(conn, h)
rpc.ServeCodec(rpcCodec)
}
}()
//RPC Communication (client side)
conn, err = net.Dial("tcp", "localhost:5555")
rpcCodec := codec.GoRpc.ClientCodec(conn, h)
//OR rpcCodec := codec.MsgpackSpecRpc.ClientCodec(conn, h)
client := rpc.NewClientWithCodec(rpcCodec)
## Representative Benchmark Results
A sample run of benchmark using "go test -bi -bench=. -benchmem":
/proc/cpuinfo: Intel(R) Core(TM) i7-2630QM CPU @ 2.00GHz (HT)
..............................................
BENCHMARK INIT: 2013-10-16 11:02:50.345970786 -0400 EDT
To run full benchmark comparing encodings (MsgPack, Binc, JSON, GOB, etc), use: "go test -bench=."
Benchmark:
Struct recursive Depth: 1
ApproxDeepSize Of benchmark Struct: 4694 bytes
Benchmark One-Pass Run:
v-msgpack: len: 1600 bytes
bson: len: 3025 bytes
msgpack: len: 1560 bytes
binc: len: 1187 bytes
gob: len: 1972 bytes
json: len: 2538 bytes
..............................................
PASS
Benchmark__Msgpack____Encode 50000 54359 ns/op 14953 B/op 83 allocs/op
Benchmark__Msgpack____Decode 10000 106531 ns/op 14990 B/op 410 allocs/op
Benchmark__Binc_NoSym_Encode 50000 53956 ns/op 14966 B/op 83 allocs/op
Benchmark__Binc_NoSym_Decode 10000 103751 ns/op 14529 B/op 386 allocs/op
Benchmark__Binc_Sym___Encode 50000 65961 ns/op 17130 B/op 88 allocs/op
Benchmark__Binc_Sym___Decode 10000 106310 ns/op 15857 B/op 287 allocs/op
Benchmark__Gob________Encode 10000 135944 ns/op 21189 B/op 237 allocs/op
Benchmark__Gob________Decode 5000 405390 ns/op 83460 B/op 1841 allocs/op
Benchmark__Json_______Encode 20000 79412 ns/op 13874 B/op 102 allocs/op
Benchmark__Json_______Decode 10000 247979 ns/op 14202 B/op 493 allocs/op
Benchmark__Bson_______Encode 10000 121762 ns/op 27814 B/op 514 allocs/op
Benchmark__Bson_______Decode 10000 162126 ns/op 16514 B/op 789 allocs/op
Benchmark__VMsgpack___Encode 50000 69155 ns/op 12370 B/op 344 allocs/op
Benchmark__VMsgpack___Decode 10000 151609 ns/op 20307 B/op 571 allocs/op
ok ugorji.net/codec 30.827s
To run full benchmark suite (including against vmsgpack and bson),
see notes in ext\_dep\_test.go

786
vendor/github.com/hashicorp/go-msgpack/codec/binc.go generated vendored Normal file
View File

@@ -0,0 +1,786 @@
// Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved.
// Use of this source code is governed by a BSD-style license found in the LICENSE file.
package codec
import (
"math"
// "reflect"
// "sync/atomic"
"time"
//"fmt"
)
const bincDoPrune = true // No longer needed. Needed before as C lib did not support pruning.
//var _ = fmt.Printf
// vd as low 4 bits (there are 16 slots)
const (
bincVdSpecial byte = iota
bincVdPosInt
bincVdNegInt
bincVdFloat
bincVdString
bincVdByteArray
bincVdArray
bincVdMap
bincVdTimestamp
bincVdSmallInt
bincVdUnicodeOther
bincVdSymbol
bincVdDecimal
_ // open slot
_ // open slot
bincVdCustomExt = 0x0f
)
const (
bincSpNil byte = iota
bincSpFalse
bincSpTrue
bincSpNan
bincSpPosInf
bincSpNegInf
bincSpZeroFloat
bincSpZero
bincSpNegOne
)
const (
bincFlBin16 byte = iota
bincFlBin32
_ // bincFlBin32e
bincFlBin64
_ // bincFlBin64e
// others not currently supported
)
type bincEncDriver struct {
w encWriter
m map[string]uint16 // symbols
s uint32 // symbols sequencer
b [8]byte
}
func (e *bincEncDriver) isBuiltinType(rt uintptr) bool {
return rt == timeTypId
}
func (e *bincEncDriver) encodeBuiltin(rt uintptr, v interface{}) {
switch rt {
case timeTypId:
bs := encodeTime(v.(time.Time))
e.w.writen1(bincVdTimestamp<<4 | uint8(len(bs)))
e.w.writeb(bs)
}
}
func (e *bincEncDriver) encodeNil() {
e.w.writen1(bincVdSpecial<<4 | bincSpNil)
}
func (e *bincEncDriver) encodeBool(b bool) {
if b {
e.w.writen1(bincVdSpecial<<4 | bincSpTrue)
} else {
e.w.writen1(bincVdSpecial<<4 | bincSpFalse)
}
}
func (e *bincEncDriver) encodeFloat32(f float32) {
if f == 0 {
e.w.writen1(bincVdSpecial<<4 | bincSpZeroFloat)
return
}
e.w.writen1(bincVdFloat<<4 | bincFlBin32)
e.w.writeUint32(math.Float32bits(f))
}
func (e *bincEncDriver) encodeFloat64(f float64) {
if f == 0 {
e.w.writen1(bincVdSpecial<<4 | bincSpZeroFloat)
return
}
bigen.PutUint64(e.b[:], math.Float64bits(f))
if bincDoPrune {
i := 7
for ; i >= 0 && (e.b[i] == 0); i-- {
}
i++
if i <= 6 {
e.w.writen1(bincVdFloat<<4 | 0x8 | bincFlBin64)
e.w.writen1(byte(i))
e.w.writeb(e.b[:i])
return
}
}
e.w.writen1(bincVdFloat<<4 | bincFlBin64)
e.w.writeb(e.b[:])
}
func (e *bincEncDriver) encIntegerPrune(bd byte, pos bool, v uint64, lim uint8) {
if lim == 4 {
bigen.PutUint32(e.b[:lim], uint32(v))
} else {
bigen.PutUint64(e.b[:lim], v)
}
if bincDoPrune {
i := pruneSignExt(e.b[:lim], pos)
e.w.writen1(bd | lim - 1 - byte(i))
e.w.writeb(e.b[i:lim])
} else {
e.w.writen1(bd | lim - 1)
e.w.writeb(e.b[:lim])
}
}
func (e *bincEncDriver) encodeInt(v int64) {
const nbd byte = bincVdNegInt << 4
switch {
case v >= 0:
e.encUint(bincVdPosInt<<4, true, uint64(v))
case v == -1:
e.w.writen1(bincVdSpecial<<4 | bincSpNegOne)
default:
e.encUint(bincVdNegInt<<4, false, uint64(-v))
}
}
func (e *bincEncDriver) encodeUint(v uint64) {
e.encUint(bincVdPosInt<<4, true, v)
}
func (e *bincEncDriver) encUint(bd byte, pos bool, v uint64) {
switch {
case v == 0:
e.w.writen1(bincVdSpecial<<4 | bincSpZero)
case pos && v >= 1 && v <= 16:
e.w.writen1(bincVdSmallInt<<4 | byte(v-1))
case v <= math.MaxUint8:
e.w.writen2(bd|0x0, byte(v))
case v <= math.MaxUint16:
e.w.writen1(bd | 0x01)
e.w.writeUint16(uint16(v))
case v <= math.MaxUint32:
e.encIntegerPrune(bd, pos, v, 4)
default:
e.encIntegerPrune(bd, pos, v, 8)
}
}
func (e *bincEncDriver) encodeExtPreamble(xtag byte, length int) {
e.encLen(bincVdCustomExt<<4, uint64(length))
e.w.writen1(xtag)
}
func (e *bincEncDriver) encodeArrayPreamble(length int) {
e.encLen(bincVdArray<<4, uint64(length))
}
func (e *bincEncDriver) encodeMapPreamble(length int) {
e.encLen(bincVdMap<<4, uint64(length))
}
func (e *bincEncDriver) encodeString(c charEncoding, v string) {
l := uint64(len(v))
e.encBytesLen(c, l)
if l > 0 {
e.w.writestr(v)
}
}
func (e *bincEncDriver) encodeSymbol(v string) {
// if WriteSymbolsNoRefs {
// e.encodeString(c_UTF8, v)
// return
// }
//symbols only offer benefit when string length > 1.
//This is because strings with length 1 take only 2 bytes to store
//(bd with embedded length, and single byte for string val).
l := len(v)
switch l {
case 0:
e.encBytesLen(c_UTF8, 0)
return
case 1:
e.encBytesLen(c_UTF8, 1)
e.w.writen1(v[0])
return
}
if e.m == nil {
e.m = make(map[string]uint16, 16)
}
ui, ok := e.m[v]
if ok {
if ui <= math.MaxUint8 {
e.w.writen2(bincVdSymbol<<4, byte(ui))
} else {
e.w.writen1(bincVdSymbol<<4 | 0x8)
e.w.writeUint16(ui)
}
} else {
e.s++
ui = uint16(e.s)
//ui = uint16(atomic.AddUint32(&e.s, 1))
e.m[v] = ui
var lenprec uint8
switch {
case l <= math.MaxUint8:
// lenprec = 0
case l <= math.MaxUint16:
lenprec = 1
case int64(l) <= math.MaxUint32:
lenprec = 2
default:
lenprec = 3
}
if ui <= math.MaxUint8 {
e.w.writen2(bincVdSymbol<<4|0x0|0x4|lenprec, byte(ui))
} else {
e.w.writen1(bincVdSymbol<<4 | 0x8 | 0x4 | lenprec)
e.w.writeUint16(ui)
}
switch lenprec {
case 0:
e.w.writen1(byte(l))
case 1:
e.w.writeUint16(uint16(l))
case 2:
e.w.writeUint32(uint32(l))
default:
e.w.writeUint64(uint64(l))
}
e.w.writestr(v)
}
}
func (e *bincEncDriver) encodeStringBytes(c charEncoding, v []byte) {
l := uint64(len(v))
e.encBytesLen(c, l)
if l > 0 {
e.w.writeb(v)
}
}
func (e *bincEncDriver) encBytesLen(c charEncoding, length uint64) {
//TODO: support bincUnicodeOther (for now, just use string or bytearray)
if c == c_RAW {
e.encLen(bincVdByteArray<<4, length)
} else {
e.encLen(bincVdString<<4, length)
}
}
func (e *bincEncDriver) encLen(bd byte, l uint64) {
if l < 12 {
e.w.writen1(bd | uint8(l+4))
} else {
e.encLenNumber(bd, l)
}
}
func (e *bincEncDriver) encLenNumber(bd byte, v uint64) {
switch {
case v <= math.MaxUint8:
e.w.writen2(bd, byte(v))
case v <= math.MaxUint16:
e.w.writen1(bd | 0x01)
e.w.writeUint16(uint16(v))
case v <= math.MaxUint32:
e.w.writen1(bd | 0x02)
e.w.writeUint32(uint32(v))
default:
e.w.writen1(bd | 0x03)
e.w.writeUint64(uint64(v))
}
}
//------------------------------------
type bincDecDriver struct {
r decReader
bdRead bool
bdType valueType
bd byte
vd byte
vs byte
b [8]byte
m map[uint32]string // symbols (use uint32 as key, as map optimizes for it)
}
func (d *bincDecDriver) initReadNext() {
if d.bdRead {
return
}
d.bd = d.r.readn1()
d.vd = d.bd >> 4
d.vs = d.bd & 0x0f
d.bdRead = true
d.bdType = valueTypeUnset
}
func (d *bincDecDriver) currentEncodedType() valueType {
if d.bdType == valueTypeUnset {
switch d.vd {
case bincVdSpecial:
switch d.vs {
case bincSpNil:
d.bdType = valueTypeNil
case bincSpFalse, bincSpTrue:
d.bdType = valueTypeBool
case bincSpNan, bincSpNegInf, bincSpPosInf, bincSpZeroFloat:
d.bdType = valueTypeFloat
case bincSpZero:
d.bdType = valueTypeUint
case bincSpNegOne:
d.bdType = valueTypeInt
default:
decErr("currentEncodedType: Unrecognized special value 0x%x", d.vs)
}
case bincVdSmallInt:
d.bdType = valueTypeUint
case bincVdPosInt:
d.bdType = valueTypeUint
case bincVdNegInt:
d.bdType = valueTypeInt
case bincVdFloat:
d.bdType = valueTypeFloat
case bincVdString:
d.bdType = valueTypeString
case bincVdSymbol:
d.bdType = valueTypeSymbol
case bincVdByteArray:
d.bdType = valueTypeBytes
case bincVdTimestamp:
d.bdType = valueTypeTimestamp
case bincVdCustomExt:
d.bdType = valueTypeExt
case bincVdArray:
d.bdType = valueTypeArray
case bincVdMap:
d.bdType = valueTypeMap
default:
decErr("currentEncodedType: Unrecognized d.vd: 0x%x", d.vd)
}
}
return d.bdType
}
func (d *bincDecDriver) tryDecodeAsNil() bool {
if d.bd == bincVdSpecial<<4|bincSpNil {
d.bdRead = false
return true
}
return false
}
func (d *bincDecDriver) isBuiltinType(rt uintptr) bool {
return rt == timeTypId
}
func (d *bincDecDriver) decodeBuiltin(rt uintptr, v interface{}) {
switch rt {
case timeTypId:
if d.vd != bincVdTimestamp {
decErr("Invalid d.vd. Expecting 0x%x. Received: 0x%x", bincVdTimestamp, d.vd)
}
tt, err := decodeTime(d.r.readn(int(d.vs)))
if err != nil {
panic(err)
}
var vt *time.Time = v.(*time.Time)
*vt = tt
d.bdRead = false
}
}
func (d *bincDecDriver) decFloatPre(vs, defaultLen byte) {
if vs&0x8 == 0 {
d.r.readb(d.b[0:defaultLen])
} else {
l := d.r.readn1()
if l > 8 {
decErr("At most 8 bytes used to represent float. Received: %v bytes", l)
}
for i := l; i < 8; i++ {
d.b[i] = 0
}
d.r.readb(d.b[0:l])
}
}
func (d *bincDecDriver) decFloat() (f float64) {
//if true { f = math.Float64frombits(d.r.readUint64()); break; }
switch vs := d.vs; vs & 0x7 {
case bincFlBin32:
d.decFloatPre(vs, 4)
f = float64(math.Float32frombits(bigen.Uint32(d.b[0:4])))
case bincFlBin64:
d.decFloatPre(vs, 8)
f = math.Float64frombits(bigen.Uint64(d.b[0:8]))
default:
decErr("only float32 and float64 are supported. d.vd: 0x%x, d.vs: 0x%x", d.vd, d.vs)
}
return
}
func (d *bincDecDriver) decUint() (v uint64) {
// need to inline the code (interface conversion and type assertion expensive)
switch d.vs {
case 0:
v = uint64(d.r.readn1())
case 1:
d.r.readb(d.b[6:])
v = uint64(bigen.Uint16(d.b[6:]))
case 2:
d.b[4] = 0
d.r.readb(d.b[5:])
v = uint64(bigen.Uint32(d.b[4:]))
case 3:
d.r.readb(d.b[4:])
v = uint64(bigen.Uint32(d.b[4:]))
case 4, 5, 6:
lim := int(7 - d.vs)
d.r.readb(d.b[lim:])
for i := 0; i < lim; i++ {
d.b[i] = 0
}
v = uint64(bigen.Uint64(d.b[:]))
case 7:
d.r.readb(d.b[:])
v = uint64(bigen.Uint64(d.b[:]))
default:
decErr("unsigned integers with greater than 64 bits of precision not supported")
}
return
}
func (d *bincDecDriver) decIntAny() (ui uint64, i int64, neg bool) {
switch d.vd {
case bincVdPosInt:
ui = d.decUint()
i = int64(ui)
case bincVdNegInt:
ui = d.decUint()
i = -(int64(ui))
neg = true
case bincVdSmallInt:
i = int64(d.vs) + 1
ui = uint64(d.vs) + 1
case bincVdSpecial:
switch d.vs {
case bincSpZero:
//i = 0
case bincSpNegOne:
neg = true
ui = 1
i = -1
default:
decErr("numeric decode fails for special value: d.vs: 0x%x", d.vs)
}
default:
decErr("number can only be decoded from uint or int values. d.bd: 0x%x, d.vd: 0x%x", d.bd, d.vd)
}
return
}
func (d *bincDecDriver) decodeInt(bitsize uint8) (i int64) {
_, i, _ = d.decIntAny()
checkOverflow(0, i, bitsize)
d.bdRead = false
return
}
func (d *bincDecDriver) decodeUint(bitsize uint8) (ui uint64) {
ui, i, neg := d.decIntAny()
if neg {
decErr("Assigning negative signed value: %v, to unsigned type", i)
}
checkOverflow(ui, 0, bitsize)
d.bdRead = false
return
}
func (d *bincDecDriver) decodeFloat(chkOverflow32 bool) (f float64) {
switch d.vd {
case bincVdSpecial:
d.bdRead = false
switch d.vs {
case bincSpNan:
return math.NaN()
case bincSpPosInf:
return math.Inf(1)
case bincSpZeroFloat, bincSpZero:
return
case bincSpNegInf:
return math.Inf(-1)
default:
decErr("Invalid d.vs decoding float where d.vd=bincVdSpecial: %v", d.vs)
}
case bincVdFloat:
f = d.decFloat()
default:
_, i, _ := d.decIntAny()
f = float64(i)
}
checkOverflowFloat32(f, chkOverflow32)
d.bdRead = false
return
}
// bool can be decoded from bool only (single byte).
func (d *bincDecDriver) decodeBool() (b bool) {
switch d.bd {
case (bincVdSpecial | bincSpFalse):
// b = false
case (bincVdSpecial | bincSpTrue):
b = true
default:
decErr("Invalid single-byte value for bool: %s: %x", msgBadDesc, d.bd)
}
d.bdRead = false
return
}
func (d *bincDecDriver) readMapLen() (length int) {
if d.vd != bincVdMap {
decErr("Invalid d.vd for map. Expecting 0x%x. Got: 0x%x", bincVdMap, d.vd)
}
length = d.decLen()
d.bdRead = false
return
}
func (d *bincDecDriver) readArrayLen() (length int) {
if d.vd != bincVdArray {
decErr("Invalid d.vd for array. Expecting 0x%x. Got: 0x%x", bincVdArray, d.vd)
}
length = d.decLen()
d.bdRead = false
return
}
func (d *bincDecDriver) decLen() int {
if d.vs <= 3 {
return int(d.decUint())
}
return int(d.vs - 4)
}
func (d *bincDecDriver) decodeString() (s string) {
switch d.vd {
case bincVdString, bincVdByteArray:
if length := d.decLen(); length > 0 {
s = string(d.r.readn(length))
}
case bincVdSymbol:
//from vs: extract numSymbolBytes, containsStringVal, strLenPrecision,
//extract symbol
//if containsStringVal, read it and put in map
//else look in map for string value
var symbol uint32
vs := d.vs
//fmt.Printf(">>>> d.vs: 0b%b, & 0x8: %v, & 0x4: %v\n", d.vs, vs & 0x8, vs & 0x4)
if vs&0x8 == 0 {
symbol = uint32(d.r.readn1())
} else {
symbol = uint32(d.r.readUint16())
}
if d.m == nil {
d.m = make(map[uint32]string, 16)
}
if vs&0x4 == 0 {
s = d.m[symbol]
} else {
var slen int
switch vs & 0x3 {
case 0:
slen = int(d.r.readn1())
case 1:
slen = int(d.r.readUint16())
case 2:
slen = int(d.r.readUint32())
case 3:
slen = int(d.r.readUint64())
}
s = string(d.r.readn(slen))
d.m[symbol] = s
}
default:
decErr("Invalid d.vd for string. Expecting string:0x%x, bytearray:0x%x or symbol: 0x%x. Got: 0x%x",
bincVdString, bincVdByteArray, bincVdSymbol, d.vd)
}
d.bdRead = false
return
}
func (d *bincDecDriver) decodeBytes(bs []byte) (bsOut []byte, changed bool) {
var clen int
switch d.vd {
case bincVdString, bincVdByteArray:
clen = d.decLen()
default:
decErr("Invalid d.vd for bytes. Expecting string:0x%x or bytearray:0x%x. Got: 0x%x",
bincVdString, bincVdByteArray, d.vd)
}
if clen > 0 {
// if no contents in stream, don't update the passed byteslice
if len(bs) != clen {
if len(bs) > clen {
bs = bs[:clen]
} else {
bs = make([]byte, clen)
}
bsOut = bs
changed = true
}
d.r.readb(bs)
}
d.bdRead = false
return
}
func (d *bincDecDriver) decodeExt(verifyTag bool, tag byte) (xtag byte, xbs []byte) {
switch d.vd {
case bincVdCustomExt:
l := d.decLen()
xtag = d.r.readn1()
if verifyTag && xtag != tag {
decErr("Wrong extension tag. Got %b. Expecting: %v", xtag, tag)
}
xbs = d.r.readn(l)
case bincVdByteArray:
xbs, _ = d.decodeBytes(nil)
default:
decErr("Invalid d.vd for extensions (Expecting extensions or byte array). Got: 0x%x", d.vd)
}
d.bdRead = false
return
}
func (d *bincDecDriver) decodeNaked() (v interface{}, vt valueType, decodeFurther bool) {
d.initReadNext()
switch d.vd {
case bincVdSpecial:
switch d.vs {
case bincSpNil:
vt = valueTypeNil
case bincSpFalse:
vt = valueTypeBool
v = false
case bincSpTrue:
vt = valueTypeBool
v = true
case bincSpNan:
vt = valueTypeFloat
v = math.NaN()
case bincSpPosInf:
vt = valueTypeFloat
v = math.Inf(1)
case bincSpNegInf:
vt = valueTypeFloat
v = math.Inf(-1)
case bincSpZeroFloat:
vt = valueTypeFloat
v = float64(0)
case bincSpZero:
vt = valueTypeUint
v = int64(0) // int8(0)
case bincSpNegOne:
vt = valueTypeInt
v = int64(-1) // int8(-1)
default:
decErr("decodeNaked: Unrecognized special value 0x%x", d.vs)
}
case bincVdSmallInt:
vt = valueTypeUint
v = uint64(int8(d.vs)) + 1 // int8(d.vs) + 1
case bincVdPosInt:
vt = valueTypeUint
v = d.decUint()
case bincVdNegInt:
vt = valueTypeInt
v = -(int64(d.decUint()))
case bincVdFloat:
vt = valueTypeFloat
v = d.decFloat()
case bincVdSymbol:
vt = valueTypeSymbol
v = d.decodeString()
case bincVdString:
vt = valueTypeString
v = d.decodeString()
case bincVdByteArray:
vt = valueTypeBytes
v, _ = d.decodeBytes(nil)
case bincVdTimestamp:
vt = valueTypeTimestamp
tt, err := decodeTime(d.r.readn(int(d.vs)))
if err != nil {
panic(err)
}
v = tt
case bincVdCustomExt:
vt = valueTypeExt
l := d.decLen()
var re RawExt
re.Tag = d.r.readn1()
re.Data = d.r.readn(l)
v = &re
vt = valueTypeExt
case bincVdArray:
vt = valueTypeArray
decodeFurther = true
case bincVdMap:
vt = valueTypeMap
decodeFurther = true
default:
decErr("decodeNaked: Unrecognized d.vd: 0x%x", d.vd)
}
if !decodeFurther {
d.bdRead = false
}
return
}
//------------------------------------
//BincHandle is a Handle for the Binc Schema-Free Encoding Format
//defined at https://github.com/ugorji/binc .
//
//BincHandle currently supports all Binc features with the following EXCEPTIONS:
// - only integers up to 64 bits of precision are supported.
// big integers are unsupported.
// - Only IEEE 754 binary32 and binary64 floats are supported (ie Go float32 and float64 types).
// extended precision and decimal IEEE 754 floats are unsupported.
// - Only UTF-8 strings supported.
// Unicode_Other Binc types (UTF16, UTF32) are currently unsupported.
//Note that these EXCEPTIONS are temporary and full support is possible and may happen soon.
type BincHandle struct {
BasicHandle
}
func (h *BincHandle) newEncDriver(w encWriter) encDriver {
return &bincEncDriver{w: w}
}
func (h *BincHandle) newDecDriver(r decReader) decDriver {
return &bincDecDriver{r: r}
}
func (_ *BincHandle) writeExt() bool {
return true
}
func (h *BincHandle) getBasicHandle() *BasicHandle {
return &h.BasicHandle
}

1048
vendor/github.com/hashicorp/go-msgpack/codec/decode.go generated vendored Normal file

File diff suppressed because it is too large Load Diff

1001
vendor/github.com/hashicorp/go-msgpack/codec/encode.go generated vendored Normal file

File diff suppressed because it is too large Load Diff

589
vendor/github.com/hashicorp/go-msgpack/codec/helper.go generated vendored Normal file
View File

@@ -0,0 +1,589 @@
// Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved.
// Use of this source code is governed by a BSD-style license found in the LICENSE file.
package codec
// Contains code shared by both encode and decode.
import (
"encoding/binary"
"fmt"
"math"
"reflect"
"sort"
"strings"
"sync"
"time"
"unicode"
"unicode/utf8"
)
const (
structTagName = "codec"
// Support
// encoding.BinaryMarshaler: MarshalBinary() (data []byte, err error)
// encoding.BinaryUnmarshaler: UnmarshalBinary(data []byte) error
// This constant flag will enable or disable it.
supportBinaryMarshal = true
// Each Encoder or Decoder uses a cache of functions based on conditionals,
// so that the conditionals are not run every time.
//
// Either a map or a slice is used to keep track of the functions.
// The map is more natural, but has a higher cost than a slice/array.
// This flag (useMapForCodecCache) controls which is used.
useMapForCodecCache = false
// For some common container types, we can short-circuit an elaborate
// reflection dance and call encode/decode directly.
// The currently supported types are:
// - slices of strings, or id's (int64,uint64) or interfaces.
// - maps of str->str, str->intf, id(int64,uint64)->intf, intf->intf
shortCircuitReflectToFastPath = true
// for debugging, set this to false, to catch panic traces.
// Note that this will always cause rpc tests to fail, since they need io.EOF sent via panic.
recoverPanicToErr = true
)
type charEncoding uint8
const (
c_RAW charEncoding = iota
c_UTF8
c_UTF16LE
c_UTF16BE
c_UTF32LE
c_UTF32BE
)
// valueType is the stream type
type valueType uint8
const (
valueTypeUnset valueType = iota
valueTypeNil
valueTypeInt
valueTypeUint
valueTypeFloat
valueTypeBool
valueTypeString
valueTypeSymbol
valueTypeBytes
valueTypeMap
valueTypeArray
valueTypeTimestamp
valueTypeExt
valueTypeInvalid = 0xff
)
var (
bigen = binary.BigEndian
structInfoFieldName = "_struct"
cachedTypeInfo = make(map[uintptr]*typeInfo, 4)
cachedTypeInfoMutex sync.RWMutex
intfSliceTyp = reflect.TypeOf([]interface{}(nil))
intfTyp = intfSliceTyp.Elem()
strSliceTyp = reflect.TypeOf([]string(nil))
boolSliceTyp = reflect.TypeOf([]bool(nil))
uintSliceTyp = reflect.TypeOf([]uint(nil))
uint8SliceTyp = reflect.TypeOf([]uint8(nil))
uint16SliceTyp = reflect.TypeOf([]uint16(nil))
uint32SliceTyp = reflect.TypeOf([]uint32(nil))
uint64SliceTyp = reflect.TypeOf([]uint64(nil))
intSliceTyp = reflect.TypeOf([]int(nil))
int8SliceTyp = reflect.TypeOf([]int8(nil))
int16SliceTyp = reflect.TypeOf([]int16(nil))
int32SliceTyp = reflect.TypeOf([]int32(nil))
int64SliceTyp = reflect.TypeOf([]int64(nil))
float32SliceTyp = reflect.TypeOf([]float32(nil))
float64SliceTyp = reflect.TypeOf([]float64(nil))
mapIntfIntfTyp = reflect.TypeOf(map[interface{}]interface{}(nil))
mapStrIntfTyp = reflect.TypeOf(map[string]interface{}(nil))
mapStrStrTyp = reflect.TypeOf(map[string]string(nil))
mapIntIntfTyp = reflect.TypeOf(map[int]interface{}(nil))
mapInt64IntfTyp = reflect.TypeOf(map[int64]interface{}(nil))
mapUintIntfTyp = reflect.TypeOf(map[uint]interface{}(nil))
mapUint64IntfTyp = reflect.TypeOf(map[uint64]interface{}(nil))
stringTyp = reflect.TypeOf("")
timeTyp = reflect.TypeOf(time.Time{})
rawExtTyp = reflect.TypeOf(RawExt{})
mapBySliceTyp = reflect.TypeOf((*MapBySlice)(nil)).Elem()
binaryMarshalerTyp = reflect.TypeOf((*binaryMarshaler)(nil)).Elem()
binaryUnmarshalerTyp = reflect.TypeOf((*binaryUnmarshaler)(nil)).Elem()
rawExtTypId = reflect.ValueOf(rawExtTyp).Pointer()
intfTypId = reflect.ValueOf(intfTyp).Pointer()
timeTypId = reflect.ValueOf(timeTyp).Pointer()
intfSliceTypId = reflect.ValueOf(intfSliceTyp).Pointer()
strSliceTypId = reflect.ValueOf(strSliceTyp).Pointer()
boolSliceTypId = reflect.ValueOf(boolSliceTyp).Pointer()
uintSliceTypId = reflect.ValueOf(uintSliceTyp).Pointer()
uint8SliceTypId = reflect.ValueOf(uint8SliceTyp).Pointer()
uint16SliceTypId = reflect.ValueOf(uint16SliceTyp).Pointer()
uint32SliceTypId = reflect.ValueOf(uint32SliceTyp).Pointer()
uint64SliceTypId = reflect.ValueOf(uint64SliceTyp).Pointer()
intSliceTypId = reflect.ValueOf(intSliceTyp).Pointer()
int8SliceTypId = reflect.ValueOf(int8SliceTyp).Pointer()
int16SliceTypId = reflect.ValueOf(int16SliceTyp).Pointer()
int32SliceTypId = reflect.ValueOf(int32SliceTyp).Pointer()
int64SliceTypId = reflect.ValueOf(int64SliceTyp).Pointer()
float32SliceTypId = reflect.ValueOf(float32SliceTyp).Pointer()
float64SliceTypId = reflect.ValueOf(float64SliceTyp).Pointer()
mapStrStrTypId = reflect.ValueOf(mapStrStrTyp).Pointer()
mapIntfIntfTypId = reflect.ValueOf(mapIntfIntfTyp).Pointer()
mapStrIntfTypId = reflect.ValueOf(mapStrIntfTyp).Pointer()
mapIntIntfTypId = reflect.ValueOf(mapIntIntfTyp).Pointer()
mapInt64IntfTypId = reflect.ValueOf(mapInt64IntfTyp).Pointer()
mapUintIntfTypId = reflect.ValueOf(mapUintIntfTyp).Pointer()
mapUint64IntfTypId = reflect.ValueOf(mapUint64IntfTyp).Pointer()
// Id = reflect.ValueOf().Pointer()
// mapBySliceTypId = reflect.ValueOf(mapBySliceTyp).Pointer()
binaryMarshalerTypId = reflect.ValueOf(binaryMarshalerTyp).Pointer()
binaryUnmarshalerTypId = reflect.ValueOf(binaryUnmarshalerTyp).Pointer()
intBitsize uint8 = uint8(reflect.TypeOf(int(0)).Bits())
uintBitsize uint8 = uint8(reflect.TypeOf(uint(0)).Bits())
bsAll0x00 = []byte{0, 0, 0, 0, 0, 0, 0, 0}
bsAll0xff = []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
)
type binaryUnmarshaler interface {
UnmarshalBinary(data []byte) error
}
type binaryMarshaler interface {
MarshalBinary() (data []byte, err error)
}
// MapBySlice represents a slice which should be encoded as a map in the stream.
// The slice contains a sequence of key-value pairs.
type MapBySlice interface {
MapBySlice()
}
// WARNING: DO NOT USE DIRECTLY. EXPORTED FOR GODOC BENEFIT. WILL BE REMOVED.
//
// BasicHandle encapsulates the common options and extension functions.
type BasicHandle struct {
extHandle
EncodeOptions
DecodeOptions
}
// Handle is the interface for a specific encoding format.
//
// Typically, a Handle is pre-configured before first time use,
// and not modified while in use. Such a pre-configured Handle
// is safe for concurrent access.
type Handle interface {
writeExt() bool
getBasicHandle() *BasicHandle
newEncDriver(w encWriter) encDriver
newDecDriver(r decReader) decDriver
}
// RawExt represents raw unprocessed extension data.
type RawExt struct {
Tag byte
Data []byte
}
type extTypeTagFn struct {
rtid uintptr
rt reflect.Type
tag byte
encFn func(reflect.Value) ([]byte, error)
decFn func(reflect.Value, []byte) error
}
type extHandle []*extTypeTagFn
// AddExt registers an encode and decode function for a reflect.Type.
// Note that the type must be a named type, and specifically not
// a pointer or Interface. An error is returned if that is not honored.
//
// To Deregister an ext, call AddExt with 0 tag, nil encfn and nil decfn.
func (o *extHandle) AddExt(
rt reflect.Type,
tag byte,
encfn func(reflect.Value) ([]byte, error),
decfn func(reflect.Value, []byte) error,
) (err error) {
// o is a pointer, because we may need to initialize it
if rt.PkgPath() == "" || rt.Kind() == reflect.Interface {
err = fmt.Errorf("codec.Handle.AddExt: Takes named type, especially not a pointer or interface: %T",
reflect.Zero(rt).Interface())
return
}
// o cannot be nil, since it is always embedded in a Handle.
// if nil, let it panic.
// if o == nil {
// err = errors.New("codec.Handle.AddExt: extHandle cannot be a nil pointer.")
// return
// }
rtid := reflect.ValueOf(rt).Pointer()
for _, v := range *o {
if v.rtid == rtid {
v.tag, v.encFn, v.decFn = tag, encfn, decfn
return
}
}
*o = append(*o, &extTypeTagFn{rtid, rt, tag, encfn, decfn})
return
}
func (o extHandle) getExt(rtid uintptr) *extTypeTagFn {
for _, v := range o {
if v.rtid == rtid {
return v
}
}
return nil
}
func (o extHandle) getExtForTag(tag byte) *extTypeTagFn {
for _, v := range o {
if v.tag == tag {
return v
}
}
return nil
}
func (o extHandle) getDecodeExtForTag(tag byte) (
rv reflect.Value, fn func(reflect.Value, []byte) error) {
if x := o.getExtForTag(tag); x != nil {
// ext is only registered for base
rv = reflect.New(x.rt).Elem()
fn = x.decFn
}
return
}
func (o extHandle) getDecodeExt(rtid uintptr) (tag byte, fn func(reflect.Value, []byte) error) {
if x := o.getExt(rtid); x != nil {
tag = x.tag
fn = x.decFn
}
return
}
func (o extHandle) getEncodeExt(rtid uintptr) (tag byte, fn func(reflect.Value) ([]byte, error)) {
if x := o.getExt(rtid); x != nil {
tag = x.tag
fn = x.encFn
}
return
}
type structFieldInfo struct {
encName string // encode name
// only one of 'i' or 'is' can be set. If 'i' is -1, then 'is' has been set.
is []int // (recursive/embedded) field index in struct
i int16 // field index in struct
omitEmpty bool
toArray bool // if field is _struct, is the toArray set?
// tag string // tag
// name string // field name
// encNameBs []byte // encoded name as byte stream
// ikind int // kind of the field as an int i.e. int(reflect.Kind)
}
func parseStructFieldInfo(fname string, stag string) *structFieldInfo {
if fname == "" {
panic("parseStructFieldInfo: No Field Name")
}
si := structFieldInfo{
// name: fname,
encName: fname,
// tag: stag,
}
if stag != "" {
for i, s := range strings.Split(stag, ",") {
if i == 0 {
if s != "" {
si.encName = s
}
} else {
switch s {
case "omitempty":
si.omitEmpty = true
case "toarray":
si.toArray = true
}
}
}
}
// si.encNameBs = []byte(si.encName)
return &si
}
type sfiSortedByEncName []*structFieldInfo
func (p sfiSortedByEncName) Len() int {
return len(p)
}
func (p sfiSortedByEncName) Less(i, j int) bool {
return p[i].encName < p[j].encName
}
func (p sfiSortedByEncName) Swap(i, j int) {
p[i], p[j] = p[j], p[i]
}
// typeInfo keeps information about each type referenced in the encode/decode sequence.
//
// During an encode/decode sequence, we work as below:
// - If base is a built in type, en/decode base value
// - If base is registered as an extension, en/decode base value
// - If type is binary(M/Unm)arshaler, call Binary(M/Unm)arshal method
// - Else decode appropriately based on the reflect.Kind
type typeInfo struct {
sfi []*structFieldInfo // sorted. Used when enc/dec struct to map.
sfip []*structFieldInfo // unsorted. Used when enc/dec struct to array.
rt reflect.Type
rtid uintptr
// baseId gives pointer to the base reflect.Type, after deferencing
// the pointers. E.g. base type of ***time.Time is time.Time.
base reflect.Type
baseId uintptr
baseIndir int8 // number of indirections to get to base
mbs bool // base type (T or *T) is a MapBySlice
m bool // base type (T or *T) is a binaryMarshaler
unm bool // base type (T or *T) is a binaryUnmarshaler
mIndir int8 // number of indirections to get to binaryMarshaler type
unmIndir int8 // number of indirections to get to binaryUnmarshaler type
toArray bool // whether this (struct) type should be encoded as an array
}
func (ti *typeInfo) indexForEncName(name string) int {
//tisfi := ti.sfi
const binarySearchThreshold = 16
if sfilen := len(ti.sfi); sfilen < binarySearchThreshold {
// linear search. faster than binary search in my testing up to 16-field structs.
for i, si := range ti.sfi {
if si.encName == name {
return i
}
}
} else {
// binary search. adapted from sort/search.go.
h, i, j := 0, 0, sfilen
for i < j {
h = i + (j-i)/2
if ti.sfi[h].encName < name {
i = h + 1
} else {
j = h
}
}
if i < sfilen && ti.sfi[i].encName == name {
return i
}
}
return -1
}
func getTypeInfo(rtid uintptr, rt reflect.Type) (pti *typeInfo) {
var ok bool
cachedTypeInfoMutex.RLock()
pti, ok = cachedTypeInfo[rtid]
cachedTypeInfoMutex.RUnlock()
if ok {
return
}
cachedTypeInfoMutex.Lock()
defer cachedTypeInfoMutex.Unlock()
if pti, ok = cachedTypeInfo[rtid]; ok {
return
}
ti := typeInfo{rt: rt, rtid: rtid}
pti = &ti
var indir int8
if ok, indir = implementsIntf(rt, binaryMarshalerTyp); ok {
ti.m, ti.mIndir = true, indir
}
if ok, indir = implementsIntf(rt, binaryUnmarshalerTyp); ok {
ti.unm, ti.unmIndir = true, indir
}
if ok, _ = implementsIntf(rt, mapBySliceTyp); ok {
ti.mbs = true
}
pt := rt
var ptIndir int8
// for ; pt.Kind() == reflect.Ptr; pt, ptIndir = pt.Elem(), ptIndir+1 { }
for pt.Kind() == reflect.Ptr {
pt = pt.Elem()
ptIndir++
}
if ptIndir == 0 {
ti.base = rt
ti.baseId = rtid
} else {
ti.base = pt
ti.baseId = reflect.ValueOf(pt).Pointer()
ti.baseIndir = ptIndir
}
if rt.Kind() == reflect.Struct {
var siInfo *structFieldInfo
if f, ok := rt.FieldByName(structInfoFieldName); ok {
siInfo = parseStructFieldInfo(structInfoFieldName, f.Tag.Get(structTagName))
ti.toArray = siInfo.toArray
}
sfip := make([]*structFieldInfo, 0, rt.NumField())
rgetTypeInfo(rt, nil, make(map[string]bool), &sfip, siInfo)
// // try to put all si close together
// const tryToPutAllStructFieldInfoTogether = true
// if tryToPutAllStructFieldInfoTogether {
// sfip2 := make([]structFieldInfo, len(sfip))
// for i, si := range sfip {
// sfip2[i] = *si
// }
// for i := range sfip {
// sfip[i] = &sfip2[i]
// }
// }
ti.sfip = make([]*structFieldInfo, len(sfip))
ti.sfi = make([]*structFieldInfo, len(sfip))
copy(ti.sfip, sfip)
sort.Sort(sfiSortedByEncName(sfip))
copy(ti.sfi, sfip)
}
// sfi = sfip
cachedTypeInfo[rtid] = pti
return
}
func rgetTypeInfo(rt reflect.Type, indexstack []int, fnameToHastag map[string]bool,
sfi *[]*structFieldInfo, siInfo *structFieldInfo,
) {
// for rt.Kind() == reflect.Ptr {
// // indexstack = append(indexstack, 0)
// rt = rt.Elem()
// }
for j := 0; j < rt.NumField(); j++ {
f := rt.Field(j)
stag := f.Tag.Get(structTagName)
if stag == "-" {
continue
}
if r1, _ := utf8.DecodeRuneInString(f.Name); r1 == utf8.RuneError || !unicode.IsUpper(r1) {
continue
}
// if anonymous and there is no struct tag and its a struct (or pointer to struct), inline it.
if f.Anonymous && stag == "" {
ft := f.Type
for ft.Kind() == reflect.Ptr {
ft = ft.Elem()
}
if ft.Kind() == reflect.Struct {
indexstack2 := append(append(make([]int, 0, len(indexstack)+4), indexstack...), j)
rgetTypeInfo(ft, indexstack2, fnameToHastag, sfi, siInfo)
continue
}
}
// do not let fields with same name in embedded structs override field at higher level.
// this must be done after anonymous check, to allow anonymous field
// still include their child fields
if _, ok := fnameToHastag[f.Name]; ok {
continue
}
si := parseStructFieldInfo(f.Name, stag)
// si.ikind = int(f.Type.Kind())
if len(indexstack) == 0 {
si.i = int16(j)
} else {
si.i = -1
si.is = append(append(make([]int, 0, len(indexstack)+4), indexstack...), j)
}
if siInfo != nil {
if siInfo.omitEmpty {
si.omitEmpty = true
}
}
*sfi = append(*sfi, si)
fnameToHastag[f.Name] = stag != ""
}
}
func panicToErr(err *error) {
if recoverPanicToErr {
if x := recover(); x != nil {
//debug.PrintStack()
panicValToErr(x, err)
}
}
}
func doPanic(tag string, format string, params ...interface{}) {
params2 := make([]interface{}, len(params)+1)
params2[0] = tag
copy(params2[1:], params)
panic(fmt.Errorf("%s: "+format, params2...))
}
func checkOverflowFloat32(f float64, doCheck bool) {
if !doCheck {
return
}
// check overflow (logic adapted from std pkg reflect/value.go OverflowFloat()
f2 := f
if f2 < 0 {
f2 = -f
}
if math.MaxFloat32 < f2 && f2 <= math.MaxFloat64 {
decErr("Overflow float32 value: %v", f2)
}
}
func checkOverflow(ui uint64, i int64, bitsize uint8) {
// check overflow (logic adapted from std pkg reflect/value.go OverflowUint()
if bitsize == 0 {
return
}
if i != 0 {
if trunc := (i << (64 - bitsize)) >> (64 - bitsize); i != trunc {
decErr("Overflow int value: %v", i)
}
}
if ui != 0 {
if trunc := (ui << (64 - bitsize)) >> (64 - bitsize); ui != trunc {
decErr("Overflow uint value: %v", ui)
}
}
}

View File

@@ -0,0 +1,127 @@
// Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved.
// Use of this source code is governed by a BSD-style license found in the LICENSE file.
package codec
// All non-std package dependencies live in this file,
// so porting to different environment is easy (just update functions).
import (
"errors"
"fmt"
"math"
"reflect"
)
var (
raisePanicAfterRecover = false
debugging = true
)
func panicValToErr(panicVal interface{}, err *error) {
switch xerr := panicVal.(type) {
case error:
*err = xerr
case string:
*err = errors.New(xerr)
default:
*err = fmt.Errorf("%v", panicVal)
}
if raisePanicAfterRecover {
panic(panicVal)
}
return
}
func isEmptyValueDeref(v reflect.Value, deref bool) bool {
switch v.Kind() {
case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
return v.Len() == 0
case reflect.Bool:
return !v.Bool()
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return v.Int() == 0
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return v.Uint() == 0
case reflect.Float32, reflect.Float64:
return v.Float() == 0
case reflect.Interface, reflect.Ptr:
if deref {
if v.IsNil() {
return true
}
return isEmptyValueDeref(v.Elem(), deref)
} else {
return v.IsNil()
}
case reflect.Struct:
// return true if all fields are empty. else return false.
// we cannot use equality check, because some fields may be maps/slices/etc
// and consequently the structs are not comparable.
// return v.Interface() == reflect.Zero(v.Type()).Interface()
for i, n := 0, v.NumField(); i < n; i++ {
if !isEmptyValueDeref(v.Field(i), deref) {
return false
}
}
return true
}
return false
}
func isEmptyValue(v reflect.Value) bool {
return isEmptyValueDeref(v, true)
}
func debugf(format string, args ...interface{}) {
if debugging {
if len(format) == 0 || format[len(format)-1] != '\n' {
format = format + "\n"
}
fmt.Printf(format, args...)
}
}
func pruneSignExt(v []byte, pos bool) (n int) {
if len(v) < 2 {
} else if pos && v[0] == 0 {
for ; v[n] == 0 && n+1 < len(v) && (v[n+1]&(1<<7) == 0); n++ {
}
} else if !pos && v[0] == 0xff {
for ; v[n] == 0xff && n+1 < len(v) && (v[n+1]&(1<<7) != 0); n++ {
}
}
return
}
func implementsIntf(typ, iTyp reflect.Type) (success bool, indir int8) {
if typ == nil {
return
}
rt := typ
// The type might be a pointer and we need to keep
// dereferencing to the base type until we find an implementation.
for {
if rt.Implements(iTyp) {
return true, indir
}
if p := rt; p.Kind() == reflect.Ptr {
indir++
if indir >= math.MaxInt8 { // insane number of indirections
return false, 0
}
rt = p.Elem()
continue
}
break
}
// No luck yet, but if this is a base type (non-pointer), the pointer might satisfy.
if typ.Kind() != reflect.Ptr {
// Not a pointer, but does the pointer work?
if reflect.PtrTo(typ).Implements(iTyp) {
return true, -1
}
}
return false, 0
}

816
vendor/github.com/hashicorp/go-msgpack/codec/msgpack.go generated vendored Normal file
View File

@@ -0,0 +1,816 @@
// Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved.
// Use of this source code is governed by a BSD-style license found in the LICENSE file.
/*
MSGPACK
Msgpack-c implementation powers the c, c++, python, ruby, etc libraries.
We need to maintain compatibility with it and how it encodes integer values
without caring about the type.
For compatibility with behaviour of msgpack-c reference implementation:
- Go intX (>0) and uintX
IS ENCODED AS
msgpack +ve fixnum, unsigned
- Go intX (<0)
IS ENCODED AS
msgpack -ve fixnum, signed
*/
package codec
import (
"fmt"
"io"
"math"
"net/rpc"
)
const (
mpPosFixNumMin byte = 0x00
mpPosFixNumMax = 0x7f
mpFixMapMin = 0x80
mpFixMapMax = 0x8f
mpFixArrayMin = 0x90
mpFixArrayMax = 0x9f
mpFixStrMin = 0xa0
mpFixStrMax = 0xbf
mpNil = 0xc0
_ = 0xc1
mpFalse = 0xc2
mpTrue = 0xc3
mpFloat = 0xca
mpDouble = 0xcb
mpUint8 = 0xcc
mpUint16 = 0xcd
mpUint32 = 0xce
mpUint64 = 0xcf
mpInt8 = 0xd0
mpInt16 = 0xd1
mpInt32 = 0xd2
mpInt64 = 0xd3
// extensions below
mpBin8 = 0xc4
mpBin16 = 0xc5
mpBin32 = 0xc6
mpExt8 = 0xc7
mpExt16 = 0xc8
mpExt32 = 0xc9
mpFixExt1 = 0xd4
mpFixExt2 = 0xd5
mpFixExt4 = 0xd6
mpFixExt8 = 0xd7
mpFixExt16 = 0xd8
mpStr8 = 0xd9 // new
mpStr16 = 0xda
mpStr32 = 0xdb
mpArray16 = 0xdc
mpArray32 = 0xdd
mpMap16 = 0xde
mpMap32 = 0xdf
mpNegFixNumMin = 0xe0
mpNegFixNumMax = 0xff
)
// MsgpackSpecRpcMultiArgs is a special type which signifies to the MsgpackSpecRpcCodec
// that the backend RPC service takes multiple arguments, which have been arranged
// in sequence in the slice.
//
// The Codec then passes it AS-IS to the rpc service (without wrapping it in an
// array of 1 element).
type MsgpackSpecRpcMultiArgs []interface{}
// A MsgpackContainer type specifies the different types of msgpackContainers.
type msgpackContainerType struct {
fixCutoff int
bFixMin, b8, b16, b32 byte
hasFixMin, has8, has8Always bool
}
var (
msgpackContainerStr = msgpackContainerType{32, mpFixStrMin, mpStr8, mpStr16, mpStr32, true, true, false}
msgpackContainerBin = msgpackContainerType{0, 0, mpBin8, mpBin16, mpBin32, false, true, true}
msgpackContainerList = msgpackContainerType{16, mpFixArrayMin, 0, mpArray16, mpArray32, true, false, false}
msgpackContainerMap = msgpackContainerType{16, mpFixMapMin, 0, mpMap16, mpMap32, true, false, false}
)
//---------------------------------------------
type msgpackEncDriver struct {
w encWriter
h *MsgpackHandle
}
func (e *msgpackEncDriver) isBuiltinType(rt uintptr) bool {
//no builtin types. All encodings are based on kinds. Types supported as extensions.
return false
}
func (e *msgpackEncDriver) encodeBuiltin(rt uintptr, v interface{}) {}
func (e *msgpackEncDriver) encodeNil() {
e.w.writen1(mpNil)
}
func (e *msgpackEncDriver) encodeInt(i int64) {
switch {
case i >= 0:
e.encodeUint(uint64(i))
case i >= -32:
e.w.writen1(byte(i))
case i >= math.MinInt8:
e.w.writen2(mpInt8, byte(i))
case i >= math.MinInt16:
e.w.writen1(mpInt16)
e.w.writeUint16(uint16(i))
case i >= math.MinInt32:
e.w.writen1(mpInt32)
e.w.writeUint32(uint32(i))
default:
e.w.writen1(mpInt64)
e.w.writeUint64(uint64(i))
}
}
func (e *msgpackEncDriver) encodeUint(i uint64) {
switch {
case i <= math.MaxInt8:
e.w.writen1(byte(i))
case i <= math.MaxUint8:
e.w.writen2(mpUint8, byte(i))
case i <= math.MaxUint16:
e.w.writen1(mpUint16)
e.w.writeUint16(uint16(i))
case i <= math.MaxUint32:
e.w.writen1(mpUint32)
e.w.writeUint32(uint32(i))
default:
e.w.writen1(mpUint64)
e.w.writeUint64(uint64(i))
}
}
func (e *msgpackEncDriver) encodeBool(b bool) {
if b {
e.w.writen1(mpTrue)
} else {
e.w.writen1(mpFalse)
}
}
func (e *msgpackEncDriver) encodeFloat32(f float32) {
e.w.writen1(mpFloat)
e.w.writeUint32(math.Float32bits(f))
}
func (e *msgpackEncDriver) encodeFloat64(f float64) {
e.w.writen1(mpDouble)
e.w.writeUint64(math.Float64bits(f))
}
func (e *msgpackEncDriver) encodeExtPreamble(xtag byte, l int) {
switch {
case l == 1:
e.w.writen2(mpFixExt1, xtag)
case l == 2:
e.w.writen2(mpFixExt2, xtag)
case l == 4:
e.w.writen2(mpFixExt4, xtag)
case l == 8:
e.w.writen2(mpFixExt8, xtag)
case l == 16:
e.w.writen2(mpFixExt16, xtag)
case l < 256:
e.w.writen2(mpExt8, byte(l))
e.w.writen1(xtag)
case l < 65536:
e.w.writen1(mpExt16)
e.w.writeUint16(uint16(l))
e.w.writen1(xtag)
default:
e.w.writen1(mpExt32)
e.w.writeUint32(uint32(l))
e.w.writen1(xtag)
}
}
func (e *msgpackEncDriver) encodeArrayPreamble(length int) {
e.writeContainerLen(msgpackContainerList, length)
}
func (e *msgpackEncDriver) encodeMapPreamble(length int) {
e.writeContainerLen(msgpackContainerMap, length)
}
func (e *msgpackEncDriver) encodeString(c charEncoding, s string) {
if c == c_RAW && e.h.WriteExt {
e.writeContainerLen(msgpackContainerBin, len(s))
} else {
e.writeContainerLen(msgpackContainerStr, len(s))
}
if len(s) > 0 {
e.w.writestr(s)
}
}
func (e *msgpackEncDriver) encodeSymbol(v string) {
e.encodeString(c_UTF8, v)
}
func (e *msgpackEncDriver) encodeStringBytes(c charEncoding, bs []byte) {
if c == c_RAW && e.h.WriteExt {
e.writeContainerLen(msgpackContainerBin, len(bs))
} else {
e.writeContainerLen(msgpackContainerStr, len(bs))
}
if len(bs) > 0 {
e.w.writeb(bs)
}
}
func (e *msgpackEncDriver) writeContainerLen(ct msgpackContainerType, l int) {
switch {
case ct.hasFixMin && l < ct.fixCutoff:
e.w.writen1(ct.bFixMin | byte(l))
case ct.has8 && l < 256 && (ct.has8Always || e.h.WriteExt):
e.w.writen2(ct.b8, uint8(l))
case l < 65536:
e.w.writen1(ct.b16)
e.w.writeUint16(uint16(l))
default:
e.w.writen1(ct.b32)
e.w.writeUint32(uint32(l))
}
}
//---------------------------------------------
type msgpackDecDriver struct {
r decReader
h *MsgpackHandle
bd byte
bdRead bool
bdType valueType
}
func (d *msgpackDecDriver) isBuiltinType(rt uintptr) bool {
//no builtin types. All encodings are based on kinds. Types supported as extensions.
return false
}
func (d *msgpackDecDriver) decodeBuiltin(rt uintptr, v interface{}) {}
// Note: This returns either a primitive (int, bool, etc) for non-containers,
// or a containerType, or a specific type denoting nil or extension.
// It is called when a nil interface{} is passed, leaving it up to the DecDriver
// to introspect the stream and decide how best to decode.
// It deciphers the value by looking at the stream first.
func (d *msgpackDecDriver) decodeNaked() (v interface{}, vt valueType, decodeFurther bool) {
d.initReadNext()
bd := d.bd
switch bd {
case mpNil:
vt = valueTypeNil
d.bdRead = false
case mpFalse:
vt = valueTypeBool
v = false
case mpTrue:
vt = valueTypeBool
v = true
case mpFloat:
vt = valueTypeFloat
v = float64(math.Float32frombits(d.r.readUint32()))
case mpDouble:
vt = valueTypeFloat
v = math.Float64frombits(d.r.readUint64())
case mpUint8:
vt = valueTypeUint
v = uint64(d.r.readn1())
case mpUint16:
vt = valueTypeUint
v = uint64(d.r.readUint16())
case mpUint32:
vt = valueTypeUint
v = uint64(d.r.readUint32())
case mpUint64:
vt = valueTypeUint
v = uint64(d.r.readUint64())
case mpInt8:
vt = valueTypeInt
v = int64(int8(d.r.readn1()))
case mpInt16:
vt = valueTypeInt
v = int64(int16(d.r.readUint16()))
case mpInt32:
vt = valueTypeInt
v = int64(int32(d.r.readUint32()))
case mpInt64:
vt = valueTypeInt
v = int64(int64(d.r.readUint64()))
default:
switch {
case bd >= mpPosFixNumMin && bd <= mpPosFixNumMax:
// positive fixnum (always signed)
vt = valueTypeInt
v = int64(int8(bd))
case bd >= mpNegFixNumMin && bd <= mpNegFixNumMax:
// negative fixnum
vt = valueTypeInt
v = int64(int8(bd))
case bd == mpStr8, bd == mpStr16, bd == mpStr32, bd >= mpFixStrMin && bd <= mpFixStrMax:
if d.h.RawToString {
var rvm string
vt = valueTypeString
v = &rvm
} else {
var rvm = []byte{}
vt = valueTypeBytes
v = &rvm
}
decodeFurther = true
case bd == mpBin8, bd == mpBin16, bd == mpBin32:
var rvm = []byte{}
vt = valueTypeBytes
v = &rvm
decodeFurther = true
case bd == mpArray16, bd == mpArray32, bd >= mpFixArrayMin && bd <= mpFixArrayMax:
vt = valueTypeArray
decodeFurther = true
case bd == mpMap16, bd == mpMap32, bd >= mpFixMapMin && bd <= mpFixMapMax:
vt = valueTypeMap
decodeFurther = true
case bd >= mpFixExt1 && bd <= mpFixExt16, bd >= mpExt8 && bd <= mpExt32:
clen := d.readExtLen()
var re RawExt
re.Tag = d.r.readn1()
re.Data = d.r.readn(clen)
v = &re
vt = valueTypeExt
default:
decErr("Nil-Deciphered DecodeValue: %s: hex: %x, dec: %d", msgBadDesc, bd, bd)
}
}
if !decodeFurther {
d.bdRead = false
}
return
}
// int can be decoded from msgpack type: intXXX or uintXXX
func (d *msgpackDecDriver) decodeInt(bitsize uint8) (i int64) {
switch d.bd {
case mpUint8:
i = int64(uint64(d.r.readn1()))
case mpUint16:
i = int64(uint64(d.r.readUint16()))
case mpUint32:
i = int64(uint64(d.r.readUint32()))
case mpUint64:
i = int64(d.r.readUint64())
case mpInt8:
i = int64(int8(d.r.readn1()))
case mpInt16:
i = int64(int16(d.r.readUint16()))
case mpInt32:
i = int64(int32(d.r.readUint32()))
case mpInt64:
i = int64(d.r.readUint64())
default:
switch {
case d.bd >= mpPosFixNumMin && d.bd <= mpPosFixNumMax:
i = int64(int8(d.bd))
case d.bd >= mpNegFixNumMin && d.bd <= mpNegFixNumMax:
i = int64(int8(d.bd))
default:
decErr("Unhandled single-byte unsigned integer value: %s: %x", msgBadDesc, d.bd)
}
}
// check overflow (logic adapted from std pkg reflect/value.go OverflowUint()
if bitsize > 0 {
if trunc := (i << (64 - bitsize)) >> (64 - bitsize); i != trunc {
decErr("Overflow int value: %v", i)
}
}
d.bdRead = false
return
}
// uint can be decoded from msgpack type: intXXX or uintXXX
func (d *msgpackDecDriver) decodeUint(bitsize uint8) (ui uint64) {
switch d.bd {
case mpUint8:
ui = uint64(d.r.readn1())
case mpUint16:
ui = uint64(d.r.readUint16())
case mpUint32:
ui = uint64(d.r.readUint32())
case mpUint64:
ui = d.r.readUint64()
case mpInt8:
if i := int64(int8(d.r.readn1())); i >= 0 {
ui = uint64(i)
} else {
decErr("Assigning negative signed value: %v, to unsigned type", i)
}
case mpInt16:
if i := int64(int16(d.r.readUint16())); i >= 0 {
ui = uint64(i)
} else {
decErr("Assigning negative signed value: %v, to unsigned type", i)
}
case mpInt32:
if i := int64(int32(d.r.readUint32())); i >= 0 {
ui = uint64(i)
} else {
decErr("Assigning negative signed value: %v, to unsigned type", i)
}
case mpInt64:
if i := int64(d.r.readUint64()); i >= 0 {
ui = uint64(i)
} else {
decErr("Assigning negative signed value: %v, to unsigned type", i)
}
default:
switch {
case d.bd >= mpPosFixNumMin && d.bd <= mpPosFixNumMax:
ui = uint64(d.bd)
case d.bd >= mpNegFixNumMin && d.bd <= mpNegFixNumMax:
decErr("Assigning negative signed value: %v, to unsigned type", int(d.bd))
default:
decErr("Unhandled single-byte unsigned integer value: %s: %x", msgBadDesc, d.bd)
}
}
// check overflow (logic adapted from std pkg reflect/value.go OverflowUint()
if bitsize > 0 {
if trunc := (ui << (64 - bitsize)) >> (64 - bitsize); ui != trunc {
decErr("Overflow uint value: %v", ui)
}
}
d.bdRead = false
return
}
// float can either be decoded from msgpack type: float, double or intX
func (d *msgpackDecDriver) decodeFloat(chkOverflow32 bool) (f float64) {
switch d.bd {
case mpFloat:
f = float64(math.Float32frombits(d.r.readUint32()))
case mpDouble:
f = math.Float64frombits(d.r.readUint64())
default:
f = float64(d.decodeInt(0))
}
checkOverflowFloat32(f, chkOverflow32)
d.bdRead = false
return
}
// bool can be decoded from bool, fixnum 0 or 1.
func (d *msgpackDecDriver) decodeBool() (b bool) {
switch d.bd {
case mpFalse, 0:
// b = false
case mpTrue, 1:
b = true
default:
decErr("Invalid single-byte value for bool: %s: %x", msgBadDesc, d.bd)
}
d.bdRead = false
return
}
func (d *msgpackDecDriver) decodeString() (s string) {
clen := d.readContainerLen(msgpackContainerStr)
if clen > 0 {
s = string(d.r.readn(clen))
}
d.bdRead = false
return
}
// Callers must check if changed=true (to decide whether to replace the one they have)
func (d *msgpackDecDriver) decodeBytes(bs []byte) (bsOut []byte, changed bool) {
// bytes can be decoded from msgpackContainerStr or msgpackContainerBin
var clen int
switch d.bd {
case mpBin8, mpBin16, mpBin32:
clen = d.readContainerLen(msgpackContainerBin)
default:
clen = d.readContainerLen(msgpackContainerStr)
}
// if clen < 0 {
// changed = true
// panic("length cannot be zero. this cannot be nil.")
// }
if clen > 0 {
// if no contents in stream, don't update the passed byteslice
if len(bs) != clen {
// Return changed=true if length of passed slice diff from length of bytes in stream
if len(bs) > clen {
bs = bs[:clen]
} else {
bs = make([]byte, clen)
}
bsOut = bs
changed = true
}
d.r.readb(bs)
}
d.bdRead = false
return
}
// Every top-level decode funcs (i.e. decodeValue, decode) must call this first.
func (d *msgpackDecDriver) initReadNext() {
if d.bdRead {
return
}
d.bd = d.r.readn1()
d.bdRead = true
d.bdType = valueTypeUnset
}
func (d *msgpackDecDriver) currentEncodedType() valueType {
if d.bdType == valueTypeUnset {
bd := d.bd
switch bd {
case mpNil:
d.bdType = valueTypeNil
case mpFalse, mpTrue:
d.bdType = valueTypeBool
case mpFloat, mpDouble:
d.bdType = valueTypeFloat
case mpUint8, mpUint16, mpUint32, mpUint64:
d.bdType = valueTypeUint
case mpInt8, mpInt16, mpInt32, mpInt64:
d.bdType = valueTypeInt
default:
switch {
case bd >= mpPosFixNumMin && bd <= mpPosFixNumMax:
d.bdType = valueTypeInt
case bd >= mpNegFixNumMin && bd <= mpNegFixNumMax:
d.bdType = valueTypeInt
case bd == mpStr8, bd == mpStr16, bd == mpStr32, bd >= mpFixStrMin && bd <= mpFixStrMax:
if d.h.RawToString {
d.bdType = valueTypeString
} else {
d.bdType = valueTypeBytes
}
case bd == mpBin8, bd == mpBin16, bd == mpBin32:
d.bdType = valueTypeBytes
case bd == mpArray16, bd == mpArray32, bd >= mpFixArrayMin && bd <= mpFixArrayMax:
d.bdType = valueTypeArray
case bd == mpMap16, bd == mpMap32, bd >= mpFixMapMin && bd <= mpFixMapMax:
d.bdType = valueTypeMap
case bd >= mpFixExt1 && bd <= mpFixExt16, bd >= mpExt8 && bd <= mpExt32:
d.bdType = valueTypeExt
default:
decErr("currentEncodedType: Undeciphered descriptor: %s: hex: %x, dec: %d", msgBadDesc, bd, bd)
}
}
}
return d.bdType
}
func (d *msgpackDecDriver) tryDecodeAsNil() bool {
if d.bd == mpNil {
d.bdRead = false
return true
}
return false
}
func (d *msgpackDecDriver) readContainerLen(ct msgpackContainerType) (clen int) {
bd := d.bd
switch {
case bd == mpNil:
clen = -1 // to represent nil
case bd == ct.b8:
clen = int(d.r.readn1())
case bd == ct.b16:
clen = int(d.r.readUint16())
case bd == ct.b32:
clen = int(d.r.readUint32())
case (ct.bFixMin & bd) == ct.bFixMin:
clen = int(ct.bFixMin ^ bd)
default:
decErr("readContainerLen: %s: hex: %x, dec: %d", msgBadDesc, bd, bd)
}
d.bdRead = false
return
}
func (d *msgpackDecDriver) readMapLen() int {
return d.readContainerLen(msgpackContainerMap)
}
func (d *msgpackDecDriver) readArrayLen() int {
return d.readContainerLen(msgpackContainerList)
}
func (d *msgpackDecDriver) readExtLen() (clen int) {
switch d.bd {
case mpNil:
clen = -1 // to represent nil
case mpFixExt1:
clen = 1
case mpFixExt2:
clen = 2
case mpFixExt4:
clen = 4
case mpFixExt8:
clen = 8
case mpFixExt16:
clen = 16
case mpExt8:
clen = int(d.r.readn1())
case mpExt16:
clen = int(d.r.readUint16())
case mpExt32:
clen = int(d.r.readUint32())
default:
decErr("decoding ext bytes: found unexpected byte: %x", d.bd)
}
return
}
func (d *msgpackDecDriver) decodeExt(verifyTag bool, tag byte) (xtag byte, xbs []byte) {
xbd := d.bd
switch {
case xbd == mpBin8, xbd == mpBin16, xbd == mpBin32:
xbs, _ = d.decodeBytes(nil)
case xbd == mpStr8, xbd == mpStr16, xbd == mpStr32,
xbd >= mpFixStrMin && xbd <= mpFixStrMax:
xbs = []byte(d.decodeString())
default:
clen := d.readExtLen()
xtag = d.r.readn1()
if verifyTag && xtag != tag {
decErr("Wrong extension tag. Got %b. Expecting: %v", xtag, tag)
}
xbs = d.r.readn(clen)
}
d.bdRead = false
return
}
//--------------------------------------------------
//MsgpackHandle is a Handle for the Msgpack Schema-Free Encoding Format.
type MsgpackHandle struct {
BasicHandle
// RawToString controls how raw bytes are decoded into a nil interface{}.
RawToString bool
// WriteExt flag supports encoding configured extensions with extension tags.
// It also controls whether other elements of the new spec are encoded (ie Str8).
//
// With WriteExt=false, configured extensions are serialized as raw bytes
// and Str8 is not encoded.
//
// A stream can still be decoded into a typed value, provided an appropriate value
// is provided, but the type cannot be inferred from the stream. If no appropriate
// type is provided (e.g. decoding into a nil interface{}), you get back
// a []byte or string based on the setting of RawToString.
WriteExt bool
}
func (h *MsgpackHandle) newEncDriver(w encWriter) encDriver {
return &msgpackEncDriver{w: w, h: h}
}
func (h *MsgpackHandle) newDecDriver(r decReader) decDriver {
return &msgpackDecDriver{r: r, h: h}
}
func (h *MsgpackHandle) writeExt() bool {
return h.WriteExt
}
func (h *MsgpackHandle) getBasicHandle() *BasicHandle {
return &h.BasicHandle
}
//--------------------------------------------------
type msgpackSpecRpcCodec struct {
rpcCodec
}
// /////////////// Spec RPC Codec ///////////////////
func (c *msgpackSpecRpcCodec) WriteRequest(r *rpc.Request, body interface{}) error {
// WriteRequest can write to both a Go service, and other services that do
// not abide by the 1 argument rule of a Go service.
// We discriminate based on if the body is a MsgpackSpecRpcMultiArgs
var bodyArr []interface{}
if m, ok := body.(MsgpackSpecRpcMultiArgs); ok {
bodyArr = ([]interface{})(m)
} else {
bodyArr = []interface{}{body}
}
r2 := []interface{}{0, uint32(r.Seq), r.ServiceMethod, bodyArr}
return c.write(r2, nil, false, true)
}
func (c *msgpackSpecRpcCodec) WriteResponse(r *rpc.Response, body interface{}) error {
var moe interface{}
if r.Error != "" {
moe = r.Error
}
if moe != nil && body != nil {
body = nil
}
r2 := []interface{}{1, uint32(r.Seq), moe, body}
return c.write(r2, nil, false, true)
}
func (c *msgpackSpecRpcCodec) ReadResponseHeader(r *rpc.Response) error {
return c.parseCustomHeader(1, &r.Seq, &r.Error)
}
func (c *msgpackSpecRpcCodec) ReadRequestHeader(r *rpc.Request) error {
return c.parseCustomHeader(0, &r.Seq, &r.ServiceMethod)
}
func (c *msgpackSpecRpcCodec) ReadRequestBody(body interface{}) error {
if body == nil { // read and discard
return c.read(nil)
}
bodyArr := []interface{}{body}
return c.read(&bodyArr)
}
func (c *msgpackSpecRpcCodec) parseCustomHeader(expectTypeByte byte, msgid *uint64, methodOrError *string) (err error) {
if c.cls {
return io.EOF
}
// We read the response header by hand
// so that the body can be decoded on its own from the stream at a later time.
const fia byte = 0x94 //four item array descriptor value
// Not sure why the panic of EOF is swallowed above.
// if bs1 := c.dec.r.readn1(); bs1 != fia {
// err = fmt.Errorf("Unexpected value for array descriptor: Expecting %v. Received %v", fia, bs1)
// return
// }
var b byte
b, err = c.br.ReadByte()
if err != nil {
return
}
if b != fia {
err = fmt.Errorf("Unexpected value for array descriptor: Expecting %v. Received %v", fia, b)
return
}
if err = c.read(&b); err != nil {
return
}
if b != expectTypeByte {
err = fmt.Errorf("Unexpected byte descriptor in header. Expecting %v. Received %v", expectTypeByte, b)
return
}
if err = c.read(msgid); err != nil {
return
}
if err = c.read(methodOrError); err != nil {
return
}
return
}
//--------------------------------------------------
// msgpackSpecRpc is the implementation of Rpc that uses custom communication protocol
// as defined in the msgpack spec at https://github.com/msgpack-rpc/msgpack-rpc/blob/master/spec.md
type msgpackSpecRpc struct{}
// MsgpackSpecRpc implements Rpc using the communication protocol defined in
// the msgpack spec at https://github.com/msgpack-rpc/msgpack-rpc/blob/master/spec.md .
// Its methods (ServerCodec and ClientCodec) return values that implement RpcCodecBuffered.
var MsgpackSpecRpc msgpackSpecRpc
func (x msgpackSpecRpc) ServerCodec(conn io.ReadWriteCloser, h Handle) rpc.ServerCodec {
return &msgpackSpecRpcCodec{newRPCCodec(conn, h)}
}
func (x msgpackSpecRpc) ClientCodec(conn io.ReadWriteCloser, h Handle) rpc.ClientCodec {
return &msgpackSpecRpcCodec{newRPCCodec(conn, h)}
}
var _ decDriver = (*msgpackDecDriver)(nil)
var _ encDriver = (*msgpackEncDriver)(nil)

110
vendor/github.com/hashicorp/go-msgpack/codec/msgpack_test.py generated vendored Executable file
View File

@@ -0,0 +1,110 @@
#!/usr/bin/env python
# This will create golden files in a directory passed to it.
# A Test calls this internally to create the golden files
# So it can process them (so we don't have to checkin the files).
import msgpack, msgpackrpc, sys, os, threading
def get_test_data_list():
# get list with all primitive types, and a combo type
l0 = [
-8,
-1616,
-32323232,
-6464646464646464,
192,
1616,
32323232,
6464646464646464,
192,
-3232.0,
-6464646464.0,
3232.0,
6464646464.0,
False,
True,
None,
"someday",
"",
"bytestring",
1328176922000002000,
-2206187877999998000,
0,
-6795364578871345152
]
l1 = [
{ "true": True,
"false": False },
{ "true": "True",
"false": False,
"uint16(1616)": 1616 },
{ "list": [1616, 32323232, True, -3232.0, {"TRUE":True, "FALSE":False}, [True, False] ],
"int32":32323232, "bool": True,
"LONG STRING": "123456789012345678901234567890123456789012345678901234567890",
"SHORT STRING": "1234567890" },
{ True: "true", 8: False, "false": 0 }
]
l = []
l.extend(l0)
l.append(l0)
l.extend(l1)
return l
def build_test_data(destdir):
l = get_test_data_list()
for i in range(len(l)):
packer = msgpack.Packer()
serialized = packer.pack(l[i])
f = open(os.path.join(destdir, str(i) + '.golden'), 'wb')
f.write(serialized)
f.close()
def doRpcServer(port, stopTimeSec):
class EchoHandler(object):
def Echo123(self, msg1, msg2, msg3):
return ("1:%s 2:%s 3:%s" % (msg1, msg2, msg3))
def EchoStruct(self, msg):
return ("%s" % msg)
addr = msgpackrpc.Address('localhost', port)
server = msgpackrpc.Server(EchoHandler())
server.listen(addr)
# run thread to stop it after stopTimeSec seconds if > 0
if stopTimeSec > 0:
def myStopRpcServer():
server.stop()
t = threading.Timer(stopTimeSec, myStopRpcServer)
t.start()
server.start()
def doRpcClientToPythonSvc(port):
address = msgpackrpc.Address('localhost', port)
client = msgpackrpc.Client(address, unpack_encoding='utf-8')
print client.call("Echo123", "A1", "B2", "C3")
print client.call("EchoStruct", {"A" :"Aa", "B":"Bb", "C":"Cc"})
def doRpcClientToGoSvc(port):
# print ">>>> port: ", port, " <<<<<"
address = msgpackrpc.Address('localhost', port)
client = msgpackrpc.Client(address, unpack_encoding='utf-8')
print client.call("TestRpcInt.Echo123", ["A1", "B2", "C3"])
print client.call("TestRpcInt.EchoStruct", {"A" :"Aa", "B":"Bb", "C":"Cc"})
def doMain(args):
if len(args) == 2 and args[0] == "testdata":
build_test_data(args[1])
elif len(args) == 3 and args[0] == "rpc-server":
doRpcServer(int(args[1]), int(args[2]))
elif len(args) == 2 and args[0] == "rpc-client-python-service":
doRpcClientToPythonSvc(int(args[1]))
elif len(args) == 2 and args[0] == "rpc-client-go-service":
doRpcClientToGoSvc(int(args[1]))
else:
print("Usage: msgpack_test.py " +
"[testdata|rpc-server|rpc-client-python-service|rpc-client-go-service] ...")
if __name__ == "__main__":
doMain(sys.argv[1:])

152
vendor/github.com/hashicorp/go-msgpack/codec/rpc.go generated vendored Normal file
View File

@@ -0,0 +1,152 @@
// Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved.
// Use of this source code is governed by a BSD-style license found in the LICENSE file.
package codec
import (
"bufio"
"io"
"net/rpc"
"sync"
)
// Rpc provides a rpc Server or Client Codec for rpc communication.
type Rpc interface {
ServerCodec(conn io.ReadWriteCloser, h Handle) rpc.ServerCodec
ClientCodec(conn io.ReadWriteCloser, h Handle) rpc.ClientCodec
}
// RpcCodecBuffered allows access to the underlying bufio.Reader/Writer
// used by the rpc connection. It accomodates use-cases where the connection
// should be used by rpc and non-rpc functions, e.g. streaming a file after
// sending an rpc response.
type RpcCodecBuffered interface {
BufferedReader() *bufio.Reader
BufferedWriter() *bufio.Writer
}
// -------------------------------------
// rpcCodec defines the struct members and common methods.
type rpcCodec struct {
rwc io.ReadWriteCloser
dec *Decoder
enc *Encoder
bw *bufio.Writer
br *bufio.Reader
mu sync.Mutex
cls bool
}
func newRPCCodec(conn io.ReadWriteCloser, h Handle) rpcCodec {
bw := bufio.NewWriter(conn)
br := bufio.NewReader(conn)
return rpcCodec{
rwc: conn,
bw: bw,
br: br,
enc: NewEncoder(bw, h),
dec: NewDecoder(br, h),
}
}
func (c *rpcCodec) BufferedReader() *bufio.Reader {
return c.br
}
func (c *rpcCodec) BufferedWriter() *bufio.Writer {
return c.bw
}
func (c *rpcCodec) write(obj1, obj2 interface{}, writeObj2, doFlush bool) (err error) {
if c.cls {
return io.EOF
}
if err = c.enc.Encode(obj1); err != nil {
return
}
if writeObj2 {
if err = c.enc.Encode(obj2); err != nil {
return
}
}
if doFlush && c.bw != nil {
return c.bw.Flush()
}
return
}
func (c *rpcCodec) read(obj interface{}) (err error) {
if c.cls {
return io.EOF
}
//If nil is passed in, we should still attempt to read content to nowhere.
if obj == nil {
var obj2 interface{}
return c.dec.Decode(&obj2)
}
return c.dec.Decode(obj)
}
func (c *rpcCodec) Close() error {
if c.cls {
return io.EOF
}
c.cls = true
return c.rwc.Close()
}
func (c *rpcCodec) ReadResponseBody(body interface{}) error {
return c.read(body)
}
// -------------------------------------
type goRpcCodec struct {
rpcCodec
}
func (c *goRpcCodec) WriteRequest(r *rpc.Request, body interface{}) error {
// Must protect for concurrent access as per API
c.mu.Lock()
defer c.mu.Unlock()
return c.write(r, body, true, true)
}
func (c *goRpcCodec) WriteResponse(r *rpc.Response, body interface{}) error {
c.mu.Lock()
defer c.mu.Unlock()
return c.write(r, body, true, true)
}
func (c *goRpcCodec) ReadResponseHeader(r *rpc.Response) error {
return c.read(r)
}
func (c *goRpcCodec) ReadRequestHeader(r *rpc.Request) error {
return c.read(r)
}
func (c *goRpcCodec) ReadRequestBody(body interface{}) error {
return c.read(body)
}
// -------------------------------------
// goRpc is the implementation of Rpc that uses the communication protocol
// as defined in net/rpc package.
type goRpc struct{}
// GoRpc implements Rpc using the communication protocol defined in net/rpc package.
// Its methods (ServerCodec and ClientCodec) return values that implement RpcCodecBuffered.
var GoRpc goRpc
func (x goRpc) ServerCodec(conn io.ReadWriteCloser, h Handle) rpc.ServerCodec {
return &goRpcCodec{newRPCCodec(conn, h)}
}
func (x goRpc) ClientCodec(conn io.ReadWriteCloser, h Handle) rpc.ClientCodec {
return &goRpcCodec{newRPCCodec(conn, h)}
}
var _ RpcCodecBuffered = (*rpcCodec)(nil) // ensure *rpcCodec implements RpcCodecBuffered

461
vendor/github.com/hashicorp/go-msgpack/codec/simple.go generated vendored Normal file
View File

@@ -0,0 +1,461 @@
// Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved.
// Use of this source code is governed by a BSD-style license found in the LICENSE file.
package codec
import "math"
const (
_ uint8 = iota
simpleVdNil = 1
simpleVdFalse = 2
simpleVdTrue = 3
simpleVdFloat32 = 4
simpleVdFloat64 = 5
// each lasts for 4 (ie n, n+1, n+2, n+3)
simpleVdPosInt = 8
simpleVdNegInt = 12
// containers: each lasts for 4 (ie n, n+1, n+2, ... n+7)
simpleVdString = 216
simpleVdByteArray = 224
simpleVdArray = 232
simpleVdMap = 240
simpleVdExt = 248
)
type simpleEncDriver struct {
h *SimpleHandle
w encWriter
//b [8]byte
}
func (e *simpleEncDriver) isBuiltinType(rt uintptr) bool {
return false
}
func (e *simpleEncDriver) encodeBuiltin(rt uintptr, v interface{}) {
}
func (e *simpleEncDriver) encodeNil() {
e.w.writen1(simpleVdNil)
}
func (e *simpleEncDriver) encodeBool(b bool) {
if b {
e.w.writen1(simpleVdTrue)
} else {
e.w.writen1(simpleVdFalse)
}
}
func (e *simpleEncDriver) encodeFloat32(f float32) {
e.w.writen1(simpleVdFloat32)
e.w.writeUint32(math.Float32bits(f))
}
func (e *simpleEncDriver) encodeFloat64(f float64) {
e.w.writen1(simpleVdFloat64)
e.w.writeUint64(math.Float64bits(f))
}
func (e *simpleEncDriver) encodeInt(v int64) {
if v < 0 {
e.encUint(uint64(-v), simpleVdNegInt)
} else {
e.encUint(uint64(v), simpleVdPosInt)
}
}
func (e *simpleEncDriver) encodeUint(v uint64) {
e.encUint(v, simpleVdPosInt)
}
func (e *simpleEncDriver) encUint(v uint64, bd uint8) {
switch {
case v <= math.MaxUint8:
e.w.writen2(bd, uint8(v))
case v <= math.MaxUint16:
e.w.writen1(bd + 1)
e.w.writeUint16(uint16(v))
case v <= math.MaxUint32:
e.w.writen1(bd + 2)
e.w.writeUint32(uint32(v))
case v <= math.MaxUint64:
e.w.writen1(bd + 3)
e.w.writeUint64(v)
}
}
func (e *simpleEncDriver) encLen(bd byte, length int) {
switch {
case length == 0:
e.w.writen1(bd)
case length <= math.MaxUint8:
e.w.writen1(bd + 1)
e.w.writen1(uint8(length))
case length <= math.MaxUint16:
e.w.writen1(bd + 2)
e.w.writeUint16(uint16(length))
case int64(length) <= math.MaxUint32:
e.w.writen1(bd + 3)
e.w.writeUint32(uint32(length))
default:
e.w.writen1(bd + 4)
e.w.writeUint64(uint64(length))
}
}
func (e *simpleEncDriver) encodeExtPreamble(xtag byte, length int) {
e.encLen(simpleVdExt, length)
e.w.writen1(xtag)
}
func (e *simpleEncDriver) encodeArrayPreamble(length int) {
e.encLen(simpleVdArray, length)
}
func (e *simpleEncDriver) encodeMapPreamble(length int) {
e.encLen(simpleVdMap, length)
}
func (e *simpleEncDriver) encodeString(c charEncoding, v string) {
e.encLen(simpleVdString, len(v))
e.w.writestr(v)
}
func (e *simpleEncDriver) encodeSymbol(v string) {
e.encodeString(c_UTF8, v)
}
func (e *simpleEncDriver) encodeStringBytes(c charEncoding, v []byte) {
e.encLen(simpleVdByteArray, len(v))
e.w.writeb(v)
}
//------------------------------------
type simpleDecDriver struct {
h *SimpleHandle
r decReader
bdRead bool
bdType valueType
bd byte
//b [8]byte
}
func (d *simpleDecDriver) initReadNext() {
if d.bdRead {
return
}
d.bd = d.r.readn1()
d.bdRead = true
d.bdType = valueTypeUnset
}
func (d *simpleDecDriver) currentEncodedType() valueType {
if d.bdType == valueTypeUnset {
switch d.bd {
case simpleVdNil:
d.bdType = valueTypeNil
case simpleVdTrue, simpleVdFalse:
d.bdType = valueTypeBool
case simpleVdPosInt, simpleVdPosInt + 1, simpleVdPosInt + 2, simpleVdPosInt + 3:
d.bdType = valueTypeUint
case simpleVdNegInt, simpleVdNegInt + 1, simpleVdNegInt + 2, simpleVdNegInt + 3:
d.bdType = valueTypeInt
case simpleVdFloat32, simpleVdFloat64:
d.bdType = valueTypeFloat
case simpleVdString, simpleVdString + 1, simpleVdString + 2, simpleVdString + 3, simpleVdString + 4:
d.bdType = valueTypeString
case simpleVdByteArray, simpleVdByteArray + 1, simpleVdByteArray + 2, simpleVdByteArray + 3, simpleVdByteArray + 4:
d.bdType = valueTypeBytes
case simpleVdExt, simpleVdExt + 1, simpleVdExt + 2, simpleVdExt + 3, simpleVdExt + 4:
d.bdType = valueTypeExt
case simpleVdArray, simpleVdArray + 1, simpleVdArray + 2, simpleVdArray + 3, simpleVdArray + 4:
d.bdType = valueTypeArray
case simpleVdMap, simpleVdMap + 1, simpleVdMap + 2, simpleVdMap + 3, simpleVdMap + 4:
d.bdType = valueTypeMap
default:
decErr("currentEncodedType: Unrecognized d.vd: 0x%x", d.bd)
}
}
return d.bdType
}
func (d *simpleDecDriver) tryDecodeAsNil() bool {
if d.bd == simpleVdNil {
d.bdRead = false
return true
}
return false
}
func (d *simpleDecDriver) isBuiltinType(rt uintptr) bool {
return false
}
func (d *simpleDecDriver) decodeBuiltin(rt uintptr, v interface{}) {
}
func (d *simpleDecDriver) decIntAny() (ui uint64, i int64, neg bool) {
switch d.bd {
case simpleVdPosInt:
ui = uint64(d.r.readn1())
i = int64(ui)
case simpleVdPosInt + 1:
ui = uint64(d.r.readUint16())
i = int64(ui)
case simpleVdPosInt + 2:
ui = uint64(d.r.readUint32())
i = int64(ui)
case simpleVdPosInt + 3:
ui = uint64(d.r.readUint64())
i = int64(ui)
case simpleVdNegInt:
ui = uint64(d.r.readn1())
i = -(int64(ui))
neg = true
case simpleVdNegInt + 1:
ui = uint64(d.r.readUint16())
i = -(int64(ui))
neg = true
case simpleVdNegInt + 2:
ui = uint64(d.r.readUint32())
i = -(int64(ui))
neg = true
case simpleVdNegInt + 3:
ui = uint64(d.r.readUint64())
i = -(int64(ui))
neg = true
default:
decErr("decIntAny: Integer only valid from pos/neg integer1..8. Invalid descriptor: %v", d.bd)
}
// don't do this check, because callers may only want the unsigned value.
// if ui > math.MaxInt64 {
// decErr("decIntAny: Integer out of range for signed int64: %v", ui)
// }
return
}
func (d *simpleDecDriver) decodeInt(bitsize uint8) (i int64) {
_, i, _ = d.decIntAny()
checkOverflow(0, i, bitsize)
d.bdRead = false
return
}
func (d *simpleDecDriver) decodeUint(bitsize uint8) (ui uint64) {
ui, i, neg := d.decIntAny()
if neg {
decErr("Assigning negative signed value: %v, to unsigned type", i)
}
checkOverflow(ui, 0, bitsize)
d.bdRead = false
return
}
func (d *simpleDecDriver) decodeFloat(chkOverflow32 bool) (f float64) {
switch d.bd {
case simpleVdFloat32:
f = float64(math.Float32frombits(d.r.readUint32()))
case simpleVdFloat64:
f = math.Float64frombits(d.r.readUint64())
default:
if d.bd >= simpleVdPosInt && d.bd <= simpleVdNegInt+3 {
_, i, _ := d.decIntAny()
f = float64(i)
} else {
decErr("Float only valid from float32/64: Invalid descriptor: %v", d.bd)
}
}
checkOverflowFloat32(f, chkOverflow32)
d.bdRead = false
return
}
// bool can be decoded from bool only (single byte).
func (d *simpleDecDriver) decodeBool() (b bool) {
switch d.bd {
case simpleVdTrue:
b = true
case simpleVdFalse:
default:
decErr("Invalid single-byte value for bool: %s: %x", msgBadDesc, d.bd)
}
d.bdRead = false
return
}
func (d *simpleDecDriver) readMapLen() (length int) {
d.bdRead = false
return d.decLen()
}
func (d *simpleDecDriver) readArrayLen() (length int) {
d.bdRead = false
return d.decLen()
}
func (d *simpleDecDriver) decLen() int {
switch d.bd % 8 {
case 0:
return 0
case 1:
return int(d.r.readn1())
case 2:
return int(d.r.readUint16())
case 3:
ui := uint64(d.r.readUint32())
checkOverflow(ui, 0, intBitsize)
return int(ui)
case 4:
ui := d.r.readUint64()
checkOverflow(ui, 0, intBitsize)
return int(ui)
}
decErr("decLen: Cannot read length: bd%8 must be in range 0..4. Got: %d", d.bd%8)
return -1
}
func (d *simpleDecDriver) decodeString() (s string) {
s = string(d.r.readn(d.decLen()))
d.bdRead = false
return
}
func (d *simpleDecDriver) decodeBytes(bs []byte) (bsOut []byte, changed bool) {
if clen := d.decLen(); clen > 0 {
// if no contents in stream, don't update the passed byteslice
if len(bs) != clen {
if len(bs) > clen {
bs = bs[:clen]
} else {
bs = make([]byte, clen)
}
bsOut = bs
changed = true
}
d.r.readb(bs)
}
d.bdRead = false
return
}
func (d *simpleDecDriver) decodeExt(verifyTag bool, tag byte) (xtag byte, xbs []byte) {
switch d.bd {
case simpleVdExt, simpleVdExt + 1, simpleVdExt + 2, simpleVdExt + 3, simpleVdExt + 4:
l := d.decLen()
xtag = d.r.readn1()
if verifyTag && xtag != tag {
decErr("Wrong extension tag. Got %b. Expecting: %v", xtag, tag)
}
xbs = d.r.readn(l)
case simpleVdByteArray, simpleVdByteArray + 1, simpleVdByteArray + 2, simpleVdByteArray + 3, simpleVdByteArray + 4:
xbs, _ = d.decodeBytes(nil)
default:
decErr("Invalid d.vd for extensions (Expecting extensions or byte array). Got: 0x%x", d.bd)
}
d.bdRead = false
return
}
func (d *simpleDecDriver) decodeNaked() (v interface{}, vt valueType, decodeFurther bool) {
d.initReadNext()
switch d.bd {
case simpleVdNil:
vt = valueTypeNil
case simpleVdFalse:
vt = valueTypeBool
v = false
case simpleVdTrue:
vt = valueTypeBool
v = true
case simpleVdPosInt, simpleVdPosInt + 1, simpleVdPosInt + 2, simpleVdPosInt + 3:
vt = valueTypeUint
ui, _, _ := d.decIntAny()
v = ui
case simpleVdNegInt, simpleVdNegInt + 1, simpleVdNegInt + 2, simpleVdNegInt + 3:
vt = valueTypeInt
_, i, _ := d.decIntAny()
v = i
case simpleVdFloat32:
vt = valueTypeFloat
v = d.decodeFloat(true)
case simpleVdFloat64:
vt = valueTypeFloat
v = d.decodeFloat(false)
case simpleVdString, simpleVdString + 1, simpleVdString + 2, simpleVdString + 3, simpleVdString + 4:
vt = valueTypeString
v = d.decodeString()
case simpleVdByteArray, simpleVdByteArray + 1, simpleVdByteArray + 2, simpleVdByteArray + 3, simpleVdByteArray + 4:
vt = valueTypeBytes
v, _ = d.decodeBytes(nil)
case simpleVdExt, simpleVdExt + 1, simpleVdExt + 2, simpleVdExt + 3, simpleVdExt + 4:
vt = valueTypeExt
l := d.decLen()
var re RawExt
re.Tag = d.r.readn1()
re.Data = d.r.readn(l)
v = &re
vt = valueTypeExt
case simpleVdArray, simpleVdArray + 1, simpleVdArray + 2, simpleVdArray + 3, simpleVdArray + 4:
vt = valueTypeArray
decodeFurther = true
case simpleVdMap, simpleVdMap + 1, simpleVdMap + 2, simpleVdMap + 3, simpleVdMap + 4:
vt = valueTypeMap
decodeFurther = true
default:
decErr("decodeNaked: Unrecognized d.vd: 0x%x", d.bd)
}
if !decodeFurther {
d.bdRead = false
}
return
}
//------------------------------------
// SimpleHandle is a Handle for a very simple encoding format.
//
// simple is a simplistic codec similar to binc, but not as compact.
// - Encoding of a value is always preceeded by the descriptor byte (bd)
// - True, false, nil are encoded fully in 1 byte (the descriptor)
// - Integers (intXXX, uintXXX) are encoded in 1, 2, 4 or 8 bytes (plus a descriptor byte).
// There are positive (uintXXX and intXXX >= 0) and negative (intXXX < 0) integers.
// - Floats are encoded in 4 or 8 bytes (plus a descriptor byte)
// - Lenght of containers (strings, bytes, array, map, extensions)
// are encoded in 0, 1, 2, 4 or 8 bytes.
// Zero-length containers have no length encoded.
// For others, the number of bytes is given by pow(2, bd%3)
// - maps are encoded as [bd] [length] [[key][value]]...
// - arrays are encoded as [bd] [length] [value]...
// - extensions are encoded as [bd] [length] [tag] [byte]...
// - strings/bytearrays are encoded as [bd] [length] [byte]...
//
// The full spec will be published soon.
type SimpleHandle struct {
BasicHandle
}
func (h *SimpleHandle) newEncDriver(w encWriter) encDriver {
return &simpleEncDriver{w: w, h: h}
}
func (h *SimpleHandle) newDecDriver(r decReader) decDriver {
return &simpleDecDriver{r: r, h: h}
}
func (_ *SimpleHandle) writeExt() bool {
return true
}
func (h *SimpleHandle) getBasicHandle() *BasicHandle {
return &h.BasicHandle
}
var _ decDriver = (*simpleDecDriver)(nil)
var _ encDriver = (*simpleEncDriver)(nil)

193
vendor/github.com/hashicorp/go-msgpack/codec/time.go generated vendored Normal file
View File

@@ -0,0 +1,193 @@
// Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved.
// Use of this source code is governed by a BSD-style license found in the LICENSE file.
package codec
import (
"time"
)
var (
timeDigits = [...]byte{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9'}
)
// EncodeTime encodes a time.Time as a []byte, including
// information on the instant in time and UTC offset.
//
// Format Description
//
// A timestamp is composed of 3 components:
//
// - secs: signed integer representing seconds since unix epoch
// - nsces: unsigned integer representing fractional seconds as a
// nanosecond offset within secs, in the range 0 <= nsecs < 1e9
// - tz: signed integer representing timezone offset in minutes east of UTC,
// and a dst (daylight savings time) flag
//
// When encoding a timestamp, the first byte is the descriptor, which
// defines which components are encoded and how many bytes are used to
// encode secs and nsecs components. *If secs/nsecs is 0 or tz is UTC, it
// is not encoded in the byte array explicitly*.
//
// Descriptor 8 bits are of the form `A B C DDD EE`:
// A: Is secs component encoded? 1 = true
// B: Is nsecs component encoded? 1 = true
// C: Is tz component encoded? 1 = true
// DDD: Number of extra bytes for secs (range 0-7).
// If A = 1, secs encoded in DDD+1 bytes.
// If A = 0, secs is not encoded, and is assumed to be 0.
// If A = 1, then we need at least 1 byte to encode secs.
// DDD says the number of extra bytes beyond that 1.
// E.g. if DDD=0, then secs is represented in 1 byte.
// if DDD=2, then secs is represented in 3 bytes.
// EE: Number of extra bytes for nsecs (range 0-3).
// If B = 1, nsecs encoded in EE+1 bytes (similar to secs/DDD above)
//
// Following the descriptor bytes, subsequent bytes are:
//
// secs component encoded in `DDD + 1` bytes (if A == 1)
// nsecs component encoded in `EE + 1` bytes (if B == 1)
// tz component encoded in 2 bytes (if C == 1)
//
// secs and nsecs components are integers encoded in a BigEndian
// 2-complement encoding format.
//
// tz component is encoded as 2 bytes (16 bits). Most significant bit 15 to
// Least significant bit 0 are described below:
//
// Timezone offset has a range of -12:00 to +14:00 (ie -720 to +840 minutes).
// Bit 15 = have\_dst: set to 1 if we set the dst flag.
// Bit 14 = dst\_on: set to 1 if dst is in effect at the time, or 0 if not.
// Bits 13..0 = timezone offset in minutes. It is a signed integer in Big Endian format.
//
func encodeTime(t time.Time) []byte {
//t := rv.Interface().(time.Time)
tsecs, tnsecs := t.Unix(), t.Nanosecond()
var (
bd byte
btmp [8]byte
bs [16]byte
i int = 1
)
l := t.Location()
if l == time.UTC {
l = nil
}
if tsecs != 0 {
bd = bd | 0x80
bigen.PutUint64(btmp[:], uint64(tsecs))
f := pruneSignExt(btmp[:], tsecs >= 0)
bd = bd | (byte(7-f) << 2)
copy(bs[i:], btmp[f:])
i = i + (8 - f)
}
if tnsecs != 0 {
bd = bd | 0x40
bigen.PutUint32(btmp[:4], uint32(tnsecs))
f := pruneSignExt(btmp[:4], true)
bd = bd | byte(3-f)
copy(bs[i:], btmp[f:4])
i = i + (4 - f)
}
if l != nil {
bd = bd | 0x20
// Note that Go Libs do not give access to dst flag.
_, zoneOffset := t.Zone()
//zoneName, zoneOffset := t.Zone()
zoneOffset /= 60
z := uint16(zoneOffset)
bigen.PutUint16(btmp[:2], z)
// clear dst flags
bs[i] = btmp[0] & 0x3f
bs[i+1] = btmp[1]
i = i + 2
}
bs[0] = bd
return bs[0:i]
}
// DecodeTime decodes a []byte into a time.Time.
func decodeTime(bs []byte) (tt time.Time, err error) {
bd := bs[0]
var (
tsec int64
tnsec uint32
tz uint16
i byte = 1
i2 byte
n byte
)
if bd&(1<<7) != 0 {
var btmp [8]byte
n = ((bd >> 2) & 0x7) + 1
i2 = i + n
copy(btmp[8-n:], bs[i:i2])
//if first bit of bs[i] is set, then fill btmp[0..8-n] with 0xff (ie sign extend it)
if bs[i]&(1<<7) != 0 {
copy(btmp[0:8-n], bsAll0xff)
//for j,k := byte(0), 8-n; j < k; j++ { btmp[j] = 0xff }
}
i = i2
tsec = int64(bigen.Uint64(btmp[:]))
}
if bd&(1<<6) != 0 {
var btmp [4]byte
n = (bd & 0x3) + 1
i2 = i + n
copy(btmp[4-n:], bs[i:i2])
i = i2
tnsec = bigen.Uint32(btmp[:])
}
if bd&(1<<5) == 0 {
tt = time.Unix(tsec, int64(tnsec)).UTC()
return
}
// In stdlib time.Parse, when a date is parsed without a zone name, it uses "" as zone name.
// However, we need name here, so it can be shown when time is printed.
// Zone name is in form: UTC-08:00.
// Note that Go Libs do not give access to dst flag, so we ignore dst bits
i2 = i + 2
tz = bigen.Uint16(bs[i:i2])
i = i2
// sign extend sign bit into top 2 MSB (which were dst bits):
if tz&(1<<13) == 0 { // positive
tz = tz & 0x3fff //clear 2 MSBs: dst bits
} else { // negative
tz = tz | 0xc000 //set 2 MSBs: dst bits
//tzname[3] = '-' (TODO: verify. this works here)
}
tzint := int16(tz)
if tzint == 0 {
tt = time.Unix(tsec, int64(tnsec)).UTC()
} else {
// For Go Time, do not use a descriptive timezone.
// It's unnecessary, and makes it harder to do a reflect.DeepEqual.
// The Offset already tells what the offset should be, if not on UTC and unknown zone name.
// var zoneName = timeLocUTCName(tzint)
tt = time.Unix(tsec, int64(tnsec)).In(time.FixedZone("", int(tzint)*60))
}
return
}
func timeLocUTCName(tzint int16) string {
if tzint == 0 {
return "UTC"
}
var tzname = []byte("UTC+00:00")
//tzname := fmt.Sprintf("UTC%s%02d:%02d", tzsign, tz/60, tz%60) //perf issue using Sprintf. inline below.
//tzhr, tzmin := tz/60, tz%60 //faster if u convert to int first
var tzhr, tzmin int16
if tzint < 0 {
tzname[3] = '-' // (TODO: verify. this works here)
tzhr, tzmin = -tzint/60, (-tzint)%60
} else {
tzhr, tzmin = tzint/60, tzint%60
}
tzname[4] = timeDigits[tzhr/10]
tzname[5] = timeDigits[tzhr%10]
tzname[7] = timeDigits[tzmin/10]
tzname[8] = timeDigits[tzmin%10]
return string(tzname)
//return time.FixedZone(string(tzname), int(tzint)*60)
}

362
vendor/github.com/hashicorp/golang-lru/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,362 @@
Mozilla Public License, version 2.0
1. Definitions
1.1. "Contributor"
means each individual or legal entity that creates, contributes to the
creation of, or owns Covered Software.
1.2. "Contributor Version"
means the combination of the Contributions of others (if any) used by a
Contributor and that particular Contributor's Contribution.
1.3. "Contribution"
means Covered Software of a particular Contributor.
1.4. "Covered Software"
means Source Code Form to which the initial Contributor has attached the
notice in Exhibit A, the Executable Form of such Source Code Form, and
Modifications of such Source Code Form, in each case including portions
thereof.
1.5. "Incompatible With Secondary Licenses"
means
a. that the initial Contributor has attached the notice described in
Exhibit B to the Covered Software; or
b. that the Covered Software was made available under the terms of
version 1.1 or earlier of the License, but not also under the terms of
a Secondary License.
1.6. "Executable Form"
means any form of the work other than Source Code Form.
1.7. "Larger Work"
means a work that combines Covered Software with other material, in a
separate file or files, that is not Covered Software.
1.8. "License"
means this document.
1.9. "Licensable"
means having the right to grant, to the maximum extent possible, whether
at the time of the initial grant or subsequently, any and all of the
rights conveyed by this License.
1.10. "Modifications"
means any of the following:
a. any file in Source Code Form that results from an addition to,
deletion from, or modification of the contents of Covered Software; or
b. any new file in Source Code Form that contains any Covered Software.
1.11. "Patent Claims" of a Contributor
means any patent claim(s), including without limitation, method,
process, and apparatus claims, in any patent Licensable by such
Contributor that would be infringed, but for the grant of the License,
by the making, using, selling, offering for sale, having made, import,
or transfer of either its Contributions or its Contributor Version.
1.12. "Secondary License"
means either the GNU General Public License, Version 2.0, the GNU Lesser
General Public License, Version 2.1, the GNU Affero General Public
License, Version 3.0, or any later versions of those licenses.
1.13. "Source Code Form"
means the form of the work preferred for making modifications.
1.14. "You" (or "Your")
means an individual or a legal entity exercising rights under this
License. For legal entities, "You" includes any entity that controls, is
controlled by, or is under common control with You. For purposes of this
definition, "control" means (a) the power, direct or indirect, to cause
the direction or management of such entity, whether by contract or
otherwise, or (b) ownership of more than fifty percent (50%) of the
outstanding shares or beneficial ownership of such entity.
2. License Grants and Conditions
2.1. Grants
Each Contributor hereby grants You a world-wide, royalty-free,
non-exclusive license:
a. under intellectual property rights (other than patent or trademark)
Licensable by such Contributor to use, reproduce, make available,
modify, display, perform, distribute, and otherwise exploit its
Contributions, either on an unmodified basis, with Modifications, or
as part of a Larger Work; and
b. under Patent Claims of such Contributor to make, use, sell, offer for
sale, have made, import, and otherwise transfer either its
Contributions or its Contributor Version.
2.2. Effective Date
The licenses granted in Section 2.1 with respect to any Contribution
become effective for each Contribution on the date the Contributor first
distributes such Contribution.
2.3. Limitations on Grant Scope
The licenses granted in this Section 2 are the only rights granted under
this License. No additional rights or licenses will be implied from the
distribution or licensing of Covered Software under this License.
Notwithstanding Section 2.1(b) above, no patent license is granted by a
Contributor:
a. for any code that a Contributor has removed from Covered Software; or
b. for infringements caused by: (i) Your and any other third party's
modifications of Covered Software, or (ii) the combination of its
Contributions with other software (except as part of its Contributor
Version); or
c. under Patent Claims infringed by Covered Software in the absence of
its Contributions.
This License does not grant any rights in the trademarks, service marks,
or logos of any Contributor (except as may be necessary to comply with
the notice requirements in Section 3.4).
2.4. Subsequent Licenses
No Contributor makes additional grants as a result of Your choice to
distribute the Covered Software under a subsequent version of this
License (see Section 10.2) or under the terms of a Secondary License (if
permitted under the terms of Section 3.3).
2.5. Representation
Each Contributor represents that the Contributor believes its
Contributions are its original creation(s) or it has sufficient rights to
grant the rights to its Contributions conveyed by this License.
2.6. Fair Use
This License is not intended to limit any rights You have under
applicable copyright doctrines of fair use, fair dealing, or other
equivalents.
2.7. Conditions
Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in
Section 2.1.
3. Responsibilities
3.1. Distribution of Source Form
All distribution of Covered Software in Source Code Form, including any
Modifications that You create or to which You contribute, must be under
the terms of this License. You must inform recipients that the Source
Code Form of the Covered Software is governed by the terms of this
License, and how they can obtain a copy of this License. You may not
attempt to alter or restrict the recipients' rights in the Source Code
Form.
3.2. Distribution of Executable Form
If You distribute Covered Software in Executable Form then:
a. such Covered Software must also be made available in Source Code Form,
as described in Section 3.1, and You must inform recipients of the
Executable Form how they can obtain a copy of such Source Code Form by
reasonable means in a timely manner, at a charge no more than the cost
of distribution to the recipient; and
b. You may distribute such Executable Form under the terms of this
License, or sublicense it under different terms, provided that the
license for the Executable Form does not attempt to limit or alter the
recipients' rights in the Source Code Form under this License.
3.3. Distribution of a Larger Work
You may create and distribute a Larger Work under terms of Your choice,
provided that You also comply with the requirements of this License for
the Covered Software. If the Larger Work is a combination of Covered
Software with a work governed by one or more Secondary Licenses, and the
Covered Software is not Incompatible With Secondary Licenses, this
License permits You to additionally distribute such Covered Software
under the terms of such Secondary License(s), so that the recipient of
the Larger Work may, at their option, further distribute the Covered
Software under the terms of either this License or such Secondary
License(s).
3.4. Notices
You may not remove or alter the substance of any license notices
(including copyright notices, patent notices, disclaimers of warranty, or
limitations of liability) contained within the Source Code Form of the
Covered Software, except that You may alter any license notices to the
extent required to remedy known factual inaccuracies.
3.5. Application of Additional Terms
You may choose to offer, and to charge a fee for, warranty, support,
indemnity or liability obligations to one or more recipients of Covered
Software. However, You may do so only on Your own behalf, and not on
behalf of any Contributor. You must make it absolutely clear that any
such warranty, support, indemnity, or liability obligation is offered by
You alone, and You hereby agree to indemnify every Contributor for any
liability incurred by such Contributor as a result of warranty, support,
indemnity or liability terms You offer. You may include additional
disclaimers of warranty and limitations of liability specific to any
jurisdiction.
4. Inability to Comply Due to Statute or Regulation
If it is impossible for You to comply with any of the terms of this License
with respect to some or all of the Covered Software due to statute,
judicial order, or regulation then You must: (a) comply with the terms of
this License to the maximum extent possible; and (b) describe the
limitations and the code they affect. Such description must be placed in a
text file included with all distributions of the Covered Software under
this License. Except to the extent prohibited by statute or regulation,
such description must be sufficiently detailed for a recipient of ordinary
skill to be able to understand it.
5. Termination
5.1. The rights granted under this License will terminate automatically if You
fail to comply with any of its terms. However, if You become compliant,
then the rights granted under this License from a particular Contributor
are reinstated (a) provisionally, unless and until such Contributor
explicitly and finally terminates Your grants, and (b) on an ongoing
basis, if such Contributor fails to notify You of the non-compliance by
some reasonable means prior to 60 days after You have come back into
compliance. Moreover, Your grants from a particular Contributor are
reinstated on an ongoing basis if such Contributor notifies You of the
non-compliance by some reasonable means, this is the first time You have
received notice of non-compliance with this License from such
Contributor, and You become compliant prior to 30 days after Your receipt
of the notice.
5.2. If You initiate litigation against any entity by asserting a patent
infringement claim (excluding declaratory judgment actions,
counter-claims, and cross-claims) alleging that a Contributor Version
directly or indirectly infringes any patent, then the rights granted to
You by any and all Contributors for the Covered Software under Section
2.1 of this License shall terminate.
5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user
license agreements (excluding distributors and resellers) which have been
validly granted by You or Your distributors under this License prior to
termination shall survive termination.
6. Disclaimer of Warranty
Covered Software is provided under this License on an "as is" basis,
without warranty of any kind, either expressed, implied, or statutory,
including, without limitation, warranties that the Covered Software is free
of defects, merchantable, fit for a particular purpose or non-infringing.
The entire risk as to the quality and performance of the Covered Software
is with You. Should any Covered Software prove defective in any respect,
You (not any Contributor) assume the cost of any necessary servicing,
repair, or correction. This disclaimer of warranty constitutes an essential
part of this License. No use of any Covered Software is authorized under
this License except under this disclaimer.
7. Limitation of Liability
Under no circumstances and under no legal theory, whether tort (including
negligence), contract, or otherwise, shall any Contributor, or anyone who
distributes Covered Software as permitted above, be liable to You for any
direct, indirect, special, incidental, or consequential damages of any
character including, without limitation, damages for lost profits, loss of
goodwill, work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses, even if such party shall have been
informed of the possibility of such damages. This limitation of liability
shall not apply to liability for death or personal injury resulting from
such party's negligence to the extent applicable law prohibits such
limitation. Some jurisdictions do not allow the exclusion or limitation of
incidental or consequential damages, so this exclusion and limitation may
not apply to You.
8. Litigation
Any litigation relating to this License may be brought only in the courts
of a jurisdiction where the defendant maintains its principal place of
business and such litigation shall be governed by laws of that
jurisdiction, without reference to its conflict-of-law provisions. Nothing
in this Section shall prevent a party's ability to bring cross-claims or
counter-claims.
9. Miscellaneous
This License represents the complete agreement concerning the subject
matter hereof. If any provision of this License is held to be
unenforceable, such provision shall be reformed only to the extent
necessary to make it enforceable. Any law or regulation which provides that
the language of a contract shall be construed against the drafter shall not
be used to construe this License against a Contributor.
10. Versions of the License
10.1. New Versions
Mozilla Foundation is the license steward. Except as provided in Section
10.3, no one other than the license steward has the right to modify or
publish new versions of this License. Each version will be given a
distinguishing version number.
10.2. Effect of New Versions
You may distribute the Covered Software under the terms of the version
of the License under which You originally received the Covered Software,
or under the terms of any subsequent version published by the license
steward.
10.3. Modified Versions
If you create software not governed by this License, and you want to
create a new license for such software, you may create and use a
modified version of this License if you rename the license and remove
any references to the name of the license steward (except to note that
such modified license differs from this License).
10.4. Distributing Source Code Form that is Incompatible With Secondary
Licenses If You choose to distribute Source Code Form that is
Incompatible With Secondary Licenses under the terms of this version of
the License, the notice described in Exhibit B of this License must be
attached.
Exhibit A - Source Code Form License Notice
This Source Code Form is subject to the
terms of the Mozilla Public License, v.
2.0. If a copy of the MPL was not
distributed with this file, You can
obtain one at
http://mozilla.org/MPL/2.0/.
If it is not possible or desirable to put the notice in a particular file,
then You may include the notice in a location (such as a LICENSE file in a
relevant directory) where a recipient would be likely to look for such a
notice.
You may add additional accurate notices of copyright ownership.
Exhibit B - "Incompatible With Secondary Licenses" Notice
This Source Code Form is "Incompatible
With Secondary Licenses", as defined by
the Mozilla Public License, v. 2.0.

468
vendor/github.com/hashicorp/nomad/CHANGELOG.md generated vendored Normal file
View File

@@ -0,0 +1,468 @@
## 0.4.1
__BACKWARDS INCOMPATIBILITIES:__
* telemetry: Operators will have to explicitly opt-in for Nomad client to
publish allocation and node metrics
IMPROVEMENTS:
* core: Allow count 0 on system jobs [GH-1421]
* core: Summarize the current status of registered jobs. [GH-1383, GH-1517]
* core: Gracefully handle short lived outages by holding RPC calls [GH-1403]
* core: Introduce a lost state for allocations that were on Nodes that died
[GH-1516]
* api: client Logs endpoint for streaming task logs [GH-1444]
* api/cli: Support for tailing/streaming files [GH-1404, GH-1420]
* api/server: Support for querying job summaries [GH-1455]
* cli: `nomad logs` command for streaming task logs [GH-1444]
* cli: `nomad status` shows the create time of allocations [GH-1540]
* cli: `nomad plan` exit code indicates if changes will occur [GH-1502]
* cli: status commands support JSON output and go template formating [GH-1503]
* cli: Validate and plan command supports reading from stdin [GH-1460,
GH-1458]
* cli: Allow basic authentication through address and environment variable
[GH-1610]
* cli: `nomad node-status` shows volume name for non-physical volumes instead
of showing 0B used [GH-1538]
* cli: Support retrieving job files using go-getter in the `run`, `plan` and
`validate` command [GH-1511]
* client: Add killing event to task state [GH-1457]
* client: Fingerprint network speed on Windows [GH-1443]
* discovery: Support for initial check status [GH-1599]
* discovery: Support for query params in health check urls [GH-1562]
* driver/docker: Allow working directory to be configured [GH-1513]
* driver/docker: Remove docker volumes when removing container [GH-1519]
* driver/docker: Set windows containers network mode to nat by default
[GH-1521]
* driver/exec: Allow chroot environment to be configurable [GH-1518]
* driver/qemu: Allows users to pass extra args to the qemu driver [GH-1596]
* telemetry: Circonus integration for telemetry metrics [GH-1459]
* telemetry: Allow operators to opt-in for publishing metrics [GH-1501]
BUG FIXES:
* agent: Reload agent configuration on SIGHUP [GH-1566]
* core: Sanitize empty slices/maps in jobs to avoid incorrect create/destroy
updates [GH-1434]
* core: Fix race in which a Node registers and doesn't receive system jobs
[GH-1456]
* core: Fix issue in which Nodes with large amount of reserved ports would
casue dynamic port allocations to fail [GH-1526]
* core: Fix a condition in which old batch allocations could get updated even
after terminal. In a rare case this could cause a server panic [GH-1471]
* core: Do not update the Job attached to Allocations that have been marked
terminal [GH-1508]
* agent: Fix advertise address when using IPv6 [GH-1465]
* cli: Fix node-status when using IPv6 advertise address [GH-1465]
* client: Task start errors adhere to restart policy mode [GH-1405]
* client: Reregister with servers if node is unregistered [GH-1593]
* client: Killing an allocation doesn't cause allocation stats to block
[GH-1454]
* driver/docker: Disable swap on docker driver [GH-1480]
* driver/docker: Fix improper gating on priviledged mode [GH-1506]
* driver/docker: Default network type is "nat" on Windows [GH-1521]
* driver/docker: Cleanup created volume when destroying container [GH-1519]
* driver/rkt: Set host environment variables [GH-1581]
* driver/rkt: Validate the command and trust_prefix configs [GH-1493]
* plan: Plan on system jobs discounts nodes that do not meet required
constraints [GH-1568]
## 0.4.0
__BACKWARDS INCOMPATIBILITIES:__
* api: Tasks are no longer allowed to have slashes in their name [GH-1210]
* cli: Remove the eval-monitor command. Users should switch to `nomad
eval-status -monitor`.
* config: Consul configuration has been moved from client options map to
consul block under client configuration
* driver/docker: Enabled SSL by default for pulling images from docker
registries. [GH-1336]
IMPROVEMENTS:
* core: Scheduler reuses blocked evaluations to avoid unbounded creation of
evaluations under high contention [GH-1199]
* core: Scheduler stores placement failures in evaluations, no longer
generating failed allocations for debug information [GH-1188]
* api: Faster JSON response encoding [GH-1182]
* api: Gzip compress HTTP API requests [GH-1203]
* api: Plan api introduced for the Job endpoint [GH-1168]
* api: Job endpoint can enforce Job Modify Index to ensure job is being
modified from a known state [GH-1243]
* api/client: Add resource usage APIs for retrieving tasks/allocations/host
resource usage [GH-1189]
* cli: Faster when displaying large amounts ouptuts [GH-1362]
* cli: Deprecate `eval-monitor` and introduce `eval-status` [GH-1206]
* cli: Unify the `fs` family of commands to be a single command [GH-1150]
* cli: Introduce `nomad plan` to dry-run a job through the scheduler and
determine its effects [GH-1181]
* cli: node-status command displays host resource usage and allocation
resources [GH-1261]
* cli: Region flag and environment variable introduced to set region
forwarding. Automatic region forwarding for run and plan [GH-1237]
* client: If Consul is available, automatically bootstrap Nomad Client
using the `_nomad` service in Consul. Nomad Servers now register
themselves with Consul to make this possible. [GH-1201]
* drivers: Qemu and Java can be run without an artifact being download. Useful
if the artifact exists inside a chrooted directory [GH-1262]
* driver/docker: Added a client options to set SELinux labels for container
bind mounts. [GH-788]
* driver/docker: Enabled SSL by default for pulling images from docker
registries. [GH-1336]
* server: If Consul is available, automatically bootstrap Nomad Servers
using the `_nomad` service in Consul. [GH-1276]
BUG FIXES:
* core: Improve garbage collection of allocations and nodes [GH-1256]
* core: Fix a potential deadlock if establishing leadership fails and is
retried [GH-1231]
* core: Do not restart successful batch jobs when the node is removed/drained
[GH-1205]
* core: Fix an issue in which the scheduler could be invoked with insufficient
state [GH-1339]
* core: Updated User, Meta or Resources in a task cause create/destroy updates
[GH-1128, GH-1153]
* core: Fix blocked evaluations being run without properly accounting for
priority [GH-1183]
* api: Tasks are no longer allowed to have slashes in their name [GH-1210]
* client: Delete tmp files used to communicate with execcutor [GH-1241]
* client: Prevent the client from restoring with incorrect task state [GH-1294]
* discovery: Ensure service and check names are unique [GH-1143, GH-1144]
* driver/docker: Ensure docker client doesn't time out after a minute.
[GH-1184]
* driver/java: Fix issue in which Java on darwin attempted to chroot [GH-1262]
* driver/docker: Fix issue in which logs could be spliced [GH-1322]
## 0.3.2 (April 22, 2016)
IMPROVEMENTS:
* core: Garbage collection partitioned to avoid system delays [GH-1012]
* core: Allow count zero task groups to enable blue/green deploys [GH-931]
* core: Validate driver configurations when submitting jobs [GH-1062, GH-1089]
* core: Job Deregister forces an evaluation for the job even if it doesn't
exist [GH-981]
* core: Rename successfully finished allocations to "Complete" rather than
"Dead" for clarity [GH-975]
* cli: `alloc-status` explains restart decisions [GH-984]
* cli: `node-drain -self` drains the local node [GH-1068]
* cli: `node-status -self` queries the local node [GH-1004]
* cli: Destructive commands now require confirmation [GH-983]
* cli: `alloc-status` display is less verbose by default [GH-946]
* cli: `server-members` displays the current leader in each region [GH-935]
* cli: `run` has an `-output` flag to emit a JSON version of the job [GH-990]
* cli: New `inspect` command to display a submitted job's specification
[GH-952]
* cli: `node-status` display is less verbose by default and shows a node's
total resources [GH-946]
* client: `artifact` source can be interpreted [GH-1070]
* client: Add IP and Port environment variables [GH-1099]
* client: Nomad fingerprinter to detect client's version [GH-965]
* client: Tasks can interpret Meta set in the task group and job [GH-985]
* client: All tasks in a task group are killed when a task fails [GH-962]
* client: Pass environment variables from host to exec based tasks [GH-970]
* client: Allow task's to be run as particular user [GH-950, GH-978]
* client: `artifact` block now supports downloading paths relative to the
task's directory [GH-944]
* docker: Timeout communications with Docker Daemon to avoid deadlocks with
misbehaving Docker Daemon [GH-1117]
* discovery: Support script based health checks [GH-986]
* discovery: Allowing registration of services which don't expose ports
[GH-1092]
* driver/docker: Support for `tty` and `interactive` options [GH-1059]
* jobspec: Improved validation of services referencing port labels [GH-1097]
* periodic: Periodic jobs are always evaluated in UTC timezone [GH-1074]
BUG FIXES:
* core: Prevent garbage collection of running batch jobs [GH-989]
* core: Trigger System scheduler when Node drain is disabled [GH-1106]
* core: Fix issue where in-place updated allocation double counted resources
[GH-957]
* core: Fix drained, batched allocations from being migrated indefinitely
[GH-1086]
* client: Garbage collect Docker containers on exit [GH-1071]
* client: Fix common exec failures on CentOS and Amazon Linux [GH-1009]
* client: Fix S3 artifact downloading with IAM credentials [GH-1113]
* client: Fix handling of environment variables containing multiple equal
signs [GH-1115]
## 0.3.1 (March 16, 2016)
__BACKWARDS INCOMPATIBILITIES:__
* Service names that dont conform to RFC-1123 and RFC-2782 will fail
validation. To fix, change service name to conform to the RFCs before
running the job [GH-915]
* Jobs that downloaded artifacts will have to be updated to the new syntax and
be resubmitted. The new syntax consolidates artifacts to the `task` rather
than being duplicated inside each driver config [GH-921]
IMPROVEMENTS:
* cli: Validate job file schemas [GH-900]
* client: Add environment variables for task name, allocation ID/Name/Index
[GH-869, GH-896]
* client: Starting task is retried under the restart policy if the error is
recoverable [GH-859]
* client: Allow tasks to download artifacts, which can be archives, prior to
starting [GH-921]
* config: Validate Nomad configuration files [GH-910]
* config: Client config allows reserving resources [GH-910]
* driver/docker: Support for ECR [GH-858]
* driver/docker: Periodic Fingerprinting [GH-893]
* driver/docker: Preventing port reservation for log collection on Unix platforms [GH-897]
* driver/rkt: Pass DNS information to rkt driver [GH-892]
* jobspec: Require RFC-1123 and RFC-2782 valid service names [GH-915]
BUG FIXES:
* core: No longer cancel evaluations that are delayed in the plan queue
[GH-884]
* api: Guard client/fs/ APIs from being accessed on a non-client node [GH-890]
* client: Allow dashes in variable names during interprelation [GH-857]
* client: Updating kill timeout adheres to operator specified maximum value [GH-878]
* client: Fix a case in which clients would pull but not run allocations
[GH-906]
* consul: Remove concurrent map access [GH-874]
* driver/exec: Stopping tasks with more than one pid in a cgroup [GH-855]
* executor/linux: Add /run/resolvconf/ to chroot so DNS works [GH-905]
## 0.3.0 (February 25, 2016)
__BACKWARDS INCOMPATIBILITIES:__
* Stdout and Stderr log files of tasks have moved from task/local to
alloc/logs [GH-851]
* Any users of the runtime environment variable `$NOMAD_PORT_` will need to
update to the new `${NOMAD_ADDR_}` varriable [GH-704]
* Service names that include periods will fail validation. To fix, remove any
periods from the service name before running the job [GH-770]
* Task resources are now validated and enforce minimum resources. If a job
specifies resources below the minimum they will need to be updated [GH-739]
* Node ID is no longer specifiable. For users who have set a custom Node
ID, the node should be drained before Nomad is updated and the data_dir
should be deleted before starting for the first time [GH-675]
* Users of custom restart policies should update to the new syntax which adds
a `mode` field. The `mode` can be either `fail` or `delay`. The default for
`batch` and `service` jobs is `fail` and `delay` respectively [GH-594]
* All jobs that interpret variables in constraints or driver configurations
will need to be updated to the new syntax which wraps the interpreted
variable in curly braces. (`$node.class` becomes `${node.class}`) [GH-760]
IMPROVEMENTS:
* core: Populate job status [GH-663]
* core: Cgroup fingerprinter [GH-712]
* core: Node class constraint [GH-618]
* core: User specifiable kill timeout [GH-624]
* core: Job queueing via blocked evaluations [GH-726]
* core: Only reschedule failed batch allocations [GH-746]
* core: Add available nodes by DC to AllocMetrics [GH-619]
* core: Improve scheduler retry logic under contention [GH-787]
* core: Computed node class and stack optimization [GH-691, GH-708]
* core: Improved restart policy with more user configuration [GH-594]
* core: Periodic specification for jobs [GH-540, GH-657, GH-659, GH-668]
* core: Batch jobs are garbage collected from the Nomad Servers [GH-586]
* core: Free half the CPUs on leader node for use in plan queue and evaluation
broker [GH-812]
* core: Seed random number generator used to randomize node traversal order
during scheduling [GH-808]
* core: Performance improvements [GH-823, GH-825, GH-827, GH-830, GH-832,
GH-833, GH-834, GH-839]
* core/api: System garbage collection endpoint [GH-828]
* core/api: Allow users to set arbitrary headers via agent config [GH-699]
* core/cli: Prefix based lookups of allocs/nodes/evals/jobs [GH-575]
* core/cli: Print short identifiers and UX cleanup [GH-675, GH-693, GH-692]
* core/client: Client pulls minimum set of required allocations [GH-731]
* cli: Output of agent-info is sorted [GH-617]
* cli: Eval monitor detects zero wait condition [GH-776]
* cli: Ability to navigate allocation directories [GH-709, GH-798]
* client: Batch allocation updates to the server [GH-835]
* client: Log rotation for all drivers [GH-685, GH-763, GH-819]
* client: Only download artifacts from http, https, and S3 [GH-841]
* client: Create a tmp/ directory inside each task directory [GH-757]
* client: Store when an allocation was received by the client [GH-821]
* client: Heartbeating and saving state resilient under high load [GH-811]
* client: Handle updates to tasks Restart Policy and KillTimeout [GH-751]
* client: Killing a driver handle is retried with an exponential backoff
[GH-809]
* client: Send Node to server when periodic fingerprinters change Node
attributes/metadata [GH-749]
* client/api: File-system access to allocation directories [GH-669]
* drivers: Validate the "command" field contains a single value [GH-842]
* drivers: Interpret Nomad variables in environment variables/args [GH-653]
* driver/rkt: Add support for CPU/Memory isolation [GH-610]
* driver/rkt: Add support for mounting alloc/task directory [GH-645]
* driver/docker: Support for .dockercfg based auth for private registries
[GH-773]
BUG FIXES:
* core: Node drain could only be partially applied [GH-750]
* core: Fix panic when eval Ack occurs at delivery limit [GH-790]
* cli: Handle parsing of un-named ports [GH-604]
* cli: Enforce absolute paths for data directories [GH-622]
* client: Cleanup of the allocation directory [GH-755]
* client: Improved stability under high contention [GH-789]
* client: Handle non-200 codes when parsing AWS metadata [GH-614]
* client: Unmounted of shared alloc dir when client is rebooted [GH-755]
* client/consul: Service name changes handled properly [GH-766]
* driver/rkt: handle broader format of rkt version outputs [GH-745]
* driver/qemu: failed to load image and kvm accelerator fixes [GH-656]
## 0.2.3 (December 17, 2015)
BUG FIXES:
* core: Task States not being properly updated [GH-600]
* client: Fixes for user lookup to support CoreOS [GH-591]
* discovery: Using a random prefix for nomad managed services [GH-579]
* discovery: De-Registering Tasks while Nomad sleeps before failed tasks are
restarted.
* discovery: Fixes for service registration when multiple allocations are bin
packed on a node [GH-583]
* configuration: Sort configuration files [GH-588]
* cli: RetryInterval was not being applied properly [GH-601]
## 0.2.2 (December 11, 2015)
IMPROVEMENTS:
* core: Enable `raw_exec` driver in dev mode [GH-558]
* cli: Server join/retry-join command line and config options [GH-527]
* cli: Nomad reports which config files are loaded at start time, or if none
are loaded [GH-536], [GH-553]
BUG FIXES:
* core: Send syslog to `LOCAL0` by default as previously documented [GH-547]
* client: remove all calls to default logger [GH-570]
* consul: Nomad is less noisy when Consul is not running [GH-567]
* consul: Nomad only deregisters services that it created [GH-568]
* driver/exec: Shutdown a task now sends the interrupt signal first to the
process before forcefully killing it. [GH-543]
* driver/docker: Docker driver no longer leaks unix domain socket connections
[GH-556]
* fingerprint/network: Now correctly detects interfaces on Windows [GH-382]
## 0.2.1 (November 28, 2015)
IMPROVEMENTS:
* core: Can specify a whitelist for activating drivers [GH-467]
* core: Can specify a whitelist for activating fingerprinters [GH-488]
* core/api: Can list all known regions in the cluster [GH-495]
* client/spawn: spawn package tests made portable (work on Windows) [GH-442]
* client/executor: executor package tests made portable (work on Windows) [GH-497]
* client/driver: driver package tests made portable (work on windows) [GH-502]
* client/discovery: Added more consul client api configuration options [GH-503]
* driver/docker: Added TLS client options to the config file [GH-480]
* jobspec: More flexibility in naming Services [GH-509]
BUG FIXES:
* core: Shared reference to DynamicPorts caused port conflicts when scheduling
count > 1 [GH-494]
* client/restart policy: Not restarting Batch Jobs if the exit code is 0 [GH-491]
* client/service discovery: Make Service IDs unique [GH-479]
* client/service: Fixes update to check definitions and services which are already registered [GH-498]
* driver/docker: Expose the container port instead of the host port [GH-466]
* driver/docker: Support `port_map` for static ports [GH-476]
* driver/docker: Pass 0.2.0-style port environment variables to the docker container [GH-476]
* jobspec: distinct_hosts constraint can be specified as a boolean (previously panicked) [GH-501]
## 0.2.0 (November 18, 2015)
__BACKWARDS INCOMPATIBILITIES:__
* core: HTTP API `/v1/node/<id>/allocations` returns full Allocation and not
stub [GH-402]
* core: Removed weight and hard/soft fields in constraints [GH-351]
* drivers: Qemu and Java driver configurations have been updated to both use
`artifact_source` as the source for external images/jars to be ran
* jobspec: New reserved and dynamic port specification [GH-415]
* jobspec/drivers: Driver configuration supports arbitrary struct to be
passed in jobspec [GH-415]
FEATURES:
* core: Blocking queries supported in API [GH-366]
* core: System Scheduler that runs tasks on every node [GH-287]
* core: Regexp, version and lexical ordering constraints [GH-271]
* core: distinctHost constraint ensures Task Groups are running on distinct
clients [GH-321]
* core: Service block definition with Consul registration [GH-463, GH-460,
GH-458, GH-455, GH-446, GH-425]
* client: GCE Fingerprinting [GH-215]
* client: Restart policy for task groups enforced by the client [GH-369,
GH-393]
* driver/rawexec: Raw Fork/Exec Driver [GH-237]
* driver/rkt: Experimental Rkt Driver [GH-165, GH-247]
* drivers: Add support for downloading external artifacts to execute for
Exec, Raw exec drivers [GH-381]
IMPROVEMENTS:
* core: Configurable Node GC threshold [GH-362]
* core: Overlap plan verification and plan application for increased
throughput [GH-272]
* cli: Output of `alloc-status` also displays task state [GH-424]
* cli: Output of `server-members` is sorted [GH-323]
* cli: Show node attributes in `node-status` [GH-313]
* client/fingerprint: Network fingerprinter detects interface suitable for
use, rather than defaulting to eth0 [GH-334, GH-356]
* client: Client Restore State properly reattaches to tasks and recreates
them as needed [GH-364, GH-380, GH-388, GH-392, GH-394, GH-397, GH-408]
* client: Periodic Fingerprinting [GH-391]
* client: Precise snapshotting of TaskRunner and AllocRunner [GH-403, GH-411]
* client: Task State is tracked by client [GH-416]
* client: Test Skip Detection [GH-221]
* driver/docker: Can now specify auth for docker pull [GH-390]
* driver/docker: Can now specify DNS and DNSSearch options [GH-390]
* driver/docker: Can now specify the container's hostname [GH-426]
* driver/docker: Containers now have names based on the task name. [GH-389]
* driver/docker: Mount task local and alloc directory to docker containers [GH-290]
* driver/docker: Now accepts any value for `network_mode` to support userspace networking plugins in docker 1.9
* driver/java: Pass JVM options in java driver [GH-293, GH-297]
* drivers: Use BlkioWeight rather than BlkioThrottleReadIopsDevice [GH-222]
* jobspec and drivers: Driver configuration supports arbitrary struct to be passed in jobspec [GH-415]
BUG FIXES:
* core: Nomad Client/Server RPC codec encodes strings properly [GH-420]
* core: Reset Nack timer in response to scheduler operations [GH-325]
* core: Scheduler checks for updates to environment variables [GH-327]
* cli: Fix crash when -config was given a directory or empty path [GH-119]
* client/fingerprint: Use correct local interface on OS X [GH-361, GH-365]
* client: Nomad Client doesn't restart failed containers [GH-198]
* client: Reap spawn-daemon process, avoiding a zombie process [GH-240]
* client: Resource exhausted errors because of link-speed zero [GH-146,
GH-205]
* client: Restarting Nomad Client leads to orphaned containers [GH-159]
* driver/docker: Apply SELinux label for mounting directories in docker
[GH-377]
* driver/docker: Docker driver exposes ports when creating container [GH-212,
GH-412]
* driver/docker: Docker driver uses docker environment variables correctly
[GH-407]
* driver/qemu: Qemu fingerprint and tests work on both windows/linux [GH-352]
## 0.1.2 (October 6, 2015)
IMPROVEMENTS:
* client: Nomad client cleans allocations on exit when in dev mode [GH-214]
* drivers: Use go-getter for artifact retrieval, add artifact support to
Exec, Raw Exec drivers [GH-288]
## 0.1.1 (October 5, 2015)
IMPROVEMENTS:
* cli: Nomad Client configurable from command-line [GH-191]
* client/fingerprint: Native IP detection and user specifiable network
interface for fingerprinting [GH-189]
* driver/docker: Docker networking mode is configurable [GH-184]
* drivers: Set task environment variables [GH-206]
BUG FIXES:
* client/fingerprint: Network fingerprinting failed if default network
interface did not exist [GH-189]
* client: Fixed issue where network resources throughput would be set to 0
MBits if the link speed could not be determined [GH-205]
* client: Improved detection of Nomad binary [GH-181]
* driver/docker: Docker dynamic port mapping were not being set properly
[GH-199]
## 0.1.0 (September 28, 2015)
* Initial release

85
vendor/github.com/hashicorp/nomad/GNUmakefile generated vendored Normal file
View File

@@ -0,0 +1,85 @@
PACKAGES = $(shell go list ./... | grep -v '/vendor/')
VETARGS?=-asmdecl -atomic -bool -buildtags -copylocks -methods \
-nilfunc -printf -rangeloops -shift -structtags -unsafeptr
EXTERNAL_TOOLS=\
github.com/kardianos/govendor \
github.com/mitchellh/gox \
golang.org/x/tools/cmd/cover \
github.com/axw/gocov/gocov \
gopkg.in/matm/v1/gocov-html \
github.com/ugorji/go/codec/codecgen
GOFILES_NOVENDOR = $(shell find . -type f -name '*.go' -not -path "./vendor/*")
all: test
dev: format generate
@NOMAD_DEV=1 sh -c "'$(PWD)/scripts/build.sh'"
bin: generate
@sh -c "'$(PWD)/scripts/build.sh'"
release:
@$(MAKE) bin
cov:
gocov test ./... | gocov-html > /tmp/coverage.html
open /tmp/coverage.html
test: generate
@echo "--> Running go fmt" ;
@if [ -n "`go fmt ${PACKAGES}`" ]; then \
echo "[ERR] go fmt updated formatting. Please commit formatted code first."; \
exit 1; \
fi
@sh -c "'$(PWD)/scripts/test.sh'"
@$(MAKE) vet
cover:
go list ./... | xargs -n1 go test --cover
format:
@echo "--> Running go fmt"
@go fmt $(PACKAGES)
generate:
@echo "--> Running go generate"
@go generate $(PACKAGES)
@sed -e 's|github.com/hashicorp/nomad/vendor/github.com/ugorji/go/codec|github.com/ugorji/go/codec|' nomad/structs/structs.generated.go >> structs.gen.tmp
@mv structs.gen.tmp nomad/structs/structs.generated.go
vet:
@go tool vet 2>/dev/null ; if [ $$? -eq 3 ]; then \
go get golang.org/x/tools/cmd/vet; \
fi
@echo "--> Running go tool vet $(VETARGS) ${GOFILES_NOVENDOR}"
@go tool vet $(VETARGS) ${GOFILES_NOVENDOR} ; if [ $$? -eq 1 ]; then \
echo ""; \
echo "[LINT] Vet found suspicious constructs. Please check the reported constructs"; \
echo "and fix them if necessary before submitting the code for review."; \
fi
@git grep -n `echo "log"".Print"` | grep -v 'vendor/' ; if [ $$? -eq 0 ]; then \
echo "[LINT] Found "log"".Printf" calls. These should use Nomad's logger instead."; \
fi
web:
./scripts/website_run.sh
web-push:
./scripts/website_push.sh
# bootstrap the build by downloading additional tools
bootstrap:
@for tool in $(EXTERNAL_TOOLS) ; do \
echo "Installing $$tool" ; \
go get $$tool; \
done
install: bin/nomad
install -o root -g wheel -m 0755 ./bin/nomad /usr/local/bin/nomad
travis:
@sh -c "'$(PWD)/scripts/travis.sh'"
.PHONY: all bin cov integ test vet web web-push test-nodep

18
vendor/github.com/hashicorp/nomad/ISSUE_TEMPLATE.md generated vendored Normal file
View File

@@ -0,0 +1,18 @@
If you have a question, prepend your issue with `[question]` or preferably use the [nomad mailing list](https://www.nomadproject.io/community.html).
If filing a bug please include the following:
### Nomad version
Output from `nomad version`
### Operating system and Environment details
### Issue
### Reproduction steps
### Nomad Server logs (if appropriate)
### Nomad Client logs (if appropriate)
### Job file (if appropriate)

363
vendor/github.com/hashicorp/nomad/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,363 @@
Mozilla Public License, version 2.0
1. Definitions
1.1. "Contributor"
means each individual or legal entity that creates, contributes to the
creation of, or owns Covered Software.
1.2. "Contributor Version"
means the combination of the Contributions of others (if any) used by a
Contributor and that particular Contributor's Contribution.
1.3. "Contribution"
means Covered Software of a particular Contributor.
1.4. "Covered Software"
means Source Code Form to which the initial Contributor has attached the
notice in Exhibit A, the Executable Form of such Source Code Form, and
Modifications of such Source Code Form, in each case including portions
thereof.
1.5. "Incompatible With Secondary Licenses"
means
a. that the initial Contributor has attached the notice described in
Exhibit B to the Covered Software; or
b. that the Covered Software was made available under the terms of
version 1.1 or earlier of the License, but not also under the terms of
a Secondary License.
1.6. "Executable Form"
means any form of the work other than Source Code Form.
1.7. "Larger Work"
means a work that combines Covered Software with other material, in a
separate file or files, that is not Covered Software.
1.8. "License"
means this document.
1.9. "Licensable"
means having the right to grant, to the maximum extent possible, whether
at the time of the initial grant or subsequently, any and all of the
rights conveyed by this License.
1.10. "Modifications"
means any of the following:
a. any file in Source Code Form that results from an addition to,
deletion from, or modification of the contents of Covered Software; or
b. any new file in Source Code Form that contains any Covered Software.
1.11. "Patent Claims" of a Contributor
means any patent claim(s), including without limitation, method,
process, and apparatus claims, in any patent Licensable by such
Contributor that would be infringed, but for the grant of the License,
by the making, using, selling, offering for sale, having made, import,
or transfer of either its Contributions or its Contributor Version.
1.12. "Secondary License"
means either the GNU General Public License, Version 2.0, the GNU Lesser
General Public License, Version 2.1, the GNU Affero General Public
License, Version 3.0, or any later versions of those licenses.
1.13. "Source Code Form"
means the form of the work preferred for making modifications.
1.14. "You" (or "Your")
means an individual or a legal entity exercising rights under this
License. For legal entities, "You" includes any entity that controls, is
controlled by, or is under common control with You. For purposes of this
definition, "control" means (a) the power, direct or indirect, to cause
the direction or management of such entity, whether by contract or
otherwise, or (b) ownership of more than fifty percent (50%) of the
outstanding shares or beneficial ownership of such entity.
2. License Grants and Conditions
2.1. Grants
Each Contributor hereby grants You a world-wide, royalty-free,
non-exclusive license:
a. under intellectual property rights (other than patent or trademark)
Licensable by such Contributor to use, reproduce, make available,
modify, display, perform, distribute, and otherwise exploit its
Contributions, either on an unmodified basis, with Modifications, or
as part of a Larger Work; and
b. under Patent Claims of such Contributor to make, use, sell, offer for
sale, have made, import, and otherwise transfer either its
Contributions or its Contributor Version.
2.2. Effective Date
The licenses granted in Section 2.1 with respect to any Contribution
become effective for each Contribution on the date the Contributor first
distributes such Contribution.
2.3. Limitations on Grant Scope
The licenses granted in this Section 2 are the only rights granted under
this License. No additional rights or licenses will be implied from the
distribution or licensing of Covered Software under this License.
Notwithstanding Section 2.1(b) above, no patent license is granted by a
Contributor:
a. for any code that a Contributor has removed from Covered Software; or
b. for infringements caused by: (i) Your and any other third party's
modifications of Covered Software, or (ii) the combination of its
Contributions with other software (except as part of its Contributor
Version); or
c. under Patent Claims infringed by Covered Software in the absence of
its Contributions.
This License does not grant any rights in the trademarks, service marks,
or logos of any Contributor (except as may be necessary to comply with
the notice requirements in Section 3.4).
2.4. Subsequent Licenses
No Contributor makes additional grants as a result of Your choice to
distribute the Covered Software under a subsequent version of this
License (see Section 10.2) or under the terms of a Secondary License (if
permitted under the terms of Section 3.3).
2.5. Representation
Each Contributor represents that the Contributor believes its
Contributions are its original creation(s) or it has sufficient rights to
grant the rights to its Contributions conveyed by this License.
2.6. Fair Use
This License is not intended to limit any rights You have under
applicable copyright doctrines of fair use, fair dealing, or other
equivalents.
2.7. Conditions
Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in
Section 2.1.
3. Responsibilities
3.1. Distribution of Source Form
All distribution of Covered Software in Source Code Form, including any
Modifications that You create or to which You contribute, must be under
the terms of this License. You must inform recipients that the Source
Code Form of the Covered Software is governed by the terms of this
License, and how they can obtain a copy of this License. You may not
attempt to alter or restrict the recipients' rights in the Source Code
Form.
3.2. Distribution of Executable Form
If You distribute Covered Software in Executable Form then:
a. such Covered Software must also be made available in Source Code Form,
as described in Section 3.1, and You must inform recipients of the
Executable Form how they can obtain a copy of such Source Code Form by
reasonable means in a timely manner, at a charge no more than the cost
of distribution to the recipient; and
b. You may distribute such Executable Form under the terms of this
License, or sublicense it under different terms, provided that the
license for the Executable Form does not attempt to limit or alter the
recipients' rights in the Source Code Form under this License.
3.3. Distribution of a Larger Work
You may create and distribute a Larger Work under terms of Your choice,
provided that You also comply with the requirements of this License for
the Covered Software. If the Larger Work is a combination of Covered
Software with a work governed by one or more Secondary Licenses, and the
Covered Software is not Incompatible With Secondary Licenses, this
License permits You to additionally distribute such Covered Software
under the terms of such Secondary License(s), so that the recipient of
the Larger Work may, at their option, further distribute the Covered
Software under the terms of either this License or such Secondary
License(s).
3.4. Notices
You may not remove or alter the substance of any license notices
(including copyright notices, patent notices, disclaimers of warranty, or
limitations of liability) contained within the Source Code Form of the
Covered Software, except that You may alter any license notices to the
extent required to remedy known factual inaccuracies.
3.5. Application of Additional Terms
You may choose to offer, and to charge a fee for, warranty, support,
indemnity or liability obligations to one or more recipients of Covered
Software. However, You may do so only on Your own behalf, and not on
behalf of any Contributor. You must make it absolutely clear that any
such warranty, support, indemnity, or liability obligation is offered by
You alone, and You hereby agree to indemnify every Contributor for any
liability incurred by such Contributor as a result of warranty, support,
indemnity or liability terms You offer. You may include additional
disclaimers of warranty and limitations of liability specific to any
jurisdiction.
4. Inability to Comply Due to Statute or Regulation
If it is impossible for You to comply with any of the terms of this License
with respect to some or all of the Covered Software due to statute,
judicial order, or regulation then You must: (a) comply with the terms of
this License to the maximum extent possible; and (b) describe the
limitations and the code they affect. Such description must be placed in a
text file included with all distributions of the Covered Software under
this License. Except to the extent prohibited by statute or regulation,
such description must be sufficiently detailed for a recipient of ordinary
skill to be able to understand it.
5. Termination
5.1. The rights granted under this License will terminate automatically if You
fail to comply with any of its terms. However, if You become compliant,
then the rights granted under this License from a particular Contributor
are reinstated (a) provisionally, unless and until such Contributor
explicitly and finally terminates Your grants, and (b) on an ongoing
basis, if such Contributor fails to notify You of the non-compliance by
some reasonable means prior to 60 days after You have come back into
compliance. Moreover, Your grants from a particular Contributor are
reinstated on an ongoing basis if such Contributor notifies You of the
non-compliance by some reasonable means, this is the first time You have
received notice of non-compliance with this License from such
Contributor, and You become compliant prior to 30 days after Your receipt
of the notice.
5.2. If You initiate litigation against any entity by asserting a patent
infringement claim (excluding declaratory judgment actions,
counter-claims, and cross-claims) alleging that a Contributor Version
directly or indirectly infringes any patent, then the rights granted to
You by any and all Contributors for the Covered Software under Section
2.1 of this License shall terminate.
5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user
license agreements (excluding distributors and resellers) which have been
validly granted by You or Your distributors under this License prior to
termination shall survive termination.
6. Disclaimer of Warranty
Covered Software is provided under this License on an "as is" basis,
without warranty of any kind, either expressed, implied, or statutory,
including, without limitation, warranties that the Covered Software is free
of defects, merchantable, fit for a particular purpose or non-infringing.
The entire risk as to the quality and performance of the Covered Software
is with You. Should any Covered Software prove defective in any respect,
You (not any Contributor) assume the cost of any necessary servicing,
repair, or correction. This disclaimer of warranty constitutes an essential
part of this License. No use of any Covered Software is authorized under
this License except under this disclaimer.
7. Limitation of Liability
Under no circumstances and under no legal theory, whether tort (including
negligence), contract, or otherwise, shall any Contributor, or anyone who
distributes Covered Software as permitted above, be liable to You for any
direct, indirect, special, incidental, or consequential damages of any
character including, without limitation, damages for lost profits, loss of
goodwill, work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses, even if such party shall have been
informed of the possibility of such damages. This limitation of liability
shall not apply to liability for death or personal injury resulting from
such party's negligence to the extent applicable law prohibits such
limitation. Some jurisdictions do not allow the exclusion or limitation of
incidental or consequential damages, so this exclusion and limitation may
not apply to You.
8. Litigation
Any litigation relating to this License may be brought only in the courts
of a jurisdiction where the defendant maintains its principal place of
business and such litigation shall be governed by laws of that
jurisdiction, without reference to its conflict-of-law provisions. Nothing
in this Section shall prevent a party's ability to bring cross-claims or
counter-claims.
9. Miscellaneous
This License represents the complete agreement concerning the subject
matter hereof. If any provision of this License is held to be
unenforceable, such provision shall be reformed only to the extent
necessary to make it enforceable. Any law or regulation which provides that
the language of a contract shall be construed against the drafter shall not
be used to construe this License against a Contributor.
10. Versions of the License
10.1. New Versions
Mozilla Foundation is the license steward. Except as provided in Section
10.3, no one other than the license steward has the right to modify or
publish new versions of this License. Each version will be given a
distinguishing version number.
10.2. Effect of New Versions
You may distribute the Covered Software under the terms of the version
of the License under which You originally received the Covered Software,
or under the terms of any subsequent version published by the license
steward.
10.3. Modified Versions
If you create software not governed by this License, and you want to
create a new license for such software, you may create and use a
modified version of this License if you rename the license and remove
any references to the name of the license steward (except to note that
such modified license differs from this License).
10.4. Distributing Source Code Form that is Incompatible With Secondary
Licenses If You choose to distribute Source Code Form that is
Incompatible With Secondary Licenses under the terms of this version of
the License, the notice described in Exhibit B of this License must be
attached.
Exhibit A - Source Code Form License Notice
This Source Code Form is subject to the
terms of the Mozilla Public License, v.
2.0. If a copy of the MPL was not
distributed with this file, You can
obtain one at
http://mozilla.org/MPL/2.0/.
If it is not possible or desirable to put the notice in a particular file,
then You may include the notice in a location (such as a LICENSE file in a
relevant directory) where a recipient would be likely to look for such a
notice.
You may add additional accurate notices of copyright ownership.
Exhibit B - "Incompatible With Secondary Licenses" Notice
This Source Code Form is "Incompatible
With Secondary Licenses", as defined by
the Mozilla Public License, v. 2.0.

117
vendor/github.com/hashicorp/nomad/README.md generated vendored Normal file
View File

@@ -0,0 +1,117 @@
Nomad [![Build Status](https://travis-ci.org/hashicorp/nomad.svg)](https://travis-ci.org/hashicorp/nomad)
=========
- Website: https://www.nomadproject.io
- IRC: `#nomad-tool` on Freenode
- Mailing list: [Google Groups](https://groups.google.com/group/nomad-tool)
![Nomad](https://raw.githubusercontent.com/hashicorp/nomad/master/website/source/assets/images/logo-header%402x.png?token=AAkIoLO_y1g3wgHMr3QO-559BN22rN0kks5V_2HpwA%3D%3D)
Nomad is a cluster manager, designed for both long lived services and short
lived batch processing workloads. Developers use a declarative job specification
to submit work, and Nomad ensures constraints are satisfied and resource utilization
is optimized by efficient task packing. Nomad supports all major operating systems
and virtualized, containerized, or standalone applications.
The key features of Nomad are:
* **Docker Support**: Jobs can specify tasks which are Docker containers.
Nomad will automatically run the containers on clients which have Docker
installed, scale up and down based on the number of instances request,
and automatically recover from failures.
* **Multi-Datacenter and Multi-Region Aware**: Nomad is designed to be
a global-scale scheduler. Multiple datacenters can be managed as part
of a larger region, and jobs can be scheduled across datacenters if
requested. Multiple regions join together and federate jobs making it
easy to run jobs anywhere.
* **Operationally Simple**: Nomad runs as a single binary that can be
either a client or server, and is completely self contained. Nomad does
not require any external services for storage or coordination. This means
Nomad combines the features of a resource manager and scheduler in a single
system.
* **Distributed and Highly-Available**: Nomad servers cluster together and
perform leader election and state replication to provide high availability
in the face of failure. The Nomad scheduling engine is optimized for
optimistic concurrency allowing all servers to make scheduling decisions to
maximize throughput.
* **HashiCorp Ecosystem**: Nomad integrates with the entire HashiCorp
ecosystem of tools. Along with all HashiCorp tools, Nomad is designed
in the unix philosophy of doing something specific and doing it well.
Nomad integrates with tools like Packer, Consul, and Terraform to support
building artifacts, service discovery, monitoring and capacity management.
For more information, see the [introduction section](https://www.nomadproject.io/intro)
of the Nomad website.
Getting Started & Documentation
-------------------------------
All documentation is available on the [Nomad website](https://www.nomadproject.io).
Developing Nomad
--------------------
If you wish to work on Nomad itself or any of its built-in systems,
you will first need [Go](https://www.golang.org) installed on your
machine (version 1.5+ is *required*).
**Developing with Vagrant**
There is an included Vagrantfile that can help bootstrap the process. The
created virtual machine is based off of Ubuntu 14, and installs several of the
base libraries that can be used by Nomad.
To use this virtual machine, checkout Nomad and run `vagrant up` from the root
of the repository:
```sh
$ git clone https://github.com/hashicorp/nomad.git
$ cd nomad
$ vagrant up
```
The virtual machine will launch, and a provisioning script will install the
needed dependencies.
**Developing locally**
For local dev first make sure Go is properly installed, including setting up a
[GOPATH](https://golang.org/doc/code.html#GOPATH). After setting up Go, clone this
repository into `$GOPATH/src/github.com/hashicorp/nomad`. Then you can
download the required build tools such as vet, cover, godep etc by bootstrapping
your environment.
```sh
$ make bootstrap
...
```
Afterwards type `make test`. This will run the tests. If this exits with exit status 0,
then everything is working!
```sh
$ make test
...
```
To compile a development version of Nomad, run `make dev`. This will put the
Nomad binary in the `bin` and `$GOPATH/bin` folders:
```sh
$ make dev
...
$ bin/nomad
...
```
To cross-compile Nomad, run `make bin`. This will compile Nomad for multiple
platforms and place the resulting binaries into the `./pkg` directory:
```sh
$ make bin
...
$ ls ./pkg
...
```

137
vendor/github.com/hashicorp/nomad/Vagrantfile generated vendored Normal file
View File

@@ -0,0 +1,137 @@
# -*- mode: ruby -*-
# vi: set ft=ruby :
# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
VAGRANTFILE_API_VERSION = "2"
DEFAULT_CPU_COUNT = 2
$script = <<SCRIPT
GO_VERSION="1.7"
CONSUL_VERSION="0.6.4"
# Install Prereq Packages
sudo apt-get update
sudo apt-get install -y build-essential curl git-core mercurial bzr libpcre3-dev pkg-config zip default-jre qemu libc6-dev-i386 silversearcher-ag jq htop vim unzip
# Setup go, for development of Nomad
SRCROOT="/opt/go"
SRCPATH="/opt/gopath"
# Get the ARCH
ARCH=`uname -m | sed 's|i686|386|' | sed 's|x86_64|amd64|'`
# Install Go
cd /tmp
wget -q https://storage.googleapis.com/golang/go${GO_VERSION}.linux-${ARCH}.tar.gz
tar -xf go${GO_VERSION}.linux-${ARCH}.tar.gz
sudo mv go $SRCROOT
sudo chmod 775 $SRCROOT
sudo chown vagrant:vagrant $SRCROOT
# Setup the GOPATH; even though the shared folder spec gives the working
# directory the right user/group, we need to set it properly on the
# parent path to allow subsequent "go get" commands to work.
sudo mkdir -p $SRCPATH
sudo chown -R vagrant:vagrant $SRCPATH 2>/dev/null || true
# ^^ silencing errors here because we expect this to fail for the shared folder
cat <<EOF >/tmp/gopath.sh
export GOPATH="$SRCPATH"
export GOROOT="$SRCROOT"
export PATH="$SRCROOT/bin:$SRCPATH/bin:\$PATH"
EOF
sudo mv /tmp/gopath.sh /etc/profile.d/gopath.sh
sudo chmod 0755 /etc/profile.d/gopath.sh
source /etc/profile.d/gopath.sh
echo Fetching Consul...
cd /tmp/
wget https://releases.hashicorp.com/consul/${CONSUL_VERSION}/consul_${CONSUL_VERSION}_linux_amd64.zip -O consul.zip
echo Installing Consul...
unzip consul.zip
sudo chmod +x consul
sudo mv consul /usr/bin/consul
# Install Docker
echo deb https://apt.dockerproject.org/repo ubuntu-`lsb_release -c | awk '{print $2}'` main | sudo tee /etc/apt/sources.list.d/docker.list
sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
sudo apt-get update
sudo apt-get install -y docker-engine
# Restart docker to make sure we get the latest version of the daemon if there is an upgrade
sudo service docker restart
# Make sure we can actually use docker as the vagrant user
sudo usermod -aG docker vagrant
# Setup Nomad for development
cd /opt/gopath/src/github.com/hashicorp/nomad && make bootstrap
# Install rkt
bash scripts/install_rkt.sh
# CD into the nomad working directory when we login to the VM
grep "cd /opt/gopath/src/github.com/hashicorp/nomad" ~/.profile || echo "cd /opt/gopath/src/github.com/hashicorp/nomad" >> ~/.profile
SCRIPT
def configureVM(vmCfg, vmParams={
numCPUs: DEFAULT_CPU_COUNT,
}
)
vmCfg.vm.box = "cbednarski/ubuntu-1404"
vmCfg.vm.provision "shell", inline: $script, privileged: false
vmCfg.vm.synced_folder '.', '/opt/gopath/src/github.com/hashicorp/nomad'
# We're going to compile go and run a concurrent system, so give ourselves
# some extra resources. Nomad will have trouble working correctly with <2
# CPUs so we should use at least that many.
cpus = vmParams.fetch(:numCPUs, DEFAULT_CPU_COUNT)
memory = 2048
vmCfg.vm.provider "parallels" do |p, o|
o.vm.box = "parallels/ubuntu-14.04"
p.memory = memory
p.cpus = cpus
end
vmCfg.vm.provider "virtualbox" do |v|
v.memory = memory
v.cpus = cpus
end
["vmware_fusion", "vmware_workstation"].each do |p|
vmCfg.vm.provider p do |v|
v.gui = false
v.memory = memory
v.cpus = cpus
end
end
return vmCfg
end
Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
1.upto(3) do |n|
vmName = "nomad-server%02d" % [n]
isFirstBox = (n == 1)
numCPUs = DEFAULT_CPU_COUNT
if isFirstBox and Object::RUBY_PLATFORM =~ /darwin/i
# Override the max CPUs for the first VM
numCPUs = [numCPUs, (`/usr/sbin/sysctl -n hw.ncpu`.to_i - 1)].max
end
config.vm.define vmName, autostart: isFirstBox, primary: isFirstBox do |vmCfg|
vmCfg.vm.hostname = vmName
vmCfg = configureVM(vmCfg, {:numCPUs => numCPUs})
end
end
1.upto(3) do |n|
vmName = "nomad-client%02d" % [n]
config.vm.define vmName, autostart: false, primary: false do |vmCfg|
vmCfg.vm.hostname = vmName
vmCfg = configureVM(vmCfg)
end
end
end

199
vendor/github.com/hashicorp/nomad/api/agent.go generated vendored Normal file
View File

@@ -0,0 +1,199 @@
package api
import (
"fmt"
"net/url"
)
// Agent encapsulates an API client which talks to Nomad's
// agent endpoints for a specific node.
type Agent struct {
client *Client
// Cache static agent info
nodeName string
datacenter string
region string
}
// Agent returns a new agent which can be used to query
// the agent-specific endpoints.
func (c *Client) Agent() *Agent {
return &Agent{client: c}
}
// Self is used to query the /v1/agent/self endpoint and
// returns information specific to the running agent.
func (a *Agent) Self() (map[string]map[string]interface{}, error) {
var out map[string]map[string]interface{}
// Query the self endpoint on the agent
_, err := a.client.query("/v1/agent/self", &out, nil)
if err != nil {
return nil, fmt.Errorf("failed querying self endpoint: %s", err)
}
// Populate the cache for faster queries
a.populateCache(out)
return out, nil
}
// populateCache is used to insert various pieces of static
// data into the agent handle. This is used during subsequent
// lookups for the same data later on to save the round trip.
func (a *Agent) populateCache(info map[string]map[string]interface{}) {
if a.nodeName == "" {
a.nodeName, _ = info["member"]["Name"].(string)
}
if tags, ok := info["member"]["Tags"].(map[string]interface{}); ok {
if a.datacenter == "" {
a.datacenter, _ = tags["dc"].(string)
}
if a.region == "" {
a.region, _ = tags["region"].(string)
}
}
}
// NodeName is used to query the Nomad agent for its node name.
func (a *Agent) NodeName() (string, error) {
// Return from cache if we have it
if a.nodeName != "" {
return a.nodeName, nil
}
// Query the node name
_, err := a.Self()
return a.nodeName, err
}
// Datacenter is used to return the name of the datacenter which
// the agent is a member of.
func (a *Agent) Datacenter() (string, error) {
// Return from cache if we have it
if a.datacenter != "" {
return a.datacenter, nil
}
// Query the agent for the DC
_, err := a.Self()
return a.datacenter, err
}
// Region is used to look up the region the agent is in.
func (a *Agent) Region() (string, error) {
// Return from cache if we have it
if a.region != "" {
return a.region, nil
}
// Query the agent for the region
_, err := a.Self()
return a.region, err
}
// Join is used to instruct a server node to join another server
// via the gossip protocol. Multiple addresses may be specified.
// We attempt to join all of the hosts in the list. Returns the
// number of nodes successfully joined and any error. If one or
// more nodes have a successful result, no error is returned.
func (a *Agent) Join(addrs ...string) (int, error) {
// Accumulate the addresses
v := url.Values{}
for _, addr := range addrs {
v.Add("address", addr)
}
// Send the join request
var resp joinResponse
_, err := a.client.write("/v1/agent/join?"+v.Encode(), nil, &resp, nil)
if err != nil {
return 0, fmt.Errorf("failed joining: %s", err)
}
if resp.Error != "" {
return 0, fmt.Errorf("failed joining: %s", resp.Error)
}
return resp.NumJoined, nil
}
// Members is used to query all of the known server members
func (a *Agent) Members() ([]*AgentMember, error) {
var resp []*AgentMember
// Query the known members
_, err := a.client.query("/v1/agent/members", &resp, nil)
if err != nil {
return nil, err
}
return resp, nil
}
// ForceLeave is used to eject an existing node from the cluster.
func (a *Agent) ForceLeave(node string) error {
_, err := a.client.write("/v1/agent/force-leave?node="+node, nil, nil, nil)
return err
}
// Servers is used to query the list of servers on a client node.
func (a *Agent) Servers() ([]string, error) {
var resp []string
_, err := a.client.query("/v1/agent/servers", &resp, nil)
if err != nil {
return nil, err
}
return resp, nil
}
// SetServers is used to update the list of servers on a client node.
func (a *Agent) SetServers(addrs []string) error {
// Accumulate the addresses
v := url.Values{}
for _, addr := range addrs {
v.Add("address", addr)
}
_, err := a.client.write("/v1/agent/servers?"+v.Encode(), nil, nil, nil)
return err
}
// joinResponse is used to decode the response we get while
// sending a member join request.
type joinResponse struct {
NumJoined int `json:"num_joined"`
Error string `json:"error"`
}
// AgentMember represents a cluster member known to the agent
type AgentMember struct {
Name string
Addr string
Port uint16
Tags map[string]string
Status string
ProtocolMin uint8
ProtocolMax uint8
ProtocolCur uint8
DelegateMin uint8
DelegateMax uint8
DelegateCur uint8
}
// AgentMembersNameSort implements sort.Interface for []*AgentMembersNameSort
// based on the Name, DC and Region
type AgentMembersNameSort []*AgentMember
func (a AgentMembersNameSort) Len() int { return len(a) }
func (a AgentMembersNameSort) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a AgentMembersNameSort) Less(i, j int) bool {
if a[i].Tags["region"] != a[j].Tags["region"] {
return a[i].Tags["region"] < a[j].Tags["region"]
}
if a[i].Tags["dc"] != a[j].Tags["dc"] {
return a[i].Tags["dc"] < a[j].Tags["dc"]
}
return a[i].Name < a[j].Name
}

136
vendor/github.com/hashicorp/nomad/api/allocations.go generated vendored Normal file
View File

@@ -0,0 +1,136 @@
package api
import (
"fmt"
"sort"
"time"
"github.com/hashicorp/go-cleanhttp"
)
// Allocations is used to query the alloc-related endpoints.
type Allocations struct {
client *Client
}
// Allocations returns a handle on the allocs endpoints.
func (c *Client) Allocations() *Allocations {
return &Allocations{client: c}
}
// List returns a list of all of the allocations.
func (a *Allocations) List(q *QueryOptions) ([]*AllocationListStub, *QueryMeta, error) {
var resp []*AllocationListStub
qm, err := a.client.query("/v1/allocations", &resp, q)
if err != nil {
return nil, nil, err
}
sort.Sort(AllocIndexSort(resp))
return resp, qm, nil
}
func (a *Allocations) PrefixList(prefix string) ([]*AllocationListStub, *QueryMeta, error) {
return a.List(&QueryOptions{Prefix: prefix})
}
// Info is used to retrieve a single allocation.
func (a *Allocations) Info(allocID string, q *QueryOptions) (*Allocation, *QueryMeta, error) {
var resp Allocation
qm, err := a.client.query("/v1/allocation/"+allocID, &resp, q)
if err != nil {
return nil, nil, err
}
return &resp, qm, nil
}
func (a *Allocations) Stats(alloc *Allocation, q *QueryOptions) (*AllocResourceUsage, error) {
node, _, err := a.client.Nodes().Info(alloc.NodeID, q)
if err != nil {
return nil, err
}
if node.HTTPAddr == "" {
return nil, fmt.Errorf("http addr of the node where alloc %q is running is not advertised", alloc.ID)
}
client, err := NewClient(&Config{
Address: fmt.Sprintf("http://%s", node.HTTPAddr),
HttpClient: cleanhttp.DefaultClient(),
})
if err != nil {
return nil, err
}
var resp AllocResourceUsage
_, err = client.query("/v1/client/allocation/"+alloc.ID+"/stats", &resp, nil)
return &resp, err
}
// Allocation is used for serialization of allocations.
type Allocation struct {
ID string
EvalID string
Name string
NodeID string
JobID string
Job *Job
TaskGroup string
Resources *Resources
TaskResources map[string]*Resources
Services map[string]string
Metrics *AllocationMetric
DesiredStatus string
DesiredDescription string
ClientStatus string
ClientDescription string
TaskStates map[string]*TaskState
CreateIndex uint64
ModifyIndex uint64
CreateTime int64
}
// AllocationMetric is used to deserialize allocation metrics.
type AllocationMetric struct {
NodesEvaluated int
NodesFiltered int
NodesAvailable map[string]int
ClassFiltered map[string]int
ConstraintFiltered map[string]int
NodesExhausted int
ClassExhausted map[string]int
DimensionExhausted map[string]int
Scores map[string]float64
AllocationTime time.Duration
CoalescedFailures int
}
// AllocationListStub is used to return a subset of an allocation
// during list operations.
type AllocationListStub struct {
ID string
EvalID string
Name string
NodeID string
JobID string
TaskGroup string
DesiredStatus string
DesiredDescription string
ClientStatus string
ClientDescription string
TaskStates map[string]*TaskState
CreateIndex uint64
ModifyIndex uint64
CreateTime int64
}
// AllocIndexSort reverse sorts allocs by CreateIndex.
type AllocIndexSort []*AllocationListStub
func (a AllocIndexSort) Len() int {
return len(a)
}
func (a AllocIndexSort) Less(i, j int) bool {
return a[i].CreateIndex > a[j].CreateIndex
}
func (a AllocIndexSort) Swap(i, j int) {
a[i], a[j] = a[j], a[i]
}

492
vendor/github.com/hashicorp/nomad/api/api.go generated vendored Normal file
View File

@@ -0,0 +1,492 @@
package api
import (
"bytes"
"compress/gzip"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strconv"
"strings"
"time"
"github.com/hashicorp/go-cleanhttp"
)
// QueryOptions are used to parameterize a query
type QueryOptions struct {
// Providing a datacenter overwrites the region provided
// by the Config
Region string
// AllowStale allows any Nomad server (non-leader) to service
// a read. This allows for lower latency and higher throughput
AllowStale bool
// WaitIndex is used to enable a blocking query. Waits
// until the timeout or the next index is reached
WaitIndex uint64
// WaitTime is used to bound the duration of a wait.
// Defaults to that of the Config, but can be overridden.
WaitTime time.Duration
// If set, used as prefix for resource list searches
Prefix string
// Set HTTP parameters on the query.
Params map[string]string
}
// WriteOptions are used to parameterize a write
type WriteOptions struct {
// Providing a datacenter overwrites the region provided
// by the Config
Region string
}
// QueryMeta is used to return meta data about a query
type QueryMeta struct {
// LastIndex. This can be used as a WaitIndex to perform
// a blocking query
LastIndex uint64
// Time of last contact from the leader for the
// server servicing the request
LastContact time.Duration
// Is there a known leader
KnownLeader bool
// How long did the request take
RequestTime time.Duration
}
// WriteMeta is used to return meta data about a write
type WriteMeta struct {
// LastIndex. This can be used as a WaitIndex to perform
// a blocking query
LastIndex uint64
// How long did the request take
RequestTime time.Duration
}
// HttpBasicAuth is used to authenticate http client with HTTP Basic Authentication
type HttpBasicAuth struct {
// Username to use for HTTP Basic Authentication
Username string
// Password to use for HTTP Basic Authentication
Password string
}
// Config is used to configure the creation of a client
type Config struct {
// Address is the address of the Nomad agent
Address string
// Region to use. If not provided, the default agent region is used.
Region string
// HttpClient is the client to use. Default will be
// used if not provided.
HttpClient *http.Client
// HttpAuth is the auth info to use for http access.
HttpAuth *HttpBasicAuth
// WaitTime limits how long a Watch will block. If not provided,
// the agent default values will be used.
WaitTime time.Duration
}
// DefaultConfig returns a default configuration for the client
func DefaultConfig() *Config {
config := &Config{
Address: "http://127.0.0.1:4646",
HttpClient: cleanhttp.DefaultClient(),
}
if addr := os.Getenv("NOMAD_ADDR"); addr != "" {
config.Address = addr
}
if auth := os.Getenv("NOMAD_HTTP_AUTH"); auth != "" {
var username, password string
if strings.Contains(auth, ":") {
split := strings.SplitN(auth, ":", 2)
username = split[0]
password = split[1]
} else {
username = auth
}
config.HttpAuth = &HttpBasicAuth{
Username: username,
Password: password,
}
}
return config
}
// Client provides a client to the Nomad API
type Client struct {
config Config
}
// NewClient returns a new client
func NewClient(config *Config) (*Client, error) {
// bootstrap the config
defConfig := DefaultConfig()
if config.Address == "" {
config.Address = defConfig.Address
} else if _, err := url.Parse(config.Address); err != nil {
return nil, fmt.Errorf("invalid address '%s': %v", config.Address, err)
}
if config.HttpClient == nil {
config.HttpClient = defConfig.HttpClient
}
client := &Client{
config: *config,
}
return client, nil
}
// SetRegion sets the region to forward API requests to.
func (c *Client) SetRegion(region string) {
c.config.Region = region
}
// request is used to help build up a request
type request struct {
config *Config
method string
url *url.URL
params url.Values
body io.Reader
obj interface{}
}
// setQueryOptions is used to annotate the request with
// additional query options
func (r *request) setQueryOptions(q *QueryOptions) {
if q == nil {
return
}
if q.Region != "" {
r.params.Set("region", q.Region)
}
if q.AllowStale {
r.params.Set("stale", "")
}
if q.WaitIndex != 0 {
r.params.Set("index", strconv.FormatUint(q.WaitIndex, 10))
}
if q.WaitTime != 0 {
r.params.Set("wait", durToMsec(q.WaitTime))
}
if q.Prefix != "" {
r.params.Set("prefix", q.Prefix)
}
for k, v := range q.Params {
r.params.Set(k, v)
}
}
// durToMsec converts a duration to a millisecond specified string
func durToMsec(dur time.Duration) string {
return fmt.Sprintf("%dms", dur/time.Millisecond)
}
// setWriteOptions is used to annotate the request with
// additional write options
func (r *request) setWriteOptions(q *WriteOptions) {
if q == nil {
return
}
if q.Region != "" {
r.params.Set("region", q.Region)
}
}
// toHTTP converts the request to an HTTP request
func (r *request) toHTTP() (*http.Request, error) {
// Encode the query parameters
r.url.RawQuery = r.params.Encode()
// Check if we should encode the body
if r.body == nil && r.obj != nil {
if b, err := encodeBody(r.obj); err != nil {
return nil, err
} else {
r.body = b
}
}
// Create the HTTP request
req, err := http.NewRequest(r.method, r.url.RequestURI(), r.body)
if err != nil {
return nil, err
}
// Optionally configure HTTP basic authentication
if r.url.User != nil {
username := r.url.User.Username()
password, _ := r.url.User.Password()
req.SetBasicAuth(username, password)
} else if r.config.HttpAuth != nil {
req.SetBasicAuth(r.config.HttpAuth.Username, r.config.HttpAuth.Password)
}
req.Header.Add("Accept-Encoding", "gzip")
req.URL.Host = r.url.Host
req.URL.Scheme = r.url.Scheme
req.Host = r.url.Host
return req, nil
}
// newRequest is used to create a new request
func (c *Client) newRequest(method, path string) *request {
base, _ := url.Parse(c.config.Address)
u, _ := url.Parse(path)
r := &request{
config: &c.config,
method: method,
url: &url.URL{
Scheme: base.Scheme,
User: base.User,
Host: base.Host,
Path: u.Path,
},
params: make(map[string][]string),
}
if c.config.Region != "" {
r.params.Set("region", c.config.Region)
}
if c.config.WaitTime != 0 {
r.params.Set("wait", durToMsec(r.config.WaitTime))
}
// Add in the query parameters, if any
for key, values := range u.Query() {
for _, value := range values {
r.params.Add(key, value)
}
}
return r
}
// multiCloser is to wrap a ReadCloser such that when close is called, multiple
// Closes occur.
type multiCloser struct {
reader io.Reader
inorderClose []io.Closer
}
func (m *multiCloser) Close() error {
for _, c := range m.inorderClose {
if err := c.Close(); err != nil {
return err
}
}
return nil
}
func (m *multiCloser) Read(p []byte) (int, error) {
return m.reader.Read(p)
}
// doRequest runs a request with our client
func (c *Client) doRequest(r *request) (time.Duration, *http.Response, error) {
req, err := r.toHTTP()
if err != nil {
return 0, nil, err
}
start := time.Now()
resp, err := c.config.HttpClient.Do(req)
diff := time.Now().Sub(start)
// If the response is compressed, we swap the body's reader.
if resp != nil && resp.Header != nil {
var reader io.ReadCloser
switch resp.Header.Get("Content-Encoding") {
case "gzip":
greader, err := gzip.NewReader(resp.Body)
if err != nil {
return 0, nil, err
}
// The gzip reader doesn't close the wrapped reader so we use
// multiCloser.
reader = &multiCloser{
reader: greader,
inorderClose: []io.Closer{greader, resp.Body},
}
default:
reader = resp.Body
}
resp.Body = reader
}
return diff, resp, err
}
// rawQuery makes a GET request to the specified endpoint but returns just the
// response body.
func (c *Client) rawQuery(endpoint string, q *QueryOptions) (io.ReadCloser, error) {
r := c.newRequest("GET", endpoint)
r.setQueryOptions(q)
_, resp, err := requireOK(c.doRequest(r))
if err != nil {
return nil, err
}
return resp.Body, nil
}
// Query is used to do a GET request against an endpoint
// and deserialize the response into an interface using
// standard Nomad conventions.
func (c *Client) query(endpoint string, out interface{}, q *QueryOptions) (*QueryMeta, error) {
r := c.newRequest("GET", endpoint)
r.setQueryOptions(q)
rtt, resp, err := requireOK(c.doRequest(r))
if err != nil {
return nil, err
}
defer resp.Body.Close()
qm := &QueryMeta{}
parseQueryMeta(resp, qm)
qm.RequestTime = rtt
if err := decodeBody(resp, out); err != nil {
return nil, err
}
return qm, nil
}
// write is used to do a PUT request against an endpoint
// and serialize/deserialized using the standard Nomad conventions.
func (c *Client) write(endpoint string, in, out interface{}, q *WriteOptions) (*WriteMeta, error) {
r := c.newRequest("PUT", endpoint)
r.setWriteOptions(q)
r.obj = in
rtt, resp, err := requireOK(c.doRequest(r))
if err != nil {
return nil, err
}
defer resp.Body.Close()
wm := &WriteMeta{RequestTime: rtt}
parseWriteMeta(resp, wm)
if out != nil {
if err := decodeBody(resp, &out); err != nil {
return nil, err
}
}
return wm, nil
}
// write is used to do a PUT request against an endpoint
// and serialize/deserialized using the standard Nomad conventions.
func (c *Client) delete(endpoint string, out interface{}, q *WriteOptions) (*WriteMeta, error) {
r := c.newRequest("DELETE", endpoint)
r.setWriteOptions(q)
rtt, resp, err := requireOK(c.doRequest(r))
if err != nil {
return nil, err
}
defer resp.Body.Close()
wm := &WriteMeta{RequestTime: rtt}
parseWriteMeta(resp, wm)
if out != nil {
if err := decodeBody(resp, &out); err != nil {
return nil, err
}
}
return wm, nil
}
// parseQueryMeta is used to help parse query meta-data
func parseQueryMeta(resp *http.Response, q *QueryMeta) error {
header := resp.Header
// Parse the X-Nomad-Index
index, err := strconv.ParseUint(header.Get("X-Nomad-Index"), 10, 64)
if err != nil {
return fmt.Errorf("Failed to parse X-Nomad-Index: %v", err)
}
q.LastIndex = index
// Parse the X-Nomad-LastContact
last, err := strconv.ParseUint(header.Get("X-Nomad-LastContact"), 10, 64)
if err != nil {
return fmt.Errorf("Failed to parse X-Nomad-LastContact: %v", err)
}
q.LastContact = time.Duration(last) * time.Millisecond
// Parse the X-Nomad-KnownLeader
switch header.Get("X-Nomad-KnownLeader") {
case "true":
q.KnownLeader = true
default:
q.KnownLeader = false
}
return nil
}
// parseWriteMeta is used to help parse write meta-data
func parseWriteMeta(resp *http.Response, q *WriteMeta) error {
header := resp.Header
// Parse the X-Nomad-Index
index, err := strconv.ParseUint(header.Get("X-Nomad-Index"), 10, 64)
if err != nil {
return fmt.Errorf("Failed to parse X-Nomad-Index: %v", err)
}
q.LastIndex = index
return nil
}
// decodeBody is used to JSON decode a body
func decodeBody(resp *http.Response, out interface{}) error {
dec := json.NewDecoder(resp.Body)
return dec.Decode(out)
}
// encodeBody is used to encode a request body
func encodeBody(obj interface{}) (io.Reader, error) {
buf := bytes.NewBuffer(nil)
enc := json.NewEncoder(buf)
if err := enc.Encode(obj); err != nil {
return nil, err
}
return buf, nil
}
// requireOK is used to wrap doRequest and check for a 200
func requireOK(d time.Duration, resp *http.Response, e error) (time.Duration, *http.Response, error) {
if e != nil {
if resp != nil {
resp.Body.Close()
}
return d, nil, e
}
if resp.StatusCode != 200 {
var buf bytes.Buffer
io.Copy(&buf, resp.Body)
resp.Body.Close()
return d, nil, fmt.Errorf("Unexpected response code: %d (%s)", resp.StatusCode, buf.Bytes())
}
return d, resp, nil
}

17
vendor/github.com/hashicorp/nomad/api/constraint.go generated vendored Normal file
View File

@@ -0,0 +1,17 @@
package api
// Constraint is used to serialize a job placement constraint.
type Constraint struct {
LTarget string
RTarget string
Operand string
}
// NewConstraint generates a new job placement constraint.
func NewConstraint(left, operand, right string) *Constraint {
return &Constraint{
LTarget: left,
RTarget: right,
Operand: operand,
}
}

90
vendor/github.com/hashicorp/nomad/api/evaluations.go generated vendored Normal file
View File

@@ -0,0 +1,90 @@
package api
import (
"sort"
"time"
)
// Evaluations is used to query the evaluation endpoints.
type Evaluations struct {
client *Client
}
// Evaluations returns a new handle on the evaluations.
func (c *Client) Evaluations() *Evaluations {
return &Evaluations{client: c}
}
// List is used to dump all of the evaluations.
func (e *Evaluations) List(q *QueryOptions) ([]*Evaluation, *QueryMeta, error) {
var resp []*Evaluation
qm, err := e.client.query("/v1/evaluations", &resp, q)
if err != nil {
return nil, nil, err
}
sort.Sort(EvalIndexSort(resp))
return resp, qm, nil
}
func (e *Evaluations) PrefixList(prefix string) ([]*Evaluation, *QueryMeta, error) {
return e.List(&QueryOptions{Prefix: prefix})
}
// Info is used to query a single evaluation by its ID.
func (e *Evaluations) Info(evalID string, q *QueryOptions) (*Evaluation, *QueryMeta, error) {
var resp Evaluation
qm, err := e.client.query("/v1/evaluation/"+evalID, &resp, q)
if err != nil {
return nil, nil, err
}
return &resp, qm, nil
}
// Allocations is used to retrieve a set of allocations given
// an evaluation ID.
func (e *Evaluations) Allocations(evalID string, q *QueryOptions) ([]*AllocationListStub, *QueryMeta, error) {
var resp []*AllocationListStub
qm, err := e.client.query("/v1/evaluation/"+evalID+"/allocations", &resp, q)
if err != nil {
return nil, nil, err
}
sort.Sort(AllocIndexSort(resp))
return resp, qm, nil
}
// Evaluation is used to serialize an evaluation.
type Evaluation struct {
ID string
Priority int
Type string
TriggeredBy string
JobID string
JobModifyIndex uint64
NodeID string
NodeModifyIndex uint64
Status string
StatusDescription string
Wait time.Duration
NextEval string
PreviousEval string
BlockedEval string
FailedTGAllocs map[string]*AllocationMetric
CreateIndex uint64
ModifyIndex uint64
}
// EvalIndexSort is a wrapper to sort evaluations by CreateIndex.
// We reverse the test so that we get the highest index first.
type EvalIndexSort []*Evaluation
func (e EvalIndexSort) Len() int {
return len(e)
}
func (e EvalIndexSort) Less(i, j int) bool {
return e[i].CreateIndex > e[j].CreateIndex
}
func (e EvalIndexSort) Swap(i, j int) {
e[i], e[j] = e[j], e[i]
}

398
vendor/github.com/hashicorp/nomad/api/fs.go generated vendored Normal file
View File

@@ -0,0 +1,398 @@
package api
import (
"encoding/json"
"fmt"
"io"
"strconv"
"sync"
"time"
)
const (
// OriginStart and OriginEnd are the available parameters for the origin
// argument when streaming a file. They respectively offset from the start
// and end of a file.
OriginStart = "start"
OriginEnd = "end"
)
// AllocFileInfo holds information about a file inside the AllocDir
type AllocFileInfo struct {
Name string
IsDir bool
Size int64
FileMode string
ModTime time.Time
}
// StreamFrame is used to frame data of a file when streaming
type StreamFrame struct {
Offset int64 `json:",omitempty"`
Data []byte `json:",omitempty"`
File string `json:",omitempty"`
FileEvent string `json:",omitempty"`
}
// IsHeartbeat returns if the frame is a heartbeat frame
func (s *StreamFrame) IsHeartbeat() bool {
return len(s.Data) == 0 && s.FileEvent == "" && s.File == "" && s.Offset == 0
}
// AllocFS is used to introspect an allocation directory on a Nomad client
type AllocFS struct {
client *Client
}
// AllocFS returns an handle to the AllocFS endpoints
func (c *Client) AllocFS() *AllocFS {
return &AllocFS{client: c}
}
// getNodeClient returns a Client that will dial the node. If the QueryOptions
// is set, the function will ensure that it is initalized and that the Params
// field is valid.
func (a *AllocFS) getNodeClient(nodeHTTPAddr, allocID string, q **QueryOptions) (*Client, error) {
if nodeHTTPAddr == "" {
return nil, fmt.Errorf("http addr of the node where alloc %q is running is not advertised", allocID)
}
// Get an API client for the node
nodeClientConfig := &Config{
Address: fmt.Sprintf("http://%s", nodeHTTPAddr),
Region: a.client.config.Region,
}
nodeClient, err := NewClient(nodeClientConfig)
if err != nil {
return nil, err
}
// Set the query params
if q == nil {
return nodeClient, nil
}
if *q == nil {
*q = &QueryOptions{}
}
if actQ := *q; actQ.Params == nil {
actQ.Params = make(map[string]string)
}
return nodeClient, nil
}
// List is used to list the files at a given path of an allocation directory
func (a *AllocFS) List(alloc *Allocation, path string, q *QueryOptions) ([]*AllocFileInfo, *QueryMeta, error) {
node, _, err := a.client.Nodes().Info(alloc.NodeID, &QueryOptions{})
if err != nil {
return nil, nil, err
}
nodeClient, err := a.getNodeClient(node.HTTPAddr, alloc.ID, &q)
if err != nil {
return nil, nil, err
}
q.Params["path"] = path
var resp []*AllocFileInfo
qm, err := nodeClient.query(fmt.Sprintf("/v1/client/fs/ls/%s", alloc.ID), &resp, q)
if err != nil {
return nil, nil, err
}
return resp, qm, nil
}
// Stat is used to stat a file at a given path of an allocation directory
func (a *AllocFS) Stat(alloc *Allocation, path string, q *QueryOptions) (*AllocFileInfo, *QueryMeta, error) {
node, _, err := a.client.Nodes().Info(alloc.NodeID, &QueryOptions{})
if err != nil {
return nil, nil, err
}
nodeClient, err := a.getNodeClient(node.HTTPAddr, alloc.ID, &q)
if err != nil {
return nil, nil, err
}
q.Params["path"] = path
var resp AllocFileInfo
qm, err := nodeClient.query(fmt.Sprintf("/v1/client/fs/stat/%s", alloc.ID), &resp, q)
if err != nil {
return nil, nil, err
}
return &resp, qm, nil
}
// ReadAt is used to read bytes at a given offset until limit at the given path
// in an allocation directory. If limit is <= 0, there is no limit.
func (a *AllocFS) ReadAt(alloc *Allocation, path string, offset int64, limit int64, q *QueryOptions) (io.ReadCloser, error) {
node, _, err := a.client.Nodes().Info(alloc.NodeID, &QueryOptions{})
if err != nil {
return nil, err
}
nodeClient, err := a.getNodeClient(node.HTTPAddr, alloc.ID, &q)
if err != nil {
return nil, err
}
q.Params["path"] = path
q.Params["offset"] = strconv.FormatInt(offset, 10)
q.Params["limit"] = strconv.FormatInt(limit, 10)
r, err := nodeClient.rawQuery(fmt.Sprintf("/v1/client/fs/readat/%s", alloc.ID), q)
if err != nil {
return nil, err
}
return r, nil
}
// Cat is used to read contents of a file at the given path in an allocation
// directory
func (a *AllocFS) Cat(alloc *Allocation, path string, q *QueryOptions) (io.ReadCloser, error) {
node, _, err := a.client.Nodes().Info(alloc.NodeID, &QueryOptions{})
if err != nil {
return nil, err
}
nodeClient, err := a.getNodeClient(node.HTTPAddr, alloc.ID, &q)
if err != nil {
return nil, err
}
q.Params["path"] = path
r, err := nodeClient.rawQuery(fmt.Sprintf("/v1/client/fs/cat/%s", alloc.ID), q)
if err != nil {
return nil, err
}
return r, nil
}
// Stream streams the content of a file blocking on EOF.
// The parameters are:
// * path: path to file to stream.
// * offset: The offset to start streaming data at.
// * origin: Either "start" or "end" and defines from where the offset is applied.
// * cancel: A channel that when closed, streaming will end.
//
// The return value is a channel that will emit StreamFrames as they are read.
func (a *AllocFS) Stream(alloc *Allocation, path, origin string, offset int64,
cancel <-chan struct{}, q *QueryOptions) (<-chan *StreamFrame, error) {
node, _, err := a.client.Nodes().Info(alloc.NodeID, q)
if err != nil {
return nil, err
}
nodeClient, err := a.getNodeClient(node.HTTPAddr, alloc.ID, &q)
if err != nil {
return nil, err
}
q.Params["path"] = path
q.Params["offset"] = strconv.FormatInt(offset, 10)
q.Params["origin"] = origin
r, err := nodeClient.rawQuery(fmt.Sprintf("/v1/client/fs/stream/%s", alloc.ID), q)
if err != nil {
return nil, err
}
// Create the output channel
frames := make(chan *StreamFrame, 10)
go func() {
// Close the body
defer r.Close()
// Create a decoder
dec := json.NewDecoder(r)
for {
// Check if we have been cancelled
select {
case <-cancel:
return
default:
}
// Decode the next frame
var frame StreamFrame
if err := dec.Decode(&frame); err != nil {
close(frames)
return
}
// Discard heartbeat frames
if frame.IsHeartbeat() {
continue
}
frames <- &frame
}
}()
return frames, nil
}
// Logs streams the content of a tasks logs blocking on EOF.
// The parameters are:
// * allocation: the allocation to stream from.
// * follow: Whether the logs should be followed.
// * task: the tasks name to stream logs for.
// * logType: Either "stdout" or "stderr"
// * origin: Either "start" or "end" and defines from where the offset is applied.
// * offset: The offset to start streaming data at.
// * cancel: A channel that when closed, streaming will end.
//
// The return value is a channel that will emit StreamFrames as they are read.
func (a *AllocFS) Logs(alloc *Allocation, follow bool, task, logType, origin string,
offset int64, cancel <-chan struct{}, q *QueryOptions) (<-chan *StreamFrame, error) {
node, _, err := a.client.Nodes().Info(alloc.NodeID, q)
if err != nil {
return nil, err
}
nodeClient, err := a.getNodeClient(node.HTTPAddr, alloc.ID, &q)
if err != nil {
return nil, err
}
q.Params["follow"] = strconv.FormatBool(follow)
q.Params["task"] = task
q.Params["type"] = logType
q.Params["origin"] = origin
q.Params["offset"] = strconv.FormatInt(offset, 10)
r, err := nodeClient.rawQuery(fmt.Sprintf("/v1/client/fs/logs/%s", alloc.ID), q)
if err != nil {
return nil, err
}
// Create the output channel
frames := make(chan *StreamFrame, 10)
go func() {
// Close the body
defer r.Close()
// Create a decoder
dec := json.NewDecoder(r)
for {
// Check if we have been cancelled
select {
case <-cancel:
return
default:
}
// Decode the next frame
var frame StreamFrame
if err := dec.Decode(&frame); err != nil {
close(frames)
return
}
// Discard heartbeat frames
if frame.IsHeartbeat() {
continue
}
frames <- &frame
}
}()
return frames, nil
}
// FrameReader is used to convert a stream of frames into a read closer.
type FrameReader struct {
frames <-chan *StreamFrame
cancelCh chan struct{}
closedLock sync.Mutex
closed bool
unblockTime time.Duration
frame *StreamFrame
frameOffset int
byteOffset int
}
// NewFrameReader takes a channel of frames and returns a FrameReader which
// implements io.ReadCloser
func NewFrameReader(frames <-chan *StreamFrame, cancelCh chan struct{}) *FrameReader {
return &FrameReader{
frames: frames,
cancelCh: cancelCh,
}
}
// SetUnblockTime sets the time to unblock and return zero bytes read. If the
// duration is unset or is zero or less, the read will block til data is read.
func (f *FrameReader) SetUnblockTime(d time.Duration) {
f.unblockTime = d
}
// Offset returns the offset into the stream.
func (f *FrameReader) Offset() int {
return f.byteOffset
}
// Read reads the data of the incoming frames into the bytes buffer. Returns EOF
// when there are no more frames.
func (f *FrameReader) Read(p []byte) (n int, err error) {
f.closedLock.Lock()
closed := f.closed
f.closedLock.Unlock()
if closed {
return 0, io.EOF
}
if f.frame == nil {
var unblock <-chan time.Time
if f.unblockTime.Nanoseconds() > 0 {
unblock = time.After(f.unblockTime)
}
select {
case frame, ok := <-f.frames:
if !ok {
return 0, io.EOF
}
f.frame = frame
// Store the total offset into the file
f.byteOffset = int(f.frame.Offset)
case <-unblock:
return 0, nil
case <-f.cancelCh:
return 0, io.EOF
}
}
// Copy the data out of the frame and update our offset
n = copy(p, f.frame.Data[f.frameOffset:])
f.frameOffset += n
// Clear the frame and its offset once we have read everything
if len(f.frame.Data) == f.frameOffset {
f.frame = nil
f.frameOffset = 0
}
return n, nil
}
// Close cancels the stream of frames
func (f *FrameReader) Close() error {
f.closedLock.Lock()
defer f.closedLock.Unlock()
if f.closed {
return nil
}
close(f.cancelCh)
f.closed = true
return nil
}

402
vendor/github.com/hashicorp/nomad/api/jobs.go generated vendored Normal file
View File

@@ -0,0 +1,402 @@
package api
import (
"fmt"
"sort"
"time"
)
const (
// JobTypeService indicates a long-running processes
JobTypeService = "service"
// JobTypeBatch indicates a short-lived process
JobTypeBatch = "batch"
)
const (
// RegisterEnforceIndexErrPrefix is the prefix to use in errors caused by
// enforcing the job modify index during registers.
RegisterEnforceIndexErrPrefix = "Enforcing job modify index"
)
// Jobs is used to access the job-specific endpoints.
type Jobs struct {
client *Client
}
// Jobs returns a handle on the jobs endpoints.
func (c *Client) Jobs() *Jobs {
return &Jobs{client: c}
}
// Register is used to register a new job. It returns the ID
// of the evaluation, along with any errors encountered.
func (j *Jobs) Register(job *Job, q *WriteOptions) (string, *WriteMeta, error) {
var resp registerJobResponse
req := &RegisterJobRequest{Job: job}
wm, err := j.client.write("/v1/jobs", req, &resp, q)
if err != nil {
return "", nil, err
}
return resp.EvalID, wm, nil
}
// EnforceRegister is used to register a job enforcing its job modify index.
func (j *Jobs) EnforceRegister(job *Job, modifyIndex uint64, q *WriteOptions) (string, *WriteMeta, error) {
var resp registerJobResponse
req := &RegisterJobRequest{
Job: job,
EnforceIndex: true,
JobModifyIndex: modifyIndex,
}
wm, err := j.client.write("/v1/jobs", req, &resp, q)
if err != nil {
return "", nil, err
}
return resp.EvalID, wm, nil
}
// List is used to list all of the existing jobs.
func (j *Jobs) List(q *QueryOptions) ([]*JobListStub, *QueryMeta, error) {
var resp []*JobListStub
qm, err := j.client.query("/v1/jobs", &resp, q)
if err != nil {
return nil, qm, err
}
sort.Sort(JobIDSort(resp))
return resp, qm, nil
}
// PrefixList is used to list all existing jobs that match the prefix.
func (j *Jobs) PrefixList(prefix string) ([]*JobListStub, *QueryMeta, error) {
return j.List(&QueryOptions{Prefix: prefix})
}
// Info is used to retrieve information about a particular
// job given its unique ID.
func (j *Jobs) Info(jobID string, q *QueryOptions) (*Job, *QueryMeta, error) {
var resp Job
qm, err := j.client.query("/v1/job/"+jobID, &resp, q)
if err != nil {
return nil, nil, err
}
return &resp, qm, nil
}
// Allocations is used to return the allocs for a given job ID.
func (j *Jobs) Allocations(jobID string, q *QueryOptions) ([]*AllocationListStub, *QueryMeta, error) {
var resp []*AllocationListStub
qm, err := j.client.query("/v1/job/"+jobID+"/allocations", &resp, q)
if err != nil {
return nil, nil, err
}
sort.Sort(AllocIndexSort(resp))
return resp, qm, nil
}
// Evaluations is used to query the evaluations associated with
// the given job ID.
func (j *Jobs) Evaluations(jobID string, q *QueryOptions) ([]*Evaluation, *QueryMeta, error) {
var resp []*Evaluation
qm, err := j.client.query("/v1/job/"+jobID+"/evaluations", &resp, q)
if err != nil {
return nil, nil, err
}
sort.Sort(EvalIndexSort(resp))
return resp, qm, nil
}
// Deregister is used to remove an existing job.
func (j *Jobs) Deregister(jobID string, q *WriteOptions) (string, *WriteMeta, error) {
var resp deregisterJobResponse
wm, err := j.client.delete("/v1/job/"+jobID, &resp, q)
if err != nil {
return "", nil, err
}
return resp.EvalID, wm, nil
}
// ForceEvaluate is used to force-evaluate an existing job.
func (j *Jobs) ForceEvaluate(jobID string, q *WriteOptions) (string, *WriteMeta, error) {
var resp registerJobResponse
wm, err := j.client.write("/v1/job/"+jobID+"/evaluate", nil, &resp, q)
if err != nil {
return "", nil, err
}
return resp.EvalID, wm, nil
}
// PeriodicForce spawns a new instance of the periodic job and returns the eval ID
func (j *Jobs) PeriodicForce(jobID string, q *WriteOptions) (string, *WriteMeta, error) {
var resp periodicForceResponse
wm, err := j.client.write("/v1/job/"+jobID+"/periodic/force", nil, &resp, q)
if err != nil {
return "", nil, err
}
return resp.EvalID, wm, nil
}
func (j *Jobs) Plan(job *Job, diff bool, q *WriteOptions) (*JobPlanResponse, *WriteMeta, error) {
if job == nil {
return nil, nil, fmt.Errorf("must pass non-nil job")
}
var resp JobPlanResponse
req := &JobPlanRequest{
Job: job,
Diff: diff,
}
wm, err := j.client.write("/v1/job/"+job.ID+"/plan", req, &resp, q)
if err != nil {
return nil, nil, err
}
return &resp, wm, nil
}
func (j *Jobs) Summary(jobID string, q *QueryOptions) (*JobSummary, *QueryMeta, error) {
var resp JobSummary
qm, err := j.client.query("/v1/job/"+jobID+"/summary", &resp, q)
if err != nil {
return nil, nil, err
}
return &resp, qm, nil
}
// periodicForceResponse is used to deserialize a force response
type periodicForceResponse struct {
EvalID string
}
// UpdateStrategy is for serializing update strategy for a job.
type UpdateStrategy struct {
Stagger time.Duration
MaxParallel int
}
// PeriodicConfig is for serializing periodic config for a job.
type PeriodicConfig struct {
Enabled bool
Spec string
SpecType string
ProhibitOverlap bool
}
// Job is used to serialize a job.
type Job struct {
Region string
ID string
Name string
Type string
Priority int
AllAtOnce bool
Datacenters []string
Constraints []*Constraint
TaskGroups []*TaskGroup
Update *UpdateStrategy
Periodic *PeriodicConfig
Meta map[string]string
Status string
StatusDescription string
CreateIndex uint64
ModifyIndex uint64
JobModifyIndex uint64
}
// JobSummary summarizes the state of the allocations of a job
type JobSummary struct {
JobID string
Summary map[string]TaskGroupSummary
// Raft Indexes
CreateIndex uint64
ModifyIndex uint64
}
// TaskGroup summarizes the state of all the allocations of a particular
// TaskGroup
type TaskGroupSummary struct {
Queued int
Complete int
Failed int
Running int
Starting int
Lost int
}
// JobListStub is used to return a subset of information about
// jobs during list operations.
type JobListStub struct {
ID string
ParentID string
Name string
Type string
Priority int
Status string
StatusDescription string
JobSummary *JobSummary
CreateIndex uint64
ModifyIndex uint64
JobModifyIndex uint64
}
// JobIDSort is used to sort jobs by their job ID's.
type JobIDSort []*JobListStub
func (j JobIDSort) Len() int {
return len(j)
}
func (j JobIDSort) Less(a, b int) bool {
return j[a].ID < j[b].ID
}
func (j JobIDSort) Swap(a, b int) {
j[a], j[b] = j[b], j[a]
}
// NewServiceJob creates and returns a new service-style job
// for long-lived processes using the provided name, ID, and
// relative job priority.
func NewServiceJob(id, name, region string, pri int) *Job {
return newJob(id, name, region, JobTypeService, pri)
}
// NewBatchJob creates and returns a new batch-style job for
// short-lived processes using the provided name and ID along
// with the relative job priority.
func NewBatchJob(id, name, region string, pri int) *Job {
return newJob(id, name, region, JobTypeBatch, pri)
}
// newJob is used to create a new Job struct.
func newJob(id, name, region, typ string, pri int) *Job {
return &Job{
Region: region,
ID: id,
Name: name,
Type: typ,
Priority: pri,
}
}
// SetMeta is used to set arbitrary k/v pairs of metadata on a job.
func (j *Job) SetMeta(key, val string) *Job {
if j.Meta == nil {
j.Meta = make(map[string]string)
}
j.Meta[key] = val
return j
}
// AddDatacenter is used to add a datacenter to a job.
func (j *Job) AddDatacenter(dc string) *Job {
j.Datacenters = append(j.Datacenters, dc)
return j
}
// Constrain is used to add a constraint to a job.
func (j *Job) Constrain(c *Constraint) *Job {
j.Constraints = append(j.Constraints, c)
return j
}
// AddTaskGroup adds a task group to an existing job.
func (j *Job) AddTaskGroup(grp *TaskGroup) *Job {
j.TaskGroups = append(j.TaskGroups, grp)
return j
}
// AddPeriodicConfig adds a periodic config to an existing job.
func (j *Job) AddPeriodicConfig(cfg *PeriodicConfig) *Job {
j.Periodic = cfg
return j
}
// RegisterJobRequest is used to serialize a job registration
type RegisterJobRequest struct {
Job *Job
EnforceIndex bool `json:",omitempty"`
JobModifyIndex uint64 `json:",omitempty"`
}
// registerJobResponse is used to deserialize a job response
type registerJobResponse struct {
EvalID string
}
// deregisterJobResponse is used to decode a deregister response
type deregisterJobResponse struct {
EvalID string
}
type JobPlanRequest struct {
Job *Job
Diff bool
}
type JobPlanResponse struct {
JobModifyIndex uint64
CreatedEvals []*Evaluation
Diff *JobDiff
Annotations *PlanAnnotations
FailedTGAllocs map[string]*AllocationMetric
NextPeriodicLaunch time.Time
}
type JobDiff struct {
Type string
ID string
Fields []*FieldDiff
Objects []*ObjectDiff
TaskGroups []*TaskGroupDiff
}
type TaskGroupDiff struct {
Type string
Name string
Fields []*FieldDiff
Objects []*ObjectDiff
Tasks []*TaskDiff
Updates map[string]uint64
}
type TaskDiff struct {
Type string
Name string
Fields []*FieldDiff
Objects []*ObjectDiff
Annotations []string
}
type FieldDiff struct {
Type string
Name string
Old, New string
Annotations []string
}
type ObjectDiff struct {
Type string
Name string
Fields []*FieldDiff
Objects []*ObjectDiff
}
type PlanAnnotations struct {
DesiredTGUpdates map[string]*DesiredUpdates
}
type DesiredUpdates struct {
Ignore uint64
Place uint64
Migrate uint64
Stop uint64
InPlaceUpdate uint64
DestructiveUpdate uint64
}

199
vendor/github.com/hashicorp/nomad/api/nodes.go generated vendored Normal file
View File

@@ -0,0 +1,199 @@
package api
import (
"fmt"
"sort"
"strconv"
"github.com/hashicorp/go-cleanhttp"
)
// Nodes is used to query node-related API endpoints
type Nodes struct {
client *Client
}
// Nodes returns a handle on the node endpoints.
func (c *Client) Nodes() *Nodes {
return &Nodes{client: c}
}
// List is used to list out all of the nodes
func (n *Nodes) List(q *QueryOptions) ([]*NodeListStub, *QueryMeta, error) {
var resp NodeIndexSort
qm, err := n.client.query("/v1/nodes", &resp, q)
if err != nil {
return nil, nil, err
}
sort.Sort(NodeIndexSort(resp))
return resp, qm, nil
}
func (n *Nodes) PrefixList(prefix string) ([]*NodeListStub, *QueryMeta, error) {
return n.List(&QueryOptions{Prefix: prefix})
}
// Info is used to query a specific node by its ID.
func (n *Nodes) Info(nodeID string, q *QueryOptions) (*Node, *QueryMeta, error) {
var resp Node
qm, err := n.client.query("/v1/node/"+nodeID, &resp, q)
if err != nil {
return nil, nil, err
}
return &resp, qm, nil
}
// ToggleDrain is used to toggle drain mode on/off for a given node.
func (n *Nodes) ToggleDrain(nodeID string, drain bool, q *WriteOptions) (*WriteMeta, error) {
drainArg := strconv.FormatBool(drain)
wm, err := n.client.write("/v1/node/"+nodeID+"/drain?enable="+drainArg, nil, nil, q)
if err != nil {
return nil, err
}
return wm, nil
}
// Allocations is used to return the allocations associated with a node.
func (n *Nodes) Allocations(nodeID string, q *QueryOptions) ([]*Allocation, *QueryMeta, error) {
var resp []*Allocation
qm, err := n.client.query("/v1/node/"+nodeID+"/allocations", &resp, q)
if err != nil {
return nil, nil, err
}
sort.Sort(AllocationSort(resp))
return resp, qm, nil
}
// ForceEvaluate is used to force-evaluate an existing node.
func (n *Nodes) ForceEvaluate(nodeID string, q *WriteOptions) (string, *WriteMeta, error) {
var resp nodeEvalResponse
wm, err := n.client.write("/v1/node/"+nodeID+"/evaluate", nil, &resp, q)
if err != nil {
return "", nil, err
}
return resp.EvalID, wm, nil
}
func (n *Nodes) Stats(nodeID string, q *QueryOptions) (*HostStats, error) {
node, _, err := n.client.Nodes().Info(nodeID, q)
if err != nil {
return nil, err
}
if node.HTTPAddr == "" {
return nil, fmt.Errorf("http addr of the node %q is running is not advertised", nodeID)
}
client, err := NewClient(&Config{
Address: fmt.Sprintf("http://%s", node.HTTPAddr),
HttpClient: cleanhttp.DefaultClient(),
})
if err != nil {
return nil, err
}
var resp HostStats
if _, err := client.query("/v1/client/stats", &resp, nil); err != nil {
return nil, err
}
return &resp, nil
}
// Node is used to deserialize a node entry.
type Node struct {
ID string
Datacenter string
Name string
HTTPAddr string
Attributes map[string]string
Resources *Resources
Reserved *Resources
Links map[string]string
Meta map[string]string
NodeClass string
Drain bool
Status string
StatusDescription string
StatusUpdatedAt int64
CreateIndex uint64
ModifyIndex uint64
}
// HostStats represents resource usage stats of the host running a Nomad client
type HostStats struct {
Memory *HostMemoryStats
CPU []*HostCPUStats
DiskStats []*HostDiskStats
Uptime uint64
CPUTicksConsumed float64
}
type HostMemoryStats struct {
Total uint64
Available uint64
Used uint64
Free uint64
}
type HostCPUStats struct {
CPU string
User float64
System float64
Idle float64
}
type HostDiskStats struct {
Device string
Mountpoint string
Size uint64
Used uint64
Available uint64
UsedPercent float64
InodesUsedPercent float64
}
// NodeListStub is a subset of information returned during
// node list operations.
type NodeListStub struct {
ID string
Datacenter string
Name string
NodeClass string
Drain bool
Status string
StatusDescription string
CreateIndex uint64
ModifyIndex uint64
}
// NodeIndexSort reverse sorts nodes by CreateIndex
type NodeIndexSort []*NodeListStub
func (n NodeIndexSort) Len() int {
return len(n)
}
func (n NodeIndexSort) Less(i, j int) bool {
return n[i].CreateIndex > n[j].CreateIndex
}
func (n NodeIndexSort) Swap(i, j int) {
n[i], n[j] = n[j], n[i]
}
// nodeEvalResponse is used to decode a force-eval.
type nodeEvalResponse struct {
EvalID string
}
// AllocationSort reverse sorts allocs by CreateIndex.
type AllocationSort []*Allocation
func (a AllocationSort) Len() int {
return len(a)
}
func (a AllocationSort) Less(i, j int) bool {
return a[i].CreateIndex > a[j].CreateIndex
}
func (a AllocationSort) Swap(i, j int) {
a[i], a[j] = a[j], a[i]
}

30
vendor/github.com/hashicorp/nomad/api/raw.go generated vendored Normal file
View File

@@ -0,0 +1,30 @@
package api
// Raw can be used to do raw queries against custom endpoints
type Raw struct {
c *Client
}
// Raw returns a handle to query endpoints
func (c *Client) Raw() *Raw {
return &Raw{c}
}
// Query is used to do a GET request against an endpoint
// and deserialize the response into an interface using
// standard Nomad conventions.
func (raw *Raw) Query(endpoint string, out interface{}, q *QueryOptions) (*QueryMeta, error) {
return raw.c.query(endpoint, out, q)
}
// Write is used to do a PUT request against an endpoint
// and serialize/deserialized using the standard Nomad conventions.
func (raw *Raw) Write(endpoint string, in, out interface{}, q *WriteOptions) (*WriteMeta, error) {
return raw.c.write(endpoint, in, out, q)
}
// Delete is used to do a DELETE request against an endpoint
// and serialize/deserialized using the standard Nomad conventions.
func (raw *Raw) Delete(endpoint string, out interface{}, q *WriteOptions) (*WriteMeta, error) {
return raw.c.delete(endpoint, out, q)
}

23
vendor/github.com/hashicorp/nomad/api/regions.go generated vendored Normal file
View File

@@ -0,0 +1,23 @@
package api
import "sort"
// Regions is used to query the regions in the cluster.
type Regions struct {
client *Client
}
// Regions returns a handle on the allocs endpoints.
func (c *Client) Regions() *Regions {
return &Regions{client: c}
}
// List returns a list of all of the regions.
func (r *Regions) List() ([]string, error) {
var resp []string
if _, err := r.client.query("/v1/regions", &resp, nil); err != nil {
return nil, err
}
sort.Strings(resp)
return resp, nil
}

27
vendor/github.com/hashicorp/nomad/api/resources.go generated vendored Normal file
View File

@@ -0,0 +1,27 @@
package api
// Resources encapsulates the required resources of
// a given task or task group.
type Resources struct {
CPU int
MemoryMB int
DiskMB int
IOPS int
Networks []*NetworkResource
}
type Port struct {
Label string
Value int
}
// NetworkResource is used to describe required network
// resources of a given task.
type NetworkResource struct {
Public bool
CIDR string
ReservedPorts []Port
DynamicPorts []Port
IP string
MBits int
}

43
vendor/github.com/hashicorp/nomad/api/status.go generated vendored Normal file
View File

@@ -0,0 +1,43 @@
package api
// Status is used to query the status-related endpoints.
type Status struct {
client *Client
}
// Status returns a handle on the status endpoints.
func (c *Client) Status() *Status {
return &Status{client: c}
}
// Leader is used to query for the current cluster leader.
func (s *Status) Leader() (string, error) {
var resp string
_, err := s.client.query("/v1/status/leader", &resp, nil)
if err != nil {
return "", err
}
return resp, nil
}
// RegionLeader is used to query for the leader in the passed region.
func (s *Status) RegionLeader(region string) (string, error) {
var resp string
q := QueryOptions{Region: region}
_, err := s.client.query("/v1/status/leader", &resp, &q)
if err != nil {
return "", err
}
return resp, nil
}
// Peers is used to query the addresses of the server peers
// in the cluster.
func (s *Status) Peers() ([]string, error) {
var resp []string
_, err := s.client.query("/v1/status/peers", &resp, nil)
if err != nil {
return nil, err
}
return resp, nil
}

17
vendor/github.com/hashicorp/nomad/api/system.go generated vendored Normal file
View File

@@ -0,0 +1,17 @@
package api
// Status is used to query the status-related endpoints.
type System struct {
client *Client
}
// System returns a handle on the system endpoints.
func (c *Client) System() *System {
return &System{client: c}
}
func (s *System) GarbageCollect() error {
var req struct{}
_, err := s.client.write("/v1/system/gc", &req, nil, nil)
return err
}

233
vendor/github.com/hashicorp/nomad/api/tasks.go generated vendored Normal file
View File

@@ -0,0 +1,233 @@
package api
import (
"time"
)
// MemoryStats holds memory usage related stats
type MemoryStats struct {
RSS uint64
Cache uint64
Swap uint64
MaxUsage uint64
KernelUsage uint64
KernelMaxUsage uint64
Measured []string
}
// CpuStats holds cpu usage related stats
type CpuStats struct {
SystemMode float64
UserMode float64
TotalTicks float64
ThrottledPeriods uint64
ThrottledTime uint64
Percent float64
Measured []string
}
// ResourceUsage holds information related to cpu and memory stats
type ResourceUsage struct {
MemoryStats *MemoryStats
CpuStats *CpuStats
}
// TaskResourceUsage holds aggregated resource usage of all processes in a Task
// and the resource usage of the individual pids
type TaskResourceUsage struct {
ResourceUsage *ResourceUsage
Timestamp int64
Pids map[string]*ResourceUsage
}
// AllocResourceUsage holds the aggregated task resource usage of the
// allocation.
type AllocResourceUsage struct {
ResourceUsage *ResourceUsage
Tasks map[string]*TaskResourceUsage
Timestamp int64
}
// RestartPolicy defines how the Nomad client restarts
// tasks in a taskgroup when they fail
type RestartPolicy struct {
Interval time.Duration
Attempts int
Delay time.Duration
Mode string
}
// The ServiceCheck data model represents the consul health check that
// Nomad registers for a Task
type ServiceCheck struct {
Id string
Name string
Type string
Command string
Args []string
Path string
Protocol string `mapstructure:"port"`
PortLabel string `mapstructure:"port"`
Interval time.Duration
Timeout time.Duration
}
// The Service model represents a Consul service definition
type Service struct {
Id string
Name string
Tags []string
PortLabel string `mapstructure:"port"`
Checks []ServiceCheck
}
// TaskGroup is the unit of scheduling.
type TaskGroup struct {
Name string
Count int
Constraints []*Constraint
Tasks []*Task
RestartPolicy *RestartPolicy
Meta map[string]string
}
// NewTaskGroup creates a new TaskGroup.
func NewTaskGroup(name string, count int) *TaskGroup {
return &TaskGroup{
Name: name,
Count: count,
}
}
// Constrain is used to add a constraint to a task group.
func (g *TaskGroup) Constrain(c *Constraint) *TaskGroup {
g.Constraints = append(g.Constraints, c)
return g
}
// AddMeta is used to add a meta k/v pair to a task group
func (g *TaskGroup) SetMeta(key, val string) *TaskGroup {
if g.Meta == nil {
g.Meta = make(map[string]string)
}
g.Meta[key] = val
return g
}
// AddTask is used to add a new task to a task group.
func (g *TaskGroup) AddTask(t *Task) *TaskGroup {
g.Tasks = append(g.Tasks, t)
return g
}
// LogConfig provides configuration for log rotation
type LogConfig struct {
MaxFiles int
MaxFileSizeMB int
}
// Task is a single process in a task group.
type Task struct {
Name string
Driver string
User string
Config map[string]interface{}
Constraints []*Constraint
Env map[string]string
Services []Service
Resources *Resources
Meta map[string]string
KillTimeout time.Duration
LogConfig *LogConfig
Artifacts []*TaskArtifact
}
// TaskArtifact is used to download artifacts before running a task.
type TaskArtifact struct {
GetterSource string
GetterOptions map[string]string
RelativeDest string
}
// NewTask creates and initializes a new Task.
func NewTask(name, driver string) *Task {
return &Task{
Name: name,
Driver: driver,
}
}
// Configure is used to configure a single k/v pair on
// the task.
func (t *Task) SetConfig(key, val string) *Task {
if t.Config == nil {
t.Config = make(map[string]interface{})
}
t.Config[key] = val
return t
}
// SetMeta is used to add metadata k/v pairs to the task.
func (t *Task) SetMeta(key, val string) *Task {
if t.Meta == nil {
t.Meta = make(map[string]string)
}
t.Meta[key] = val
return t
}
// Require is used to add resource requirements to a task.
func (t *Task) Require(r *Resources) *Task {
t.Resources = r
return t
}
// Constraint adds a new constraints to a single task.
func (t *Task) Constrain(c *Constraint) *Task {
t.Constraints = append(t.Constraints, c)
return t
}
// SetLogConfig sets a log config to a task
func (t *Task) SetLogConfig(l *LogConfig) *Task {
t.LogConfig = l
return t
}
// TaskState tracks the current state of a task and events that caused state
// transitions.
type TaskState struct {
State string
Events []*TaskEvent
}
const (
TaskDriverFailure = "Driver Failure"
TaskReceived = "Received"
TaskFailedValidation = "Failed Validation"
TaskStarted = "Started"
TaskTerminated = "Terminated"
TaskKilling = "Killing"
TaskKilled = "Killed"
TaskRestarting = "Restarting"
TaskNotRestarting = "Not Restarting"
TaskDownloadingArtifacts = "Downloading Artifacts"
TaskArtifactDownloadFailed = "Failed Artifact Download"
)
// TaskEvent is an event that effects the state of a task and contains meta-data
// appropriate to the events type.
type TaskEvent struct {
Type string
Time int64
RestartReason string
DriverError string
ExitCode int
Signal int
Message string
KillTimeout time.Duration
KillError string
StartDelay int64
DownloadError string
ValidationError string
}

View File

@@ -0,0 +1,579 @@
package client
import (
"fmt"
"log"
"os"
"path/filepath"
"sync"
"time"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/client/driver"
"github.com/hashicorp/nomad/nomad/structs"
cstructs "github.com/hashicorp/nomad/client/structs"
)
const (
// taskReceivedSyncLimit is how long the client will wait before sending
// that a task was received to the server. The client does not immediately
// send that the task was received to the server because another transition
// to running or failed is likely to occur immediately after and a single
// update will transfer all past state information. If not other transition
// has occurred up to this limit, we will send to the server.
taskReceivedSyncLimit = 30 * time.Second
)
// AllocStateUpdater is used to update the status of an allocation
type AllocStateUpdater func(alloc *structs.Allocation)
type AllocStatsReporter interface {
LatestAllocStats(taskFilter string) (*cstructs.AllocResourceUsage, error)
}
// AllocRunner is used to wrap an allocation and provide the execution context.
type AllocRunner struct {
config *config.Config
updater AllocStateUpdater
logger *log.Logger
alloc *structs.Allocation
allocClientStatus string // Explicit status of allocation. Set when there are failures
allocClientDescription string
allocLock sync.Mutex
dirtyCh chan struct{}
ctx *driver.ExecContext
ctxLock sync.Mutex
tasks map[string]*TaskRunner
taskStates map[string]*structs.TaskState
restored map[string]struct{}
taskLock sync.RWMutex
taskStatusLock sync.RWMutex
updateCh chan *structs.Allocation
destroy bool
destroyCh chan struct{}
destroyLock sync.Mutex
waitCh chan struct{}
}
// allocRunnerState is used to snapshot the state of the alloc runner
type allocRunnerState struct {
Version string
Alloc *structs.Allocation
AllocClientStatus string
AllocClientDescription string
TaskStates map[string]*structs.TaskState
Context *driver.ExecContext
}
// NewAllocRunner is used to create a new allocation context
func NewAllocRunner(logger *log.Logger, config *config.Config, updater AllocStateUpdater,
alloc *structs.Allocation) *AllocRunner {
ar := &AllocRunner{
config: config,
updater: updater,
logger: logger,
alloc: alloc,
dirtyCh: make(chan struct{}, 1),
tasks: make(map[string]*TaskRunner),
taskStates: copyTaskStates(alloc.TaskStates),
restored: make(map[string]struct{}),
updateCh: make(chan *structs.Allocation, 64),
destroyCh: make(chan struct{}),
waitCh: make(chan struct{}),
}
return ar
}
// stateFilePath returns the path to our state file
func (r *AllocRunner) stateFilePath() string {
r.allocLock.Lock()
defer r.allocLock.Unlock()
path := filepath.Join(r.config.StateDir, "alloc", r.alloc.ID, "state.json")
return path
}
// RestoreState is used to restore the state of the alloc runner
func (r *AllocRunner) RestoreState() error {
// Load the snapshot
var snap allocRunnerState
if err := restoreState(r.stateFilePath(), &snap); err != nil {
return err
}
// Restore fields
r.alloc = snap.Alloc
r.ctx = snap.Context
r.allocClientStatus = snap.AllocClientStatus
r.allocClientDescription = snap.AllocClientDescription
r.taskStates = snap.TaskStates
var snapshotErrors multierror.Error
if r.alloc == nil {
snapshotErrors.Errors = append(snapshotErrors.Errors, fmt.Errorf("alloc_runner snapshot includes a nil allocation"))
}
if r.ctx == nil {
snapshotErrors.Errors = append(snapshotErrors.Errors, fmt.Errorf("alloc_runner snapshot includes a nil context"))
}
if e := snapshotErrors.ErrorOrNil(); e != nil {
return e
}
// Restore the task runners
var mErr multierror.Error
for name, state := range r.taskStates {
// Mark the task as restored.
r.restored[name] = struct{}{}
task := &structs.Task{Name: name}
tr := NewTaskRunner(r.logger, r.config, r.setTaskState, r.ctx, r.Alloc(),
task)
r.tasks[name] = tr
// Skip tasks in terminal states.
if state.State == structs.TaskStateDead {
continue
}
if err := tr.RestoreState(); err != nil {
r.logger.Printf("[ERR] client: failed to restore state for alloc %s task '%s': %v", r.alloc.ID, name, err)
mErr.Errors = append(mErr.Errors, err)
} else if !r.alloc.TerminalStatus() {
// Only start if the alloc isn't in a terminal status.
go tr.Run()
}
}
return mErr.ErrorOrNil()
}
// SaveState is used to snapshot the state of the alloc runner
// if the fullSync is marked as false only the state of the Alloc Runner
// is snapshotted. If fullSync is marked as true, we snapshot
// all the Task Runners associated with the Alloc
func (r *AllocRunner) SaveState() error {
if err := r.saveAllocRunnerState(); err != nil {
return err
}
// Save state for each task
runners := r.getTaskRunners()
var mErr multierror.Error
for _, tr := range runners {
if err := r.saveTaskRunnerState(tr); err != nil {
mErr.Errors = append(mErr.Errors, err)
}
}
return mErr.ErrorOrNil()
}
func (r *AllocRunner) saveAllocRunnerState() error {
// Create the snapshot.
r.taskStatusLock.RLock()
states := copyTaskStates(r.taskStates)
r.taskStatusLock.RUnlock()
alloc := r.Alloc()
r.allocLock.Lock()
allocClientStatus := r.allocClientStatus
allocClientDescription := r.allocClientDescription
r.allocLock.Unlock()
r.ctxLock.Lock()
ctx := r.ctx
r.ctxLock.Unlock()
snap := allocRunnerState{
Version: r.config.Version,
Alloc: alloc,
Context: ctx,
AllocClientStatus: allocClientStatus,
AllocClientDescription: allocClientDescription,
TaskStates: states,
}
return persistState(r.stateFilePath(), &snap)
}
func (r *AllocRunner) saveTaskRunnerState(tr *TaskRunner) error {
if err := tr.SaveState(); err != nil {
return fmt.Errorf("failed to save state for alloc %s task '%s': %v",
r.alloc.ID, tr.task.Name, err)
}
return nil
}
// DestroyState is used to cleanup after ourselves
func (r *AllocRunner) DestroyState() error {
return os.RemoveAll(filepath.Dir(r.stateFilePath()))
}
// DestroyContext is used to destroy the context
func (r *AllocRunner) DestroyContext() error {
return r.ctx.AllocDir.Destroy()
}
// copyTaskStates returns a copy of the passed task states.
func copyTaskStates(states map[string]*structs.TaskState) map[string]*structs.TaskState {
copy := make(map[string]*structs.TaskState, len(states))
for task, state := range states {
copy[task] = state.Copy()
}
return copy
}
// Alloc returns the associated allocation
func (r *AllocRunner) Alloc() *structs.Allocation {
r.allocLock.Lock()
alloc := r.alloc.Copy()
// The status has explicitly been set.
if r.allocClientStatus != "" || r.allocClientDescription != "" {
alloc.ClientStatus = r.allocClientStatus
alloc.ClientDescription = r.allocClientDescription
r.allocLock.Unlock()
return alloc
}
r.allocLock.Unlock()
// Scan the task states to determine the status of the alloc
var pending, running, dead, failed bool
r.taskStatusLock.RLock()
alloc.TaskStates = copyTaskStates(r.taskStates)
for _, state := range r.taskStates {
switch state.State {
case structs.TaskStateRunning:
running = true
case structs.TaskStatePending:
pending = true
case structs.TaskStateDead:
if state.Failed() {
failed = true
} else {
dead = true
}
}
}
r.taskStatusLock.RUnlock()
// Determine the alloc status
if failed {
alloc.ClientStatus = structs.AllocClientStatusFailed
} else if running {
alloc.ClientStatus = structs.AllocClientStatusRunning
} else if pending {
alloc.ClientStatus = structs.AllocClientStatusPending
} else if dead {
alloc.ClientStatus = structs.AllocClientStatusComplete
}
return alloc
}
// dirtySyncState is used to watch for state being marked dirty to sync
func (r *AllocRunner) dirtySyncState() {
for {
select {
case <-r.dirtyCh:
r.syncStatus()
case <-r.destroyCh:
return
}
}
}
// syncStatus is used to run and sync the status when it changes
func (r *AllocRunner) syncStatus() error {
// Get a copy of our alloc, update status server side and sync to disk
alloc := r.Alloc()
r.updater(alloc)
return r.saveAllocRunnerState()
}
// setStatus is used to update the allocation status
func (r *AllocRunner) setStatus(status, desc string) {
r.allocLock.Lock()
r.allocClientStatus = status
r.allocClientDescription = desc
r.allocLock.Unlock()
select {
case r.dirtyCh <- struct{}{}:
default:
}
}
// setTaskState is used to set the status of a task
func (r *AllocRunner) setTaskState(taskName, state string, event *structs.TaskEvent) {
r.taskStatusLock.Lock()
defer r.taskStatusLock.Unlock()
taskState, ok := r.taskStates[taskName]
if !ok {
taskState = &structs.TaskState{}
r.taskStates[taskName] = taskState
}
// Set the tasks state.
taskState.State = state
r.appendTaskEvent(taskState, event)
// If the task failed, we should kill all the other tasks in the task group.
if state == structs.TaskStateDead && taskState.Failed() {
var destroyingTasks []string
for task, tr := range r.tasks {
if task != taskName {
destroyingTasks = append(destroyingTasks, task)
tr.Destroy()
}
}
if len(destroyingTasks) > 0 {
r.logger.Printf("[DEBUG] client: task %q failed, destroying other tasks in task group: %v", taskName, destroyingTasks)
}
}
select {
case r.dirtyCh <- struct{}{}:
default:
}
}
// appendTaskEvent updates the task status by appending the new event.
func (r *AllocRunner) appendTaskEvent(state *structs.TaskState, event *structs.TaskEvent) {
capacity := 10
if state.Events == nil {
state.Events = make([]*structs.TaskEvent, 0, capacity)
}
// If we hit capacity, then shift it.
if len(state.Events) == capacity {
old := state.Events
state.Events = make([]*structs.TaskEvent, 0, capacity)
state.Events = append(state.Events, old[1:]...)
}
state.Events = append(state.Events, event)
}
// Run is a long running goroutine used to manage an allocation
func (r *AllocRunner) Run() {
defer close(r.waitCh)
go r.dirtySyncState()
// Find the task group to run in the allocation
alloc := r.alloc
tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup)
if tg == nil {
r.logger.Printf("[ERR] client: alloc '%s' for missing task group '%s'", alloc.ID, alloc.TaskGroup)
r.setStatus(structs.AllocClientStatusFailed, fmt.Sprintf("missing task group '%s'", alloc.TaskGroup))
return
}
// Create the execution context
r.ctxLock.Lock()
if r.ctx == nil {
allocDir := allocdir.NewAllocDir(filepath.Join(r.config.AllocDir, r.alloc.ID))
if err := allocDir.Build(tg.Tasks); err != nil {
r.logger.Printf("[WARN] client: failed to build task directories: %v", err)
r.setStatus(structs.AllocClientStatusFailed, fmt.Sprintf("failed to build task dirs for '%s'", alloc.TaskGroup))
r.ctxLock.Unlock()
return
}
r.ctx = driver.NewExecContext(allocDir, r.alloc.ID)
}
r.ctxLock.Unlock()
// Check if the allocation is in a terminal status. In this case, we don't
// start any of the task runners and directly wait for the destroy signal to
// clean up the allocation.
if alloc.TerminalStatus() {
r.logger.Printf("[DEBUG] client: alloc %q in terminal status, waiting for destroy", r.alloc.ID)
r.handleDestroy()
r.logger.Printf("[DEBUG] client: terminating runner for alloc '%s'", r.alloc.ID)
return
}
// Start the task runners
r.logger.Printf("[DEBUG] client: starting task runners for alloc '%s'", r.alloc.ID)
r.taskLock.Lock()
for _, task := range tg.Tasks {
if _, ok := r.restored[task.Name]; ok {
continue
}
tr := NewTaskRunner(r.logger, r.config, r.setTaskState, r.ctx, r.Alloc(),
task.Copy())
r.tasks[task.Name] = tr
tr.MarkReceived()
go tr.Run()
}
r.taskLock.Unlock()
OUTER:
// Wait for updates
for {
select {
case update := <-r.updateCh:
// Store the updated allocation.
r.allocLock.Lock()
r.alloc = update
r.allocLock.Unlock()
// Check if we're in a terminal status
if update.TerminalStatus() {
break OUTER
}
// Update the task groups
runners := r.getTaskRunners()
for _, tr := range runners {
tr.Update(update)
}
case <-r.destroyCh:
break OUTER
}
}
// Destroy each sub-task
runners := r.getTaskRunners()
for _, tr := range runners {
tr.Destroy()
}
// Wait for termination of the task runners
for _, tr := range runners {
<-tr.WaitCh()
}
// Final state sync
r.syncStatus()
// Block until we should destroy the state of the alloc
r.handleDestroy()
r.logger.Printf("[DEBUG] client: terminating runner for alloc '%s'", r.alloc.ID)
}
// handleDestroy blocks till the AllocRunner should be destroyed and does the
// necessary cleanup.
func (r *AllocRunner) handleDestroy() {
select {
case <-r.destroyCh:
if err := r.DestroyContext(); err != nil {
r.logger.Printf("[ERR] client: failed to destroy context for alloc '%s': %v",
r.alloc.ID, err)
}
if err := r.DestroyState(); err != nil {
r.logger.Printf("[ERR] client: failed to destroy state for alloc '%s': %v",
r.alloc.ID, err)
}
}
}
// Update is used to update the allocation of the context
func (r *AllocRunner) Update(update *structs.Allocation) {
select {
case r.updateCh <- update:
default:
r.logger.Printf("[ERR] client: dropping update to alloc '%s'", update.ID)
}
}
// StatsReporter returns an interface to query resource usage statistics of an
// allocation
func (r *AllocRunner) StatsReporter() AllocStatsReporter {
return r
}
// getTaskRunners is a helper that returns a copy of the task runners list using
// the taskLock.
func (r *AllocRunner) getTaskRunners() []*TaskRunner {
// Get the task runners
r.taskLock.RLock()
defer r.taskLock.RUnlock()
runners := make([]*TaskRunner, 0, len(r.tasks))
for _, tr := range r.tasks {
runners = append(runners, tr)
}
return runners
}
// LatestAllocStats returns the latest allocation stats. If the optional taskFilter is set
// the allocation stats will only include the given task.
func (r *AllocRunner) LatestAllocStats(taskFilter string) (*cstructs.AllocResourceUsage, error) {
astat := &cstructs.AllocResourceUsage{
Tasks: make(map[string]*cstructs.TaskResourceUsage),
}
var flat []*cstructs.TaskResourceUsage
if taskFilter != "" {
r.taskLock.RLock()
tr, ok := r.tasks[taskFilter]
r.taskLock.RUnlock()
if !ok {
return nil, fmt.Errorf("allocation %q has no task %q", r.alloc.ID, taskFilter)
}
l := tr.LatestResourceUsage()
if l != nil {
astat.Tasks[taskFilter] = l
flat = []*cstructs.TaskResourceUsage{l}
astat.Timestamp = l.Timestamp
}
} else {
// Get the task runners
runners := r.getTaskRunners()
for _, tr := range runners {
l := tr.LatestResourceUsage()
if l != nil {
astat.Tasks[tr.task.Name] = l
flat = append(flat, l)
if l.Timestamp > astat.Timestamp {
astat.Timestamp = l.Timestamp
}
}
}
}
astat.ResourceUsage = sumTaskResourceUsage(flat)
return astat, nil
}
// sumTaskResourceUsage takes a set of task resources and sums their resources
func sumTaskResourceUsage(usages []*cstructs.TaskResourceUsage) *cstructs.ResourceUsage {
summed := &cstructs.ResourceUsage{
MemoryStats: &cstructs.MemoryStats{},
CpuStats: &cstructs.CpuStats{},
}
for _, usage := range usages {
summed.Add(usage.ResourceUsage)
}
return summed
}
// shouldUpdate takes the AllocModifyIndex of an allocation sent from the server and
// checks if the current running allocation is behind and should be updated.
func (r *AllocRunner) shouldUpdate(serverIndex uint64) bool {
r.allocLock.Lock()
defer r.allocLock.Unlock()
return r.alloc.AllocModifyIndex < serverIndex
}
// Destroy is used to indicate that the allocation context should be destroyed
func (r *AllocRunner) Destroy() {
r.destroyLock.Lock()
defer r.destroyLock.Unlock()
if r.destroy {
return
}
r.destroy = true
close(r.destroyCh)
}
// WaitCh returns a channel to wait for termination
func (r *AllocRunner) WaitCh() <-chan struct{} {
return r.waitCh
}

View File

@@ -0,0 +1,399 @@
package allocdir
import (
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"time"
"gopkg.in/tomb.v1"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hpcloud/tail/watch"
)
var (
// The name of the directory that is shared across tasks in a task group.
SharedAllocName = "alloc"
// Name of the directory where logs of Tasks are written
LogDirName = "logs"
// The set of directories that exist inside eache shared alloc directory.
SharedAllocDirs = []string{LogDirName, "tmp", "data"}
// The name of the directory that exists inside each task directory
// regardless of driver.
TaskLocal = "local"
// TaskDirs is the set of directories created in each tasks directory.
TaskDirs = []string{"tmp"}
)
type AllocDir struct {
// AllocDir is the directory used for storing any state
// of this allocation. It will be purged on alloc destroy.
AllocDir string
// The shared directory is available to all tasks within the same task
// group.
SharedDir string
// TaskDirs is a mapping of task names to their non-shared directory.
TaskDirs map[string]string
}
// AllocFileInfo holds information about a file inside the AllocDir
type AllocFileInfo struct {
Name string
IsDir bool
Size int64
FileMode string
ModTime time.Time
}
// AllocDirFS exposes file operations on the alloc dir
type AllocDirFS interface {
List(path string) ([]*AllocFileInfo, error)
Stat(path string) (*AllocFileInfo, error)
ReadAt(path string, offset int64) (io.ReadCloser, error)
BlockUntilExists(path string, t *tomb.Tomb) chan error
ChangeEvents(path string, curOffset int64, t *tomb.Tomb) (*watch.FileChanges, error)
}
func NewAllocDir(allocDir string) *AllocDir {
d := &AllocDir{AllocDir: allocDir, TaskDirs: make(map[string]string)}
d.SharedDir = filepath.Join(d.AllocDir, SharedAllocName)
return d
}
// Tears down previously build directory structure.
func (d *AllocDir) Destroy() error {
// Unmount all mounted shared alloc dirs.
var mErr multierror.Error
if err := d.UnmountAll(); err != nil {
mErr.Errors = append(mErr.Errors, err)
}
if err := os.RemoveAll(d.AllocDir); err != nil {
mErr.Errors = append(mErr.Errors, err)
}
return mErr.ErrorOrNil()
}
func (d *AllocDir) UnmountAll() error {
var mErr multierror.Error
for _, dir := range d.TaskDirs {
// Check if the directory has the shared alloc mounted.
taskAlloc := filepath.Join(dir, SharedAllocName)
if d.pathExists(taskAlloc) {
if err := d.unmountSharedDir(taskAlloc); err != nil {
mErr.Errors = append(mErr.Errors,
fmt.Errorf("failed to unmount shared alloc dir %q: %v", taskAlloc, err))
} else if err := os.RemoveAll(taskAlloc); err != nil {
mErr.Errors = append(mErr.Errors,
fmt.Errorf("failed to delete shared alloc dir %q: %v", taskAlloc, err))
}
}
// Unmount dev/ and proc/ have been mounted.
d.unmountSpecialDirs(dir)
}
return mErr.ErrorOrNil()
}
// Given a list of a task build the correct alloc structure.
func (d *AllocDir) Build(tasks []*structs.Task) error {
// Make the alloc directory, owned by the nomad process.
if err := os.MkdirAll(d.AllocDir, 0755); err != nil {
return fmt.Errorf("Failed to make the alloc directory %v: %v", d.AllocDir, err)
}
// Make the shared directory and make it available to all user/groups.
if err := os.MkdirAll(d.SharedDir, 0777); err != nil {
return err
}
// Make the shared directory have non-root permissions.
if err := d.dropDirPermissions(d.SharedDir); err != nil {
return err
}
for _, dir := range SharedAllocDirs {
p := filepath.Join(d.SharedDir, dir)
if err := os.MkdirAll(p, 0777); err != nil {
return err
}
if err := d.dropDirPermissions(p); err != nil {
return err
}
}
// Make the task directories.
for _, t := range tasks {
taskDir := filepath.Join(d.AllocDir, t.Name)
if err := os.MkdirAll(taskDir, 0777); err != nil {
return err
}
// Make the task directory have non-root permissions.
if err := d.dropDirPermissions(taskDir); err != nil {
return err
}
// Create a local directory that each task can use.
local := filepath.Join(taskDir, TaskLocal)
if err := os.MkdirAll(local, 0777); err != nil {
return err
}
if err := d.dropDirPermissions(local); err != nil {
return err
}
d.TaskDirs[t.Name] = taskDir
// Create the directories that should be in every task.
for _, dir := range TaskDirs {
local := filepath.Join(taskDir, dir)
if err := os.MkdirAll(local, 0777); err != nil {
return err
}
if err := d.dropDirPermissions(local); err != nil {
return err
}
}
}
return nil
}
// Embed takes a mapping of absolute directory or file paths on the host to
// their intended, relative location within the task directory. Embed attempts
// hardlink and then defaults to copying. If the path exists on the host and
// can't be embedded an error is returned.
func (d *AllocDir) Embed(task string, entries map[string]string) error {
taskdir, ok := d.TaskDirs[task]
if !ok {
return fmt.Errorf("Task directory doesn't exist for task %v", task)
}
subdirs := make(map[string]string)
for source, dest := range entries {
// Check to see if directory exists on host.
s, err := os.Stat(source)
if os.IsNotExist(err) {
continue
}
// Embedding a single file
if !s.IsDir() {
destDir := filepath.Join(taskdir, filepath.Dir(dest))
if err := os.MkdirAll(destDir, s.Mode().Perm()); err != nil {
return fmt.Errorf("Couldn't create destination directory %v: %v", destDir, err)
}
// Copy the file.
taskEntry := filepath.Join(destDir, filepath.Base(dest))
if err := d.linkOrCopy(source, taskEntry, s.Mode().Perm()); err != nil {
return err
}
continue
}
// Create destination directory.
destDir := filepath.Join(taskdir, dest)
if err := os.MkdirAll(destDir, s.Mode().Perm()); err != nil {
return fmt.Errorf("Couldn't create destination directory %v: %v", destDir, err)
}
// Enumerate the files in source.
dirEntries, err := ioutil.ReadDir(source)
if err != nil {
return fmt.Errorf("Couldn't read directory %v: %v", source, err)
}
for _, entry := range dirEntries {
hostEntry := filepath.Join(source, entry.Name())
taskEntry := filepath.Join(destDir, filepath.Base(hostEntry))
if entry.IsDir() {
subdirs[hostEntry] = filepath.Join(dest, filepath.Base(hostEntry))
continue
}
// Check if entry exists. This can happen if restarting a failed
// task.
if _, err := os.Lstat(taskEntry); err == nil {
continue
}
if !entry.Mode().IsRegular() {
// If it is a symlink we can create it, otherwise we skip it.
if entry.Mode()&os.ModeSymlink == 0 {
continue
}
link, err := os.Readlink(hostEntry)
if err != nil {
return fmt.Errorf("Couldn't resolve symlink for %v: %v", source, err)
}
if err := os.Symlink(link, taskEntry); err != nil {
// Symlinking twice
if err.(*os.LinkError).Err.Error() != "file exists" {
return fmt.Errorf("Couldn't create symlink: %v", err)
}
}
continue
}
if err := d.linkOrCopy(hostEntry, taskEntry, entry.Mode().Perm()); err != nil {
return err
}
}
}
// Recurse on self to copy subdirectories.
if len(subdirs) != 0 {
return d.Embed(task, subdirs)
}
return nil
}
// MountSharedDir mounts the shared directory into the specified task's
// directory. Mount is documented at an OS level in their respective
// implementation files.
func (d *AllocDir) MountSharedDir(task string) error {
taskDir, ok := d.TaskDirs[task]
if !ok {
return fmt.Errorf("No task directory exists for %v", task)
}
taskLoc := filepath.Join(taskDir, SharedAllocName)
if err := d.mountSharedDir(taskLoc); err != nil {
return fmt.Errorf("Failed to mount shared directory for task %v: %v", task, err)
}
return nil
}
// LogDir returns the log dir in the current allocation directory
func (d *AllocDir) LogDir() string {
return filepath.Join(d.AllocDir, SharedAllocName, LogDirName)
}
// List returns the list of files at a path relative to the alloc dir
func (d *AllocDir) List(path string) ([]*AllocFileInfo, error) {
p := filepath.Join(d.AllocDir, path)
finfos, err := ioutil.ReadDir(p)
if err != nil {
return []*AllocFileInfo{}, err
}
files := make([]*AllocFileInfo, len(finfos))
for idx, info := range finfos {
files[idx] = &AllocFileInfo{
Name: info.Name(),
IsDir: info.IsDir(),
Size: info.Size(),
FileMode: info.Mode().String(),
ModTime: info.ModTime(),
}
}
return files, err
}
// Stat returns information about the file at a path relative to the alloc dir
func (d *AllocDir) Stat(path string) (*AllocFileInfo, error) {
p := filepath.Join(d.AllocDir, path)
info, err := os.Stat(p)
if err != nil {
return nil, err
}
return &AllocFileInfo{
Size: info.Size(),
Name: info.Name(),
IsDir: info.IsDir(),
FileMode: info.Mode().String(),
ModTime: info.ModTime(),
}, nil
}
// ReadAt returns a reader for a file at the path relative to the alloc dir
func (d *AllocDir) ReadAt(path string, offset int64) (io.ReadCloser, error) {
p := filepath.Join(d.AllocDir, path)
f, err := os.Open(p)
if err != nil {
return nil, err
}
if _, err := f.Seek(offset, 0); err != nil {
return nil, fmt.Errorf("can't seek to offset %q: %v", offset, err)
}
return f, nil
}
// BlockUntilExists blocks until the passed file relative the allocation
// directory exists. The block can be cancelled with the passed tomb.
func (d *AllocDir) BlockUntilExists(path string, t *tomb.Tomb) chan error {
// Get the path relative to the alloc directory
p := filepath.Join(d.AllocDir, path)
watcher := getFileWatcher(p)
returnCh := make(chan error, 1)
go func() {
returnCh <- watcher.BlockUntilExists(t)
close(returnCh)
}()
return returnCh
}
// ChangeEvents watches for changes to the passed path relative to the
// allocation directory. The offset should be the last read offset. The tomb is
// used to clean up the watch.
func (d *AllocDir) ChangeEvents(path string, curOffset int64, t *tomb.Tomb) (*watch.FileChanges, error) {
// Get the path relative to the alloc directory
p := filepath.Join(d.AllocDir, path)
watcher := getFileWatcher(p)
return watcher.ChangeEvents(t, curOffset)
}
// getFileWatcher returns a FileWatcher for the given path.
func getFileWatcher(path string) watch.FileWatcher {
return watch.NewPollingFileWatcher(path)
}
func fileCopy(src, dst string, perm os.FileMode) error {
// Do a simple copy.
srcFile, err := os.Open(src)
if err != nil {
return fmt.Errorf("Couldn't open src file %v: %v", src, err)
}
dstFile, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE, perm)
if err != nil {
return fmt.Errorf("Couldn't create destination file %v: %v", dst, err)
}
if _, err := io.Copy(dstFile, srcFile); err != nil {
return fmt.Errorf("Couldn't copy %v to %v: %v", src, dst, err)
}
return nil
}
// pathExists is a helper function to check if the path exists.
func (d *AllocDir) pathExists(path string) bool {
if _, err := os.Stat(path); err != nil {
if os.IsNotExist(err) {
return false
}
}
return true
}

View File

@@ -0,0 +1,26 @@
package allocdir
import (
"syscall"
)
// Hardlinks the shared directory. As a side-effect the shared directory and
// task directory must be on the same filesystem.
func (d *AllocDir) mountSharedDir(dir string) error {
return syscall.Link(d.SharedDir, dir)
}
func (d *AllocDir) unmountSharedDir(dir string) error {
return syscall.Unlink(dir)
}
// MountSpecialDirs mounts the dev and proc file system on the chroot of the
// task. It's a no-op on darwin.
func (d *AllocDir) MountSpecialDirs(taskDir string) error {
return nil
}
// unmountSpecialDirs unmounts the dev and proc file system from the chroot
func (d *AllocDir) unmountSpecialDirs(taskDir string) error {
return nil
}

View File

@@ -0,0 +1,26 @@
package allocdir
import (
"syscall"
)
// Hardlinks the shared directory. As a side-effect the shared directory and
// task directory must be on the same filesystem.
func (d *AllocDir) mountSharedDir(dir string) error {
return syscall.Link(d.SharedDir, dir)
}
func (d *AllocDir) unmountSharedDir(dir string) error {
return syscall.Unlink(dir)
}
// MountSpecialDirs mounts the dev and proc file system on the chroot of the
// task. It's a no-op on FreeBSD right now.
func (d *AllocDir) MountSpecialDirs(taskDir string) error {
return nil
}
// unmountSpecialDirs unmounts the dev and proc file system from the chroot
func (d *AllocDir) unmountSpecialDirs(taskDir string) error {
return nil
}

View File

@@ -0,0 +1,79 @@
package allocdir
import (
"fmt"
"os"
"path/filepath"
"syscall"
"github.com/hashicorp/go-multierror"
)
// Bind mounts the shared directory into the task directory. Must be root to
// run.
func (d *AllocDir) mountSharedDir(taskDir string) error {
if err := os.MkdirAll(taskDir, 0777); err != nil {
return err
}
return syscall.Mount(d.SharedDir, taskDir, "", syscall.MS_BIND, "")
}
func (d *AllocDir) unmountSharedDir(dir string) error {
return syscall.Unmount(dir, 0)
}
// MountSpecialDirs mounts the dev and proc file system from the host to the
// chroot
func (d *AllocDir) MountSpecialDirs(taskDir string) error {
// Mount dev
dev := filepath.Join(taskDir, "dev")
if !d.pathExists(dev) {
if err := os.MkdirAll(dev, 0777); err != nil {
return fmt.Errorf("Mkdir(%v) failed: %v", dev, err)
}
if err := syscall.Mount("none", dev, "devtmpfs", syscall.MS_RDONLY, ""); err != nil {
return fmt.Errorf("Couldn't mount /dev to %v: %v", dev, err)
}
}
// Mount proc
proc := filepath.Join(taskDir, "proc")
if !d.pathExists(proc) {
if err := os.MkdirAll(proc, 0777); err != nil {
return fmt.Errorf("Mkdir(%v) failed: %v", proc, err)
}
if err := syscall.Mount("none", proc, "proc", syscall.MS_RDONLY, ""); err != nil {
return fmt.Errorf("Couldn't mount /proc to %v: %v", proc, err)
}
}
return nil
}
// unmountSpecialDirs unmounts the dev and proc file system from the chroot
func (d *AllocDir) unmountSpecialDirs(taskDir string) error {
errs := new(multierror.Error)
dev := filepath.Join(taskDir, "dev")
if d.pathExists(dev) {
if err := syscall.Unmount(dev, 0); err != nil {
errs = multierror.Append(errs, fmt.Errorf("Failed to unmount dev (%v): %v", dev, err))
} else if err := os.RemoveAll(dev); err != nil {
errs = multierror.Append(errs, fmt.Errorf("Failed to delete dev directory (%v): %v", dev, err))
}
}
// Unmount proc.
proc := filepath.Join(taskDir, "proc")
if d.pathExists(proc) {
if err := syscall.Unmount(proc, 0); err != nil {
errs = multierror.Append(errs, fmt.Errorf("Failed to unmount proc (%v): %v", proc, err))
} else if err := os.RemoveAll(proc); err != nil {
errs = multierror.Append(errs, fmt.Errorf("Failed to delete proc directory (%v): %v", dev, err))
}
}
return errs.ErrorOrNil()
}

View File

@@ -0,0 +1,81 @@
// +build darwin dragonfly freebsd linux netbsd openbsd solaris
// Functions shared between linux/darwin.
package allocdir
import (
"fmt"
"os"
"os/user"
"path/filepath"
"strconv"
"golang.org/x/sys/unix"
)
var (
//Path inside container for mounted directory shared across tasks in a task group.
SharedAllocContainerPath = filepath.Join("/", SharedAllocName)
//Path inside container for mounted directory for local storage.
TaskLocalContainerPath = filepath.Join("/", TaskLocal)
)
func (d *AllocDir) linkOrCopy(src, dst string, perm os.FileMode) error {
// Attempt to hardlink.
if err := os.Link(src, dst); err == nil {
return nil
}
return fileCopy(src, dst, perm)
}
func (d *AllocDir) dropDirPermissions(path string) error {
// Can't do anything if not root.
if unix.Geteuid() != 0 {
return nil
}
u, err := user.Lookup("nobody")
if err != nil {
return err
}
uid, err := getUid(u)
if err != nil {
return err
}
gid, err := getGid(u)
if err != nil {
return err
}
if err := os.Chown(path, uid, gid); err != nil {
return fmt.Errorf("Couldn't change owner/group of %v to (uid: %v, gid: %v): %v", path, uid, gid, err)
}
if err := os.Chmod(path, 0777); err != nil {
return fmt.Errorf("Chmod(%v) failed: %v", path, err)
}
return nil
}
func getUid(u *user.User) (int, error) {
uid, err := strconv.Atoi(u.Uid)
if err != nil {
return 0, fmt.Errorf("Unable to convert Uid to an int: %v", err)
}
return uid, nil
}
func getGid(u *user.User) (int, error) {
gid, err := strconv.Atoi(u.Gid)
if err != nil {
return 0, fmt.Errorf("Unable to convert Gid to an int: %v", err)
}
return gid, nil
}

View File

@@ -0,0 +1,45 @@
package allocdir
import (
"errors"
"os"
"path/filepath"
)
var (
//Path inside container for mounted directory that is shared across tasks in a task group.
SharedAllocContainerPath = filepath.Join("c:\\", SharedAllocName)
//Path inside container for mounted directory for local storage.
TaskLocalContainerPath = filepath.Join("c:\\", TaskLocal)
)
func (d *AllocDir) linkOrCopy(src, dst string, perm os.FileMode) error {
return fileCopy(src, dst, perm)
}
// The windows version does nothing currently.
func (d *AllocDir) mountSharedDir(dir string) error {
return errors.New("Mount on Windows not supported.")
}
// The windows version does nothing currently.
func (d *AllocDir) dropDirPermissions(path string) error {
return nil
}
// The windows version does nothing currently.
func (d *AllocDir) unmountSharedDir(dir string) error {
return nil
}
// MountSpecialDirs mounts the dev and proc file system on the chroot of the
// task. It's a no-op on windows.
func (d *AllocDir) MountSpecialDirs(taskDir string) error {
return nil
}
// unmountSpecialDirs unmounts the dev and proc file system from the chroot
func (d *AllocDir) unmountSpecialDirs(taskDir string) error {
return nil
}

1450
vendor/github.com/hashicorp/nomad/client/client.go generated vendored Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,221 @@
package config
import (
"fmt"
"io"
"os"
"strconv"
"strings"
"time"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/nomad/structs/config"
)
var (
// DefaultEnvBlacklist is the default set of environment variables that are
// filtered when passing the environment variables of the host to a task.
DefaultEnvBlacklist = strings.Join([]string{
"CONSUL_TOKEN",
"VAULT_TOKEN",
"ATLAS_TOKEN",
"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN",
"GOOGLE_APPLICATION_CREDENTIALS",
}, ",")
// DefaulUserBlacklist is the default set of users that tasks are not
// allowed to run as when using a driver in "user.checked_drivers"
DefaultUserBlacklist = strings.Join([]string{
"root",
"Administrator",
}, ",")
// DefaultUserCheckedDrivers is the set of drivers we apply the user
// blacklist onto. For virtualized drivers it often doesn't make sense to
// make this stipulation so by default they are ignored.
DefaultUserCheckedDrivers = strings.Join([]string{
"exec",
"qemu",
"java",
}, ",")
)
// RPCHandler can be provided to the Client if there is a local server
// to avoid going over the network. If not provided, the Client will
// maintain a connection pool to the servers
type RPCHandler interface {
RPC(method string, args interface{}, reply interface{}) error
}
// Config is used to parameterize and configure the behavior of the client
type Config struct {
// DevMode controls if we are in a development mode which
// avoids persistent storage.
DevMode bool
// StateDir is where we store our state
StateDir string
// AllocDir is where we store data for allocations
AllocDir string
// LogOutput is the destination for logs
LogOutput io.Writer
// Region is the clients region
Region string
// Network interface to be used in network fingerprinting
NetworkInterface string
// Network speed is the default speed of network interfaces if they can not
// be determined dynamically.
NetworkSpeed int
// MaxKillTimeout allows capping the user-specifiable KillTimeout. If the
// task's KillTimeout is greater than the MaxKillTimeout, MaxKillTimeout is
// used.
MaxKillTimeout time.Duration
// Servers is a list of known server addresses. These are as "host:port"
Servers []string
// RPCHandler can be provided to avoid network traffic if the
// server is running locally.
RPCHandler RPCHandler
// Node provides the base node
Node *structs.Node
// ClientMaxPort is the upper range of the ports that the client uses for
// communicating with plugin subsystems over loopback
ClientMaxPort uint
// ClientMinPort is the lower range of the ports that the client uses for
// communicating with plugin subsystems over loopback
ClientMinPort uint
// GloballyReservedPorts are ports that are reserved across all network
// devices and IPs.
GloballyReservedPorts []int
// A mapping of directories on the host OS to attempt to embed inside each
// task's chroot.
ChrootEnv map[string]string
// Options provides arbitrary key-value configuration for nomad internals,
// like fingerprinters and drivers. The format is:
//
// namespace.option = value
Options map[string]string
// Version is the version of the Nomad client
Version string
// Revision is the commit number of the Nomad client
Revision string
// ConsulConfig is this Agent's Consul configuration
ConsulConfig *config.ConsulConfig
// StatsCollectionInterval is the interval at which the Nomad client
// collects resource usage stats
StatsCollectionInterval time.Duration
// PublishNodeMetrics determines whether nomad is going to publish node
// level metrics to remote Telemetry sinks
PublishNodeMetrics bool
// PublishAllocationMetrics determines whether nomad is going to publish
// allocation metrics to remote Telemetry sinks
PublishAllocationMetrics bool
}
func (c *Config) Copy() *Config {
nc := new(Config)
*nc = *c
nc.Node = nc.Node.Copy()
nc.Servers = structs.CopySliceString(nc.Servers)
nc.Options = structs.CopyMapStringString(nc.Options)
return nc
}
// DefaultConfig returns the default configuration
func DefaultConfig() *Config {
return &Config{
ConsulConfig: config.DefaultConsulConfig(),
LogOutput: os.Stderr,
Region: "global",
StatsCollectionInterval: 1 * time.Second,
}
}
// Read returns the specified configuration value or "".
func (c *Config) Read(id string) string {
return c.Options[id]
}
// ReadDefault returns the specified configuration value, or the specified
// default value if none is set.
func (c *Config) ReadDefault(id string, defaultValue string) string {
val, ok := c.Options[id]
if !ok {
return defaultValue
}
return val
}
// ReadBool parses the specified option as a boolean.
func (c *Config) ReadBool(id string) (bool, error) {
val, ok := c.Options[id]
if !ok {
return false, fmt.Errorf("Specified config is missing from options")
}
bval, err := strconv.ParseBool(val)
if err != nil {
return false, fmt.Errorf("Failed to parse %s as bool: %s", val, err)
}
return bval, nil
}
// ReadBoolDefault tries to parse the specified option as a boolean. If there is
// an error in parsing, the default option is returned.
func (c *Config) ReadBoolDefault(id string, defaultValue bool) bool {
val, err := c.ReadBool(id)
if err != nil {
return defaultValue
}
return val
}
// ReadStringListToMap tries to parse the specified option as a comma separated list.
// If there is an error in parsing, an empty list is returned.
func (c *Config) ReadStringListToMap(key string) map[string]struct{} {
s := strings.TrimSpace(c.Read(key))
list := make(map[string]struct{})
if s != "" {
for _, e := range strings.Split(s, ",") {
trimmed := strings.TrimSpace(e)
list[trimmed] = struct{}{}
}
}
return list
}
// ReadStringListToMap tries to parse the specified option as a comma separated list.
// If there is an error in parsing, an empty list is returned.
func (c *Config) ReadStringListToMapDefault(key, defaultValue string) map[string]struct{} {
val, ok := c.Options[key]
if !ok {
val = defaultValue
}
list := make(map[string]struct{})
if val != "" {
for _, e := range strings.Split(val, ",") {
trimmed := strings.TrimSpace(e)
list[trimmed] = struct{}{}
}
}
return list
}

1085
vendor/github.com/hashicorp/nomad/client/driver/docker.go generated vendored Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,14 @@
//+build !windows
package driver
import docker "github.com/fsouza/go-dockerclient"
const (
// Setting default network mode for non-windows OS as bridge
defaultNetworkMode = "bridge"
)
func getPortBinding(ip string, port string) []docker.PortBinding {
return []docker.PortBinding{docker.PortBinding{HostIP: ip, HostPort: port}}
}

View File

@@ -0,0 +1,13 @@
package driver
import docker "github.com/fsouza/go-dockerclient"
const (
// Default network mode for windows containers is nat
defaultNetworkMode = "nat"
)
//Currently Windows containers don't support host ip in port binding.
func getPortBinding(ip string, port string) []docker.PortBinding {
return []docker.PortBinding{docker.PortBinding{HostIP: "", HostPort: port}}
}

View File

@@ -0,0 +1,192 @@
package driver
import (
"fmt"
"log"
"path/filepath"
"sync"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/client/driver/env"
"github.com/hashicorp/nomad/client/fingerprint"
"github.com/hashicorp/nomad/nomad/structs"
dstructs "github.com/hashicorp/nomad/client/driver/structs"
cstructs "github.com/hashicorp/nomad/client/structs"
)
// BuiltinDrivers contains the built in registered drivers
// which are available for allocation handling
var BuiltinDrivers = map[string]Factory{
"docker": NewDockerDriver,
"exec": NewExecDriver,
"raw_exec": NewRawExecDriver,
"java": NewJavaDriver,
"qemu": NewQemuDriver,
"rkt": NewRktDriver,
}
// NewDriver is used to instantiate and return a new driver
// given the name and a logger
func NewDriver(name string, ctx *DriverContext) (Driver, error) {
// Lookup the factory function
factory, ok := BuiltinDrivers[name]
if !ok {
return nil, fmt.Errorf("unknown driver '%s'", name)
}
// Instantiate the driver
f := factory(ctx)
return f, nil
}
// Factory is used to instantiate a new Driver
type Factory func(*DriverContext) Driver
// Driver is used for execution of tasks. This allows Nomad
// to support many pluggable implementations of task drivers.
// Examples could include LXC, Docker, Qemu, etc.
type Driver interface {
// Drivers must support the fingerprint interface for detection
fingerprint.Fingerprint
// Start is used to being task execution
Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error)
// Open is used to re-open a handle to a task
Open(ctx *ExecContext, handleID string) (DriverHandle, error)
// Drivers must validate their configuration
Validate(map[string]interface{}) error
}
// DriverContext is a means to inject dependencies such as loggers, configs, and
// node attributes into a Driver without having to change the Driver interface
// each time we do it. Used in conjection with Factory, above.
type DriverContext struct {
taskName string
config *config.Config
logger *log.Logger
node *structs.Node
taskEnv *env.TaskEnvironment
}
// NewEmptyDriverContext returns a DriverContext with all fields set to their
// zero value.
func NewEmptyDriverContext() *DriverContext {
return &DriverContext{
taskName: "",
config: nil,
node: nil,
logger: nil,
taskEnv: nil,
}
}
// NewDriverContext initializes a new DriverContext with the specified fields.
// This enables other packages to create DriverContexts but keeps the fields
// private to the driver. If we want to change this later we can gorename all of
// the fields in DriverContext.
func NewDriverContext(taskName string, config *config.Config, node *structs.Node,
logger *log.Logger, taskEnv *env.TaskEnvironment) *DriverContext {
return &DriverContext{
taskName: taskName,
config: config,
node: node,
logger: logger,
taskEnv: taskEnv,
}
}
// DriverHandle is an opaque handle into a driver used for task
// manipulation
type DriverHandle interface {
// Returns an opaque handle that can be used to re-open the handle
ID() string
// WaitCh is used to return a channel used wait for task completion
WaitCh() chan *dstructs.WaitResult
// Update is used to update the task if possible and update task related
// configurations.
Update(task *structs.Task) error
// Kill is used to stop the task
Kill() error
// Stats returns aggregated stats of the driver
Stats() (*cstructs.TaskResourceUsage, error)
}
// ExecContext is shared between drivers within an allocation
type ExecContext struct {
sync.Mutex
// AllocDir contains information about the alloc directory structure.
AllocDir *allocdir.AllocDir
// Alloc ID
AllocID string
}
// NewExecContext is used to create a new execution context
func NewExecContext(alloc *allocdir.AllocDir, allocID string) *ExecContext {
return &ExecContext{AllocDir: alloc, AllocID: allocID}
}
// GetTaskEnv converts the alloc dir, the node, task and alloc into a
// TaskEnvironment.
func GetTaskEnv(allocDir *allocdir.AllocDir, node *structs.Node,
task *structs.Task, alloc *structs.Allocation) (*env.TaskEnvironment, error) {
tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup)
env := env.NewTaskEnvironment(node).
SetTaskMeta(task.Meta).
SetTaskGroupMeta(tg.Meta).
SetJobMeta(alloc.Job.Meta).
SetEnvvars(task.Env).
SetTaskName(task.Name)
if allocDir != nil {
env.SetAllocDir(allocDir.SharedDir)
taskdir, ok := allocDir.TaskDirs[task.Name]
if !ok {
return nil, fmt.Errorf("failed to get task directory for task %q", task.Name)
}
env.SetTaskLocalDir(filepath.Join(taskdir, allocdir.TaskLocal))
}
if task.Resources != nil {
env.SetMemLimit(task.Resources.MemoryMB).
SetCpuLimit(task.Resources.CPU).
SetNetworks(task.Resources.Networks)
}
if alloc != nil {
env.SetAlloc(alloc)
}
return env.Build(), nil
}
func mapMergeStrInt(maps ...map[string]int) map[string]int {
out := map[string]int{}
for _, in := range maps {
for key, val := range in {
out[key] = val
}
}
return out
}
func mapMergeStrStr(maps ...map[string]string) map[string]string {
out := map[string]string{}
for _, in := range maps {
for key, val := range in {
out[key] = val
}
}
return out
}

View File

@@ -0,0 +1,430 @@
package env
import (
"fmt"
"os"
"strconv"
"strings"
hargs "github.com/hashicorp/nomad/helper/args"
"github.com/hashicorp/nomad/nomad/structs"
)
// A set of environment variables that are exported by each driver.
const (
// AllocDir is the environment variable with the path to the alloc directory
// that is shared across tasks within a task group.
AllocDir = "NOMAD_ALLOC_DIR"
// TaskLocalDir is the environment variable with the path to the tasks local
// directory where it can store data that is persisted to the alloc is
// removed.
TaskLocalDir = "NOMAD_TASK_DIR"
// MemLimit is the environment variable with the tasks memory limit in MBs.
MemLimit = "NOMAD_MEMORY_LIMIT"
// CpuLimit is the environment variable with the tasks CPU limit in MHz.
CpuLimit = "NOMAD_CPU_LIMIT"
// AllocID is the environment variable for passing the allocation ID.
AllocID = "NOMAD_ALLOC_ID"
// AllocName is the environment variable for passing the allocation name.
AllocName = "NOMAD_ALLOC_NAME"
// TaskName is the environment variable for passing the task name.
TaskName = "NOMAD_TASK_NAME"
// AllocIndex is the environment variable for passing the allocation index.
AllocIndex = "NOMAD_ALLOC_INDEX"
// AddrPrefix is the prefix for passing both dynamic and static port
// allocations to tasks.
// E.g$NOMAD_ADDR_http=127.0.0.1:80
AddrPrefix = "NOMAD_ADDR_"
// IpPrefix is the prefix for passing the IP of a port allocation to a task.
IpPrefix = "NOMAD_IP_"
// PortPrefix is the prefix for passing the port allocation to a task.
PortPrefix = "NOMAD_PORT_"
// HostPortPrefix is the prefix for passing the host port when a portmap is
// specified.
HostPortPrefix = "NOMAD_HOST_PORT_"
// MetaPrefix is the prefix for passing task meta data.
MetaPrefix = "NOMAD_META_"
)
// The node values that can be interpreted.
const (
nodeIdKey = "node.unique.id"
nodeDcKey = "node.datacenter"
nodeNameKey = "node.unique.name"
nodeClassKey = "node.class"
// Prefixes used for lookups.
nodeAttributePrefix = "attr."
nodeMetaPrefix = "meta."
)
// TaskEnvironment is used to expose information to a task via environment
// variables and provide interpolation of Nomad variables.
type TaskEnvironment struct {
Env map[string]string
TaskMeta map[string]string
TaskGroupMeta map[string]string
JobMeta map[string]string
AllocDir string
TaskDir string
CpuLimit int
MemLimit int
TaskName string
AllocIndex int
AllocId string
AllocName string
Node *structs.Node
Networks []*structs.NetworkResource
PortMap map[string]int
// taskEnv is the variables that will be set in the tasks environment
TaskEnv map[string]string
// nodeValues is the values that are allowed for interprolation from the
// node.
NodeValues map[string]string
}
func NewTaskEnvironment(node *structs.Node) *TaskEnvironment {
return &TaskEnvironment{Node: node, AllocIndex: -1}
}
// ParseAndReplace takes the user supplied args replaces any instance of an
// environment variable or nomad variable in the args with the actual value.
func (t *TaskEnvironment) ParseAndReplace(args []string) []string {
replaced := make([]string, len(args))
for i, arg := range args {
replaced[i] = hargs.ReplaceEnv(arg, t.TaskEnv, t.NodeValues)
}
return replaced
}
// ReplaceEnv takes an arg and replaces all occurrences of environment variables
// and nomad variables. If the variable is found in the passed map it is
// replaced, otherwise the original string is returned.
func (t *TaskEnvironment) ReplaceEnv(arg string) string {
return hargs.ReplaceEnv(arg, t.TaskEnv, t.NodeValues)
}
// Build must be called after all the tasks environment values have been set.
func (t *TaskEnvironment) Build() *TaskEnvironment {
t.NodeValues = make(map[string]string)
t.TaskEnv = make(map[string]string)
// Build the meta with the following precedence: task, task group, job.
for _, meta := range []map[string]string{t.JobMeta, t.TaskGroupMeta, t.TaskMeta} {
for k, v := range meta {
t.TaskEnv[fmt.Sprintf("%s%s", MetaPrefix, strings.ToUpper(k))] = v
}
}
// Build the ports
for _, network := range t.Networks {
for label, value := range network.MapLabelToValues(nil) {
t.TaskEnv[fmt.Sprintf("%s%s", IpPrefix, label)] = network.IP
t.TaskEnv[fmt.Sprintf("%s%s", HostPortPrefix, label)] = strconv.Itoa(value)
if forwardedPort, ok := t.PortMap[label]; ok {
value = forwardedPort
}
t.TaskEnv[fmt.Sprintf("%s%s", PortPrefix, label)] = fmt.Sprintf("%d", value)
IPPort := fmt.Sprintf("%s:%d", network.IP, value)
t.TaskEnv[fmt.Sprintf("%s%s", AddrPrefix, label)] = IPPort
}
}
// Build the directories
if t.AllocDir != "" {
t.TaskEnv[AllocDir] = t.AllocDir
}
if t.TaskDir != "" {
t.TaskEnv[TaskLocalDir] = t.TaskDir
}
// Build the resource limits
if t.MemLimit != 0 {
t.TaskEnv[MemLimit] = strconv.Itoa(t.MemLimit)
}
if t.CpuLimit != 0 {
t.TaskEnv[CpuLimit] = strconv.Itoa(t.CpuLimit)
}
// Build the tasks ids
if t.AllocId != "" {
t.TaskEnv[AllocID] = t.AllocId
}
if t.AllocName != "" {
t.TaskEnv[AllocName] = t.AllocName
}
if t.AllocIndex != -1 {
t.TaskEnv[AllocIndex] = strconv.Itoa(t.AllocIndex)
}
if t.TaskName != "" {
t.TaskEnv[TaskName] = t.TaskName
}
// Build the node
if t.Node != nil {
// Set up the node values.
t.NodeValues[nodeIdKey] = t.Node.ID
t.NodeValues[nodeDcKey] = t.Node.Datacenter
t.NodeValues[nodeNameKey] = t.Node.Name
t.NodeValues[nodeClassKey] = t.Node.NodeClass
// Set up the attributes.
for k, v := range t.Node.Attributes {
t.NodeValues[fmt.Sprintf("%s%s", nodeAttributePrefix, k)] = v
}
// Set up the meta.
for k, v := range t.Node.Meta {
t.NodeValues[fmt.Sprintf("%s%s", nodeMetaPrefix, k)] = v
}
}
// Interpret the environment variables
interpreted := make(map[string]string, len(t.Env))
for k, v := range t.Env {
interpreted[k] = hargs.ReplaceEnv(v, t.NodeValues, t.TaskEnv)
}
for k, v := range interpreted {
t.TaskEnv[k] = v
}
return t
}
// EnvList returns a list of strings with NAME=value pairs.
func (t *TaskEnvironment) EnvList() []string {
env := []string{}
for k, v := range t.TaskEnv {
env = append(env, fmt.Sprintf("%s=%s", k, v))
}
return env
}
// EnvMap returns a copy of the tasks environment variables.
func (t *TaskEnvironment) EnvMap() map[string]string {
m := make(map[string]string, len(t.TaskEnv))
for k, v := range t.TaskEnv {
m[k] = v
}
return m
}
// Builder methods to build the TaskEnvironment
func (t *TaskEnvironment) SetAllocDir(dir string) *TaskEnvironment {
t.AllocDir = dir
return t
}
func (t *TaskEnvironment) ClearAllocDir() *TaskEnvironment {
t.AllocDir = ""
return t
}
func (t *TaskEnvironment) SetTaskLocalDir(dir string) *TaskEnvironment {
t.TaskDir = dir
return t
}
func (t *TaskEnvironment) ClearTaskLocalDir() *TaskEnvironment {
t.TaskDir = ""
return t
}
func (t *TaskEnvironment) SetMemLimit(limit int) *TaskEnvironment {
t.MemLimit = limit
return t
}
func (t *TaskEnvironment) ClearMemLimit() *TaskEnvironment {
t.MemLimit = 0
return t
}
func (t *TaskEnvironment) SetCpuLimit(limit int) *TaskEnvironment {
t.CpuLimit = limit
return t
}
func (t *TaskEnvironment) ClearCpuLimit() *TaskEnvironment {
t.CpuLimit = 0
return t
}
func (t *TaskEnvironment) SetNetworks(networks []*structs.NetworkResource) *TaskEnvironment {
t.Networks = networks
return t
}
func (t *TaskEnvironment) clearNetworks() *TaskEnvironment {
t.Networks = nil
return t
}
func (t *TaskEnvironment) SetPortMap(portMap map[string]int) *TaskEnvironment {
t.PortMap = portMap
return t
}
func (t *TaskEnvironment) clearPortMap() *TaskEnvironment {
t.PortMap = nil
return t
}
// Takes a map of meta values to be passed to the task. The keys are capatilized
// when the environent variable is set.
func (t *TaskEnvironment) SetTaskMeta(m map[string]string) *TaskEnvironment {
t.TaskMeta = m
return t
}
func (t *TaskEnvironment) ClearTaskMeta() *TaskEnvironment {
t.TaskMeta = nil
return t
}
func (t *TaskEnvironment) SetTaskGroupMeta(m map[string]string) *TaskEnvironment {
t.TaskGroupMeta = m
return t
}
func (t *TaskEnvironment) ClearTaskGroupMeta() *TaskEnvironment {
t.TaskGroupMeta = nil
return t
}
func (t *TaskEnvironment) SetJobMeta(m map[string]string) *TaskEnvironment {
t.JobMeta = m
return t
}
func (t *TaskEnvironment) ClearJobMeta() *TaskEnvironment {
t.JobMeta = nil
return t
}
func (t *TaskEnvironment) SetEnvvars(m map[string]string) *TaskEnvironment {
t.Env = m
return t
}
// Appends the given environment variables.
func (t *TaskEnvironment) AppendEnvvars(m map[string]string) *TaskEnvironment {
if t.Env == nil {
t.Env = make(map[string]string, len(m))
}
for k, v := range m {
t.Env[k] = v
}
return t
}
// AppendHostEnvvars adds the host environment variables to the tasks. The
// filter parameter can be use to filter host environment from entering the
// tasks.
func (t *TaskEnvironment) AppendHostEnvvars(filter []string) *TaskEnvironment {
hostEnv := os.Environ()
if t.Env == nil {
t.Env = make(map[string]string, len(hostEnv))
}
// Index the filtered environment variables.
index := make(map[string]struct{}, len(filter))
for _, f := range filter {
index[f] = struct{}{}
}
for _, e := range hostEnv {
parts := strings.SplitN(e, "=", 2)
key, value := parts[0], parts[1]
// Skip filtered environment variables
if _, filtered := index[key]; filtered {
continue
}
// Don't override the tasks environment variables.
if _, existing := t.Env[key]; !existing {
t.Env[key] = value
}
}
return t
}
func (t *TaskEnvironment) ClearEnvvars() *TaskEnvironment {
t.Env = nil
return t
}
// Helper method for setting all fields from an allocation.
func (t *TaskEnvironment) SetAlloc(alloc *structs.Allocation) *TaskEnvironment {
t.AllocId = alloc.ID
t.AllocName = alloc.Name
t.AllocIndex = alloc.Index()
return t
}
// Helper method for clearing all fields from an allocation.
func (t *TaskEnvironment) ClearAlloc(alloc *structs.Allocation) *TaskEnvironment {
return t.ClearAllocId().ClearAllocName().ClearAllocIndex()
}
func (t *TaskEnvironment) SetAllocIndex(index int) *TaskEnvironment {
t.AllocIndex = index
return t
}
func (t *TaskEnvironment) ClearAllocIndex() *TaskEnvironment {
t.AllocIndex = -1
return t
}
func (t *TaskEnvironment) SetAllocId(id string) *TaskEnvironment {
t.AllocId = id
return t
}
func (t *TaskEnvironment) ClearAllocId() *TaskEnvironment {
t.AllocId = ""
return t
}
func (t *TaskEnvironment) SetAllocName(name string) *TaskEnvironment {
t.AllocName = name
return t
}
func (t *TaskEnvironment) ClearAllocName() *TaskEnvironment {
t.AllocName = ""
return t
}
func (t *TaskEnvironment) SetTaskName(name string) *TaskEnvironment {
t.TaskName = name
return t
}
func (t *TaskEnvironment) ClearTaskName() *TaskEnvironment {
t.TaskName = ""
return t
}

319
vendor/github.com/hashicorp/nomad/client/driver/exec.go generated vendored Normal file
View File

@@ -0,0 +1,319 @@
package driver
import (
"encoding/json"
"fmt"
"log"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/go-plugin"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/client/driver/executor"
dstructs "github.com/hashicorp/nomad/client/driver/structs"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/helper/discover"
"github.com/hashicorp/nomad/helper/fields"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/mitchellh/mapstructure"
)
const (
// The key populated in Node Attributes to indicate the presence of the Exec
// driver
execDriverAttr = "driver.exec"
)
// ExecDriver fork/execs tasks using as many of the underlying OS's isolation
// features.
type ExecDriver struct {
DriverContext
}
type ExecDriverConfig struct {
Command string `mapstructure:"command"`
Args []string `mapstructure:"args"`
}
// execHandle is returned from Start/Open as a handle to the PID
type execHandle struct {
pluginClient *plugin.Client
executor executor.Executor
isolationConfig *dstructs.IsolationConfig
userPid int
allocDir *allocdir.AllocDir
killTimeout time.Duration
maxKillTimeout time.Duration
logger *log.Logger
waitCh chan *dstructs.WaitResult
doneCh chan struct{}
version string
}
// NewExecDriver is used to create a new exec driver
func NewExecDriver(ctx *DriverContext) Driver {
return &ExecDriver{DriverContext: *ctx}
}
// Validate is used to validate the driver configuration
func (d *ExecDriver) Validate(config map[string]interface{}) error {
fd := &fields.FieldData{
Raw: config,
Schema: map[string]*fields.FieldSchema{
"command": &fields.FieldSchema{
Type: fields.TypeString,
Required: true,
},
"args": &fields.FieldSchema{
Type: fields.TypeArray,
},
},
}
if err := fd.Validate(); err != nil {
return err
}
return nil
}
func (d *ExecDriver) Periodic() (bool, time.Duration) {
return true, 15 * time.Second
}
func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
var driverConfig ExecDriverConfig
if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
return nil, err
}
// Get the command to be ran
command := driverConfig.Command
if err := validateCommand(command, "args"); err != nil {
return nil, err
}
// Set the host environment variables.
filter := strings.Split(d.config.ReadDefault("env.blacklist", config.DefaultEnvBlacklist), ",")
d.taskEnv.AppendHostEnvvars(filter)
// Get the task directory for storing the executor logs.
taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName]
if !ok {
return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName)
}
bin, err := discover.NomadExecutable()
if err != nil {
return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
}
pluginLogFile := filepath.Join(taskDir, fmt.Sprintf("%s-executor.out", task.Name))
pluginConfig := &plugin.ClientConfig{
Cmd: exec.Command(bin, "executor", pluginLogFile),
}
exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
if err != nil {
return nil, err
}
executorCtx := &executor.ExecutorContext{
TaskEnv: d.taskEnv,
Driver: "exec",
AllocDir: ctx.AllocDir,
AllocID: ctx.AllocID,
ChrootEnv: d.config.ChrootEnv,
Task: task,
}
ps, err := exec.LaunchCmd(&executor.ExecCommand{
Cmd: command,
Args: driverConfig.Args,
FSIsolation: true,
ResourceLimits: true,
User: getExecutorUser(task),
}, executorCtx)
if err != nil {
pluginClient.Kill()
return nil, err
}
d.logger.Printf("[DEBUG] driver.exec: started process via plugin with pid: %v", ps.Pid)
// Return a driver handle
maxKill := d.DriverContext.config.MaxKillTimeout
h := &execHandle{
pluginClient: pluginClient,
userPid: ps.Pid,
executor: exec,
allocDir: ctx.AllocDir,
isolationConfig: ps.IsolationConfig,
killTimeout: GetKillTimeout(task.KillTimeout, maxKill),
maxKillTimeout: maxKill,
logger: d.logger,
version: d.config.Version,
doneCh: make(chan struct{}),
waitCh: make(chan *dstructs.WaitResult, 1),
}
if err := exec.SyncServices(consulContext(d.config, "")); err != nil {
d.logger.Printf("[ERR] driver.exec: error registering services with consul for task: %q: %v", task.Name, err)
}
go h.run()
return h, nil
}
type execId struct {
Version string
KillTimeout time.Duration
MaxKillTimeout time.Duration
UserPid int
TaskDir string
AllocDir *allocdir.AllocDir
IsolationConfig *dstructs.IsolationConfig
PluginConfig *PluginReattachConfig
}
func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
id := &execId{}
if err := json.Unmarshal([]byte(handleID), id); err != nil {
return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
}
pluginConfig := &plugin.ClientConfig{
Reattach: id.PluginConfig.PluginConfig(),
}
exec, client, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
if err != nil {
merrs := new(multierror.Error)
merrs.Errors = append(merrs.Errors, err)
d.logger.Println("[ERR] driver.exec: error connecting to plugin so destroying plugin pid and user pid")
if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
merrs.Errors = append(merrs.Errors, fmt.Errorf("error destroying plugin and userpid: %v", e))
}
if id.IsolationConfig != nil {
ePid := pluginConfig.Reattach.Pid
if e := executor.ClientCleanup(id.IsolationConfig, ePid); e != nil {
merrs.Errors = append(merrs.Errors, fmt.Errorf("destroying cgroup failed: %v", e))
}
}
if e := ctx.AllocDir.UnmountAll(); e != nil {
merrs.Errors = append(merrs.Errors, e)
}
return nil, fmt.Errorf("error connecting to plugin: %v", merrs.ErrorOrNil())
}
ver, _ := exec.Version()
d.logger.Printf("[DEBUG] driver.exec : version of executor: %v", ver.Version)
// Return a driver handle
h := &execHandle{
pluginClient: client,
executor: exec,
userPid: id.UserPid,
allocDir: id.AllocDir,
isolationConfig: id.IsolationConfig,
logger: d.logger,
version: id.Version,
killTimeout: id.KillTimeout,
maxKillTimeout: id.MaxKillTimeout,
doneCh: make(chan struct{}),
waitCh: make(chan *dstructs.WaitResult, 1),
}
if err := exec.SyncServices(consulContext(d.config, "")); err != nil {
d.logger.Printf("[ERR] driver.exec: error registering services with consul: %v", err)
}
go h.run()
return h, nil
}
func (h *execHandle) ID() string {
id := execId{
Version: h.version,
KillTimeout: h.killTimeout,
MaxKillTimeout: h.maxKillTimeout,
PluginConfig: NewPluginReattachConfig(h.pluginClient.ReattachConfig()),
UserPid: h.userPid,
AllocDir: h.allocDir,
IsolationConfig: h.isolationConfig,
}
data, err := json.Marshal(id)
if err != nil {
h.logger.Printf("[ERR] driver.exec: failed to marshal ID to JSON: %s", err)
}
return string(data)
}
func (h *execHandle) WaitCh() chan *dstructs.WaitResult {
return h.waitCh
}
func (h *execHandle) Update(task *structs.Task) error {
// Store the updated kill timeout.
h.killTimeout = GetKillTimeout(task.KillTimeout, h.maxKillTimeout)
h.executor.UpdateTask(task)
// Update is not possible
return nil
}
func (h *execHandle) Kill() error {
if err := h.executor.ShutDown(); err != nil {
if h.pluginClient.Exited() {
return nil
}
return fmt.Errorf("executor Shutdown failed: %v", err)
}
select {
case <-h.doneCh:
return nil
case <-time.After(h.killTimeout):
if h.pluginClient.Exited() {
return nil
}
if err := h.executor.Exit(); err != nil {
return fmt.Errorf("executor Exit failed: %v", err)
}
return nil
}
}
func (h *execHandle) Stats() (*cstructs.TaskResourceUsage, error) {
return h.executor.Stats()
}
func (h *execHandle) run() {
ps, err := h.executor.Wait()
close(h.doneCh)
// If the exitcode is 0 and we had an error that means the plugin didn't
// connect and doesn't know the state of the user process so we are killing
// the user process so that when we create a new executor on restarting the
// new user process doesn't have collisions with resources that the older
// user pid might be holding onto.
if ps.ExitCode == 0 && err != nil {
if h.isolationConfig != nil {
ePid := h.pluginClient.ReattachConfig().Pid
if e := executor.ClientCleanup(h.isolationConfig, ePid); e != nil {
h.logger.Printf("[ERR] driver.exec: destroying resource container failed: %v", e)
}
}
if e := h.allocDir.UnmountAll(); e != nil {
h.logger.Printf("[ERR] driver.exec: unmounting dev,proc and alloc dirs failed: %v", e)
}
}
h.waitCh <- dstructs.NewWaitResult(ps.ExitCode, ps.Signal, err)
close(h.waitCh)
// Remove services
if err := h.executor.DeregisterServices(); err != nil {
h.logger.Printf("[ERR] driver.exec: failed to deregister services: %v", err)
}
if err := h.executor.Exit(); err != nil {
h.logger.Printf("[ERR] driver.exec: error destroying executor: %v", err)
}
h.pluginClient.Kill()
}

View File

@@ -0,0 +1,12 @@
//+build darwin dragonfly freebsd netbsd openbsd solaris windows
package driver
import (
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/nomad/structs"
)
func (d *ExecDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
return false, nil
}

View File

@@ -0,0 +1,34 @@
package driver
import (
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/nomad/structs"
"golang.org/x/sys/unix"
)
func (d *ExecDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
// Get the current status so that we can log any debug messages only if the
// state changes
_, currentlyEnabled := node.Attributes[execDriverAttr]
// Only enable if cgroups are available and we are root
if _, ok := node.Attributes["unique.cgroup.mountpoint"]; !ok {
if currentlyEnabled {
d.logger.Printf("[DEBUG] driver.exec: cgroups unavailable, disabling")
}
delete(node.Attributes, execDriverAttr)
return false, nil
} else if unix.Geteuid() != 0 {
if currentlyEnabled {
d.logger.Printf("[DEBUG] driver.exec: must run as root user, disabling")
}
delete(node.Attributes, execDriverAttr)
return false, nil
}
if !currentlyEnabled {
d.logger.Printf("[DEBUG] driver.exec: exec driver is enabled")
}
node.Attributes[execDriverAttr] = "1"
return true, nil
}

View File

@@ -0,0 +1,206 @@
package executor
import (
"fmt"
"log"
"os/exec"
"sync"
"syscall"
"time"
"github.com/armon/circbuf"
docker "github.com/fsouza/go-dockerclient"
cstructs "github.com/hashicorp/nomad/client/driver/structs"
)
var (
// We store the client globally to cache the connection to the docker daemon.
createClient sync.Once
client *docker.Client
)
const (
// The default check timeout
defaultCheckTimeout = 30 * time.Second
)
// DockerScriptCheck runs nagios compatible scripts in a docker container and
// provides the check result
type DockerScriptCheck struct {
id string // id of the check
interval time.Duration // interval of the check
timeout time.Duration // timeout of the check
containerID string // container id in which the check will be invoked
logger *log.Logger
cmd string // check command
args []string // check command arguments
dockerEndpoint string // docker endpoint
tlsCert string // path to tls certificate
tlsCa string // path to tls ca
tlsKey string // path to tls key
}
// dockerClient creates the client to interact with the docker daemon
func (d *DockerScriptCheck) dockerClient() (*docker.Client, error) {
if client != nil {
return client, nil
}
var err error
createClient.Do(func() {
if d.dockerEndpoint != "" {
if d.tlsCert+d.tlsKey+d.tlsCa != "" {
d.logger.Printf("[DEBUG] executor.checks: using TLS client connection to %s", d.dockerEndpoint)
client, err = docker.NewTLSClient(d.dockerEndpoint, d.tlsCert, d.tlsKey, d.tlsCa)
} else {
d.logger.Printf("[DEBUG] executor.checks: using standard client connection to %s", d.dockerEndpoint)
client, err = docker.NewClient(d.dockerEndpoint)
}
return
}
d.logger.Println("[DEBUG] executor.checks: using client connection initialized from environment")
client, err = docker.NewClientFromEnv()
})
return client, err
}
// Run runs a script check inside a docker container
func (d *DockerScriptCheck) Run() *cstructs.CheckResult {
var (
exec *docker.Exec
err error
execRes *docker.ExecInspect
time = time.Now()
)
if client, err = d.dockerClient(); err != nil {
return &cstructs.CheckResult{Err: err}
}
client = client
execOpts := docker.CreateExecOptions{
AttachStdin: false,
AttachStdout: true,
AttachStderr: true,
Tty: false,
Cmd: append([]string{d.cmd}, d.args...),
Container: d.containerID,
}
if exec, err = client.CreateExec(execOpts); err != nil {
return &cstructs.CheckResult{Err: err}
}
output, _ := circbuf.NewBuffer(int64(cstructs.CheckBufSize))
startOpts := docker.StartExecOptions{
Detach: false,
Tty: false,
OutputStream: output,
ErrorStream: output,
}
if err = client.StartExec(exec.ID, startOpts); err != nil {
return &cstructs.CheckResult{Err: err}
}
if execRes, err = client.InspectExec(exec.ID); err != nil {
return &cstructs.CheckResult{Err: err}
}
return &cstructs.CheckResult{
ExitCode: execRes.ExitCode,
Output: string(output.Bytes()),
Timestamp: time,
}
}
// ID returns the check id
func (d *DockerScriptCheck) ID() string {
return d.id
}
// Interval returns the interval at which the check has to run
func (d *DockerScriptCheck) Interval() time.Duration {
return d.interval
}
// Timeout returns the duration after which a check is timed out.
func (d *DockerScriptCheck) Timeout() time.Duration {
if d.timeout == 0 {
return defaultCheckTimeout
}
return d.timeout
}
// ExecScriptCheck runs a nagios compatible script and returns the check result
type ExecScriptCheck struct {
id string // id of the script check
interval time.Duration // interval at which the check is invoked
timeout time.Duration // timeout duration of the check
cmd string // command of the check
args []string // args passed to the check
taskDir string // the root directory of the check
FSIsolation bool // indicates whether the check has to be run within a chroot
}
// Run runs an exec script check
func (e *ExecScriptCheck) Run() *cstructs.CheckResult {
buf, _ := circbuf.NewBuffer(int64(cstructs.CheckBufSize))
cmd := exec.Command(e.cmd, e.args...)
cmd.Stdout = buf
cmd.Stderr = buf
e.setChroot(cmd)
ts := time.Now()
if err := cmd.Start(); err != nil {
return &cstructs.CheckResult{Err: err}
}
errCh := make(chan error, 2)
go func() {
errCh <- cmd.Wait()
}()
for {
select {
case err := <-errCh:
endTime := time.Now()
if err == nil {
return &cstructs.CheckResult{
ExitCode: 0,
Output: string(buf.Bytes()),
Timestamp: ts,
}
}
exitCode := 1
if exitErr, ok := err.(*exec.ExitError); ok {
if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
exitCode = status.ExitStatus()
}
}
return &cstructs.CheckResult{
ExitCode: exitCode,
Output: string(buf.Bytes()),
Timestamp: ts,
Duration: endTime.Sub(ts),
}
case <-time.After(e.Timeout()):
errCh <- fmt.Errorf("timed out after waiting 30s")
}
}
return nil
}
// ID returns the check id
func (e *ExecScriptCheck) ID() string {
return e.id
}
// Interval returns the interval at which the check has to run
func (e *ExecScriptCheck) Interval() time.Duration {
return e.interval
}
// Timeout returns the duration after which a check is timed out.
func (e *ExecScriptCheck) Timeout() time.Duration {
if e.timeout == 0 {
return defaultCheckTimeout
}
return e.timeout
}

View File

@@ -0,0 +1,18 @@
// +build darwin dragonfly freebsd linux netbsd openbsd solaris
package executor
import (
"os/exec"
"syscall"
)
func (e *ExecScriptCheck) setChroot(cmd *exec.Cmd) {
if e.FSIsolation {
if cmd.SysProcAttr == nil {
cmd.SysProcAttr = &syscall.SysProcAttr{}
}
cmd.SysProcAttr.Chroot = e.taskDir
}
cmd.Dir = "/"
}

View File

@@ -0,0 +1,8 @@
// +build windows
package executor
import "os/exec"
func (e *ExecScriptCheck) setChroot(cmd *exec.Cmd) {
}

View File

@@ -0,0 +1,856 @@
package executor
import (
"fmt"
"io/ioutil"
"log"
"net"
"os"
"os/exec"
"path/filepath"
"runtime"
"strconv"
"strings"
"sync"
"syscall"
"time"
"github.com/hashicorp/go-multierror"
"github.com/mitchellh/go-ps"
"github.com/shirou/gopsutil/process"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/driver/env"
"github.com/hashicorp/nomad/client/driver/logging"
"github.com/hashicorp/nomad/client/stats"
"github.com/hashicorp/nomad/command/agent/consul"
shelpers "github.com/hashicorp/nomad/helper/stats"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/nomad/structs/config"
dstructs "github.com/hashicorp/nomad/client/driver/structs"
cstructs "github.com/hashicorp/nomad/client/structs"
)
const (
// pidScanInterval is the interval at which the executor scans the process
// tree for finding out the pids that the executor and it's child processes
// have forked
pidScanInterval = 5 * time.Second
)
var (
// The statistics the basic executor exposes
ExecutorBasicMeasuredMemStats = []string{"RSS", "Swap"}
ExecutorBasicMeasuredCpuStats = []string{"System Mode", "User Mode", "Percent"}
)
// Executor is the interface which allows a driver to launch and supervise
// a process
type Executor interface {
LaunchCmd(command *ExecCommand, ctx *ExecutorContext) (*ProcessState, error)
LaunchSyslogServer(ctx *ExecutorContext) (*SyslogServerState, error)
Wait() (*ProcessState, error)
ShutDown() error
Exit() error
UpdateLogConfig(logConfig *structs.LogConfig) error
UpdateTask(task *structs.Task) error
SyncServices(ctx *ConsulContext) error
DeregisterServices() error
Version() (*ExecutorVersion, error)
Stats() (*cstructs.TaskResourceUsage, error)
}
// ConsulContext holds context to configure the Consul client and run checks
type ConsulContext struct {
// ConsulConfig contains the configuration information for talking
// with this Nomad Agent's Consul Agent.
ConsulConfig *config.ConsulConfig
// ContainerID is the ID of the container
ContainerID string
// TLSCert is the cert which docker client uses while interactng with the docker
// daemon over TLS
TLSCert string
// TLSCa is the CA which the docker client uses while interacting with the docker
// daeemon over TLS
TLSCa string
// TLSKey is the TLS key which the docker client uses while interacting with
// the docker daemon
TLSKey string
// DockerEndpoint is the endpoint of the docker daemon
DockerEndpoint string
}
// ExecutorContext holds context to configure the command user
// wants to run and isolate it
type ExecutorContext struct {
// TaskEnv holds information about the environment of a Task
TaskEnv *env.TaskEnvironment
// AllocDir is the handle to do operations on the alloc dir of
// the task
AllocDir *allocdir.AllocDir
// Task is the task whose executor is being launched
Task *structs.Task
// AllocID is the allocation id to which the task belongs
AllocID string
// A mapping of directories on the host OS to attempt to embed inside each
// task's chroot.
ChrootEnv map[string]string
// Driver is the name of the driver that invoked the executor
Driver string
// PortUpperBound is the upper bound of the ports that we can use to start
// the syslog server
PortUpperBound uint
// PortLowerBound is the lower bound of the ports that we can use to start
// the syslog server
PortLowerBound uint
}
// ExecCommand holds the user command, args, and other isolation related
// settings.
type ExecCommand struct {
// Cmd is the command that the user wants to run.
Cmd string
// Args is the args of the command that the user wants to run.
Args []string
// FSIsolation determines whether the command would be run in a chroot.
FSIsolation bool
// User is the user which the executor uses to run the command.
User string
// ResourceLimits determines whether resource limits are enforced by the
// executor.
ResourceLimits bool
}
// ProcessState holds information about the state of a user process.
type ProcessState struct {
Pid int
ExitCode int
Signal int
IsolationConfig *dstructs.IsolationConfig
Time time.Time
}
// nomadPid holds a pid and it's cpu percentage calculator
type nomadPid struct {
pid int
cpuStatsTotal *stats.CpuStats
cpuStatsUser *stats.CpuStats
cpuStatsSys *stats.CpuStats
}
// SyslogServerState holds the address and islation information of a launched
// syslog server
type SyslogServerState struct {
IsolationConfig *dstructs.IsolationConfig
Addr string
}
// ExecutorVersion is the version of the executor
type ExecutorVersion struct {
Version string
}
func (v *ExecutorVersion) GoString() string {
return v.Version
}
// UniversalExecutor is an implementation of the Executor which launches and
// supervises processes. In addition to process supervision it provides resource
// and file system isolation
type UniversalExecutor struct {
cmd exec.Cmd
ctx *ExecutorContext
command *ExecCommand
pids map[int]*nomadPid
pidLock sync.RWMutex
taskDir string
exitState *ProcessState
processExited chan interface{}
fsIsolationEnforced bool
lre *logging.FileRotator
lro *logging.FileRotator
rotatorLock sync.Mutex
shutdownCh chan struct{}
syslogServer *logging.SyslogServer
syslogChan chan *logging.SyslogMessage
resConCtx resourceContainerContext
consulSyncer *consul.Syncer
consulCtx *ConsulContext
totalCpuStats *stats.CpuStats
userCpuStats *stats.CpuStats
systemCpuStats *stats.CpuStats
logger *log.Logger
}
// NewExecutor returns an Executor
func NewExecutor(logger *log.Logger) Executor {
if err := shelpers.Init(); err != nil {
logger.Printf("[FATAL] executor: unable to initialize stats: %v", err)
return nil
}
exec := &UniversalExecutor{
logger: logger,
processExited: make(chan interface{}),
totalCpuStats: stats.NewCpuStats(),
userCpuStats: stats.NewCpuStats(),
systemCpuStats: stats.NewCpuStats(),
pids: make(map[int]*nomadPid),
}
return exec
}
// Version returns the api version of the executor
func (e *UniversalExecutor) Version() (*ExecutorVersion, error) {
return &ExecutorVersion{Version: "1.0.0"}, nil
}
// LaunchCmd launches a process and returns it's state. It also configures an
// applies isolation on certain platforms.
func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext) (*ProcessState, error) {
e.logger.Printf("[DEBUG] executor: launching command %v %v", command.Cmd, strings.Join(command.Args, " "))
e.ctx = ctx
e.command = command
// setting the user of the process
if command.User != "" {
e.logger.Printf("[DEBUG] executor: running command as %s", command.User)
if err := e.runAs(command.User); err != nil {
return nil, err
}
}
// configuring the task dir
if err := e.configureTaskDir(); err != nil {
return nil, err
}
e.ctx.TaskEnv.Build()
// configuring the chroot, resource container, and start the plugin
// process in the chroot.
if err := e.configureIsolation(); err != nil {
return nil, err
}
// Apply ourselves into the resource container. The executor MUST be in
// the resource container before the user task is started, otherwise we
// are subject to a fork attack in which a process escapes isolation by
// immediately forking.
if err := e.applyLimits(os.Getpid()); err != nil {
return nil, err
}
// Setup the loggers
if err := e.configureLoggers(); err != nil {
return nil, err
}
e.cmd.Stdout = e.lro
e.cmd.Stderr = e.lre
// Look up the binary path and make it executable
absPath, err := e.lookupBin(ctx.TaskEnv.ReplaceEnv(command.Cmd))
if err != nil {
return nil, err
}
if err := e.makeExecutable(absPath); err != nil {
return nil, err
}
path := absPath
// Determine the path to run as it may have to be relative to the chroot.
if e.fsIsolationEnforced {
rel, err := filepath.Rel(e.taskDir, path)
if err != nil {
return nil, err
}
path = rel
}
// Set the commands arguments
e.cmd.Path = path
e.cmd.Args = append([]string{e.cmd.Path}, ctx.TaskEnv.ParseAndReplace(command.Args)...)
e.cmd.Env = ctx.TaskEnv.EnvList()
// Start the process
if err := e.cmd.Start(); err != nil {
return nil, err
}
go e.collectPids()
go e.wait()
ic := e.resConCtx.getIsolationConfig()
return &ProcessState{Pid: e.cmd.Process.Pid, ExitCode: -1, IsolationConfig: ic, Time: time.Now()}, nil
}
// configureLoggers sets up the standard out/error file rotators
func (e *UniversalExecutor) configureLoggers() error {
e.rotatorLock.Lock()
defer e.rotatorLock.Unlock()
logFileSize := int64(e.ctx.Task.LogConfig.MaxFileSizeMB * 1024 * 1024)
if e.lro == nil {
lro, err := logging.NewFileRotator(e.ctx.AllocDir.LogDir(), fmt.Sprintf("%v.stdout", e.ctx.Task.Name),
e.ctx.Task.LogConfig.MaxFiles, logFileSize, e.logger)
if err != nil {
return err
}
e.lro = lro
}
if e.lre == nil {
lre, err := logging.NewFileRotator(e.ctx.AllocDir.LogDir(), fmt.Sprintf("%v.stderr", e.ctx.Task.Name),
e.ctx.Task.LogConfig.MaxFiles, logFileSize, e.logger)
if err != nil {
return err
}
e.lre = lre
}
return nil
}
// Wait waits until a process has exited and returns it's exitcode and errors
func (e *UniversalExecutor) Wait() (*ProcessState, error) {
<-e.processExited
return e.exitState, nil
}
// COMPAT: prior to Nomad 0.3.2, UpdateTask didn't exist.
// UpdateLogConfig updates the log configuration
func (e *UniversalExecutor) UpdateLogConfig(logConfig *structs.LogConfig) error {
e.ctx.Task.LogConfig = logConfig
if e.lro == nil {
return fmt.Errorf("log rotator for stdout doesn't exist")
}
e.lro.MaxFiles = logConfig.MaxFiles
e.lro.FileSize = int64(logConfig.MaxFileSizeMB * 1024 * 1024)
if e.lre == nil {
return fmt.Errorf("log rotator for stderr doesn't exist")
}
e.lre.MaxFiles = logConfig.MaxFiles
e.lre.FileSize = int64(logConfig.MaxFileSizeMB * 1024 * 1024)
return nil
}
func (e *UniversalExecutor) UpdateTask(task *structs.Task) error {
e.ctx.Task = task
// Updating Log Config
fileSize := int64(task.LogConfig.MaxFileSizeMB * 1024 * 1024)
e.lro.MaxFiles = task.LogConfig.MaxFiles
e.lro.FileSize = fileSize
e.lre.MaxFiles = task.LogConfig.MaxFiles
e.lre.FileSize = fileSize
// Re-syncing task with Consul agent
if e.consulSyncer != nil {
e.interpolateServices(e.ctx.Task)
domain := consul.NewExecutorDomain(e.ctx.AllocID, task.Name)
serviceMap := generateServiceKeys(e.ctx.AllocID, task.Services)
e.consulSyncer.SetServices(domain, serviceMap)
}
return nil
}
// generateServiceKeys takes a list of interpolated Nomad Services and returns a map
// of ServiceKeys to Nomad Services.
func generateServiceKeys(allocID string, services []*structs.Service) map[consul.ServiceKey]*structs.Service {
keys := make(map[consul.ServiceKey]*structs.Service, len(services))
for _, service := range services {
key := consul.GenerateServiceKey(service)
keys[key] = service
}
return keys
}
func (e *UniversalExecutor) wait() {
defer close(e.processExited)
err := e.cmd.Wait()
ic := e.resConCtx.getIsolationConfig()
if err == nil {
e.exitState = &ProcessState{Pid: 0, ExitCode: 0, IsolationConfig: ic, Time: time.Now()}
return
}
exitCode := 1
var signal int
if exitErr, ok := err.(*exec.ExitError); ok {
if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
exitCode = status.ExitStatus()
if status.Signaled() {
// bash(1) uses the lower 7 bits of a uint8
// to indicate normal program failure (see
// <sysexits.h>). If a process terminates due
// to a signal, encode the signal number to
// indicate which signal caused the process
// to terminate. Mirror this exit code
// encoding scheme.
const exitSignalBase = 128
signal = int(status.Signal())
exitCode = exitSignalBase + signal
}
}
} else {
e.logger.Printf("[DEBUG] executor: unexpected Wait() error type: %v", err)
}
e.exitState = &ProcessState{Pid: 0, ExitCode: exitCode, Signal: signal, IsolationConfig: ic, Time: time.Now()}
}
var (
// finishedErr is the error message received when trying to kill and already
// exited process.
finishedErr = "os: process already finished"
)
// ClientCleanup is the cleanup routine that a Nomad Client uses to remove the
// reminants of a child UniversalExecutor.
func ClientCleanup(ic *dstructs.IsolationConfig, pid int) error {
return clientCleanup(ic, pid)
}
// Exit cleans up the alloc directory, destroys resource container and kills the
// user process
func (e *UniversalExecutor) Exit() error {
var merr multierror.Error
if e.syslogServer != nil {
e.syslogServer.Shutdown()
}
e.lre.Close()
e.lro.Close()
if e.consulSyncer != nil {
e.consulSyncer.Shutdown()
}
// If the executor did not launch a process, return.
if e.command == nil {
return nil
}
// Prefer killing the process via the resource container.
if e.cmd.Process != nil && !e.command.ResourceLimits {
proc, err := os.FindProcess(e.cmd.Process.Pid)
if err != nil {
e.logger.Printf("[ERR] executor: can't find process with pid: %v, err: %v",
e.cmd.Process.Pid, err)
} else if err := proc.Kill(); err != nil && err.Error() != finishedErr {
merr.Errors = append(merr.Errors,
fmt.Errorf("can't kill process with pid: %v, err: %v", e.cmd.Process.Pid, err))
}
}
if e.command.ResourceLimits {
if err := e.resConCtx.executorCleanup(); err != nil {
merr.Errors = append(merr.Errors, err)
}
}
if e.command.FSIsolation {
if err := e.removeChrootMounts(); err != nil {
merr.Errors = append(merr.Errors, err)
}
}
return merr.ErrorOrNil()
}
// Shutdown sends an interrupt signal to the user process
func (e *UniversalExecutor) ShutDown() error {
if e.cmd.Process == nil {
return fmt.Errorf("executor.shutdown error: no process found")
}
proc, err := os.FindProcess(e.cmd.Process.Pid)
if err != nil {
return fmt.Errorf("executor.shutdown failed to find process: %v", err)
}
if runtime.GOOS == "windows" {
if err := proc.Kill(); err != nil && err.Error() != finishedErr {
return err
}
return nil
}
if err = proc.Signal(os.Interrupt); err != nil && err.Error() != finishedErr {
return fmt.Errorf("executor.shutdown error: %v", err)
}
return nil
}
// SyncServices syncs the services of the task that the executor is running with
// Consul
func (e *UniversalExecutor) SyncServices(ctx *ConsulContext) error {
e.logger.Printf("[INFO] executor: registering services")
e.consulCtx = ctx
if e.consulSyncer == nil {
cs, err := consul.NewSyncer(ctx.ConsulConfig, e.shutdownCh, e.logger)
if err != nil {
return err
}
e.consulSyncer = cs
go e.consulSyncer.Run()
}
e.interpolateServices(e.ctx.Task)
e.consulSyncer.SetDelegatedChecks(e.createCheckMap(), e.createCheck)
e.consulSyncer.SetAddrFinder(e.ctx.Task.FindHostAndPortFor)
domain := consul.NewExecutorDomain(e.ctx.AllocID, e.ctx.Task.Name)
serviceMap := generateServiceKeys(e.ctx.AllocID, e.ctx.Task.Services)
e.consulSyncer.SetServices(domain, serviceMap)
return nil
}
// DeregisterServices removes the services of the task that the executor is
// running from Consul
func (e *UniversalExecutor) DeregisterServices() error {
e.logger.Printf("[INFO] executor: de-registering services and shutting down consul service")
if e.consulSyncer != nil {
return e.consulSyncer.Shutdown()
}
return nil
}
// pidStats returns the resource usage stats per pid
func (e *UniversalExecutor) pidStats() (map[string]*cstructs.ResourceUsage, error) {
stats := make(map[string]*cstructs.ResourceUsage)
e.pidLock.RLock()
pids := make(map[int]*nomadPid, len(e.pids))
for k, v := range e.pids {
pids[k] = v
}
e.pidLock.RUnlock()
for pid, np := range pids {
p, err := process.NewProcess(int32(pid))
if err != nil {
e.logger.Printf("[DEBUG] executor: unable to create new process with pid: %v", pid)
continue
}
ms := &cstructs.MemoryStats{}
if memInfo, err := p.MemoryInfo(); err == nil {
ms.RSS = memInfo.RSS
ms.Swap = memInfo.Swap
ms.Measured = ExecutorBasicMeasuredMemStats
}
cs := &cstructs.CpuStats{}
if cpuStats, err := p.Times(); err == nil {
cs.SystemMode = np.cpuStatsSys.Percent(cpuStats.System * float64(time.Second))
cs.UserMode = np.cpuStatsUser.Percent(cpuStats.User * float64(time.Second))
cs.Measured = ExecutorBasicMeasuredCpuStats
// calculate cpu usage percent
cs.Percent = np.cpuStatsTotal.Percent(cpuStats.Total() * float64(time.Second))
}
stats[strconv.Itoa(pid)] = &cstructs.ResourceUsage{MemoryStats: ms, CpuStats: cs}
}
return stats, nil
}
// configureTaskDir sets the task dir in the executor
func (e *UniversalExecutor) configureTaskDir() error {
taskDir, ok := e.ctx.AllocDir.TaskDirs[e.ctx.Task.Name]
e.taskDir = taskDir
if !ok {
return fmt.Errorf("couldn't find task directory for task %v", e.ctx.Task.Name)
}
e.cmd.Dir = taskDir
return nil
}
// lookupBin looks for path to the binary to run by looking for the binary in
// the following locations, in-order: task/local/, task/, based on host $PATH.
// The return path is absolute.
func (e *UniversalExecutor) lookupBin(bin string) (string, error) {
// Check in the local directory
local := filepath.Join(e.taskDir, allocdir.TaskLocal, bin)
if _, err := os.Stat(local); err == nil {
return local, nil
}
// Check at the root of the task's directory
root := filepath.Join(e.taskDir, bin)
if _, err := os.Stat(root); err == nil {
return root, nil
}
// Check the $PATH
if host, err := exec.LookPath(bin); err == nil {
return host, nil
}
return "", fmt.Errorf("binary %q could not be found", bin)
}
// makeExecutable makes the given file executable for root,group,others.
func (e *UniversalExecutor) makeExecutable(binPath string) error {
if runtime.GOOS == "windows" {
return nil
}
fi, err := os.Stat(binPath)
if err != nil {
if os.IsNotExist(err) {
return fmt.Errorf("binary %q does not exist", binPath)
}
return fmt.Errorf("specified binary is invalid: %v", err)
}
// If it is not executable, make it so.
perm := fi.Mode().Perm()
req := os.FileMode(0555)
if perm&req != req {
if err := os.Chmod(binPath, perm|req); err != nil {
return fmt.Errorf("error making %q executable: %s", binPath, err)
}
}
return nil
}
// getFreePort returns a free port ready to be listened on between upper and
// lower bounds
func (e *UniversalExecutor) getListener(lowerBound uint, upperBound uint) (net.Listener, error) {
if runtime.GOOS == "windows" {
return e.listenerTCP(lowerBound, upperBound)
}
return e.listenerUnix()
}
// listenerTCP creates a TCP listener using an unused port between an upper and
// lower bound
func (e *UniversalExecutor) listenerTCP(lowerBound uint, upperBound uint) (net.Listener, error) {
for i := lowerBound; i <= upperBound; i++ {
addr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("localhost:%v", i))
if err != nil {
return nil, err
}
l, err := net.ListenTCP("tcp", addr)
if err != nil {
continue
}
return l, nil
}
return nil, fmt.Errorf("No free port found")
}
// listenerUnix creates a Unix domain socket
func (e *UniversalExecutor) listenerUnix() (net.Listener, error) {
f, err := ioutil.TempFile("", "plugin")
if err != nil {
return nil, err
}
path := f.Name()
if err := f.Close(); err != nil {
return nil, err
}
if err := os.Remove(path); err != nil {
return nil, err
}
return net.Listen("unix", path)
}
// createCheckMap creates a map of checks that the executor will handle on it's
// own
func (e *UniversalExecutor) createCheckMap() map[string]struct{} {
checks := map[string]struct{}{
"script": struct{}{},
}
return checks
}
// createCheck creates NomadCheck from a ServiceCheck
func (e *UniversalExecutor) createCheck(check *structs.ServiceCheck, checkID string) (consul.Check, error) {
if check.Type == structs.ServiceCheckScript && e.ctx.Driver == "docker" {
return &DockerScriptCheck{
id: checkID,
interval: check.Interval,
timeout: check.Timeout,
containerID: e.consulCtx.ContainerID,
logger: e.logger,
cmd: check.Command,
args: check.Args,
}, nil
}
if check.Type == structs.ServiceCheckScript && (e.ctx.Driver == "exec" ||
e.ctx.Driver == "raw_exec" || e.ctx.Driver == "java") {
return &ExecScriptCheck{
id: checkID,
interval: check.Interval,
timeout: check.Timeout,
cmd: check.Command,
args: check.Args,
taskDir: e.taskDir,
FSIsolation: e.command.FSIsolation,
}, nil
}
return nil, fmt.Errorf("couldn't create check for %v", check.Name)
}
// interpolateServices interpolates tags in a service and checks with values from the
// task's environment.
func (e *UniversalExecutor) interpolateServices(task *structs.Task) {
e.ctx.TaskEnv.Build()
for _, service := range task.Services {
for _, check := range service.Checks {
if check.Type == structs.ServiceCheckScript {
check.Name = e.ctx.TaskEnv.ReplaceEnv(check.Name)
check.Command = e.ctx.TaskEnv.ReplaceEnv(check.Command)
check.Args = e.ctx.TaskEnv.ParseAndReplace(check.Args)
check.Path = e.ctx.TaskEnv.ReplaceEnv(check.Path)
check.Protocol = e.ctx.TaskEnv.ReplaceEnv(check.Protocol)
}
}
service.Name = e.ctx.TaskEnv.ReplaceEnv(service.Name)
service.Tags = e.ctx.TaskEnv.ParseAndReplace(service.Tags)
}
}
// collectPids collects the pids of the child processes that the executor is
// running every 5 seconds
func (e *UniversalExecutor) collectPids() {
// Fire the timer right away when the executor starts from there on the pids
// are collected every scan interval
timer := time.NewTimer(0)
defer timer.Stop()
for {
select {
case <-timer.C:
pids, err := e.getAllPids()
if err != nil {
e.logger.Printf("[DEBUG] executor: error collecting pids: %v", err)
}
e.pidLock.Lock()
// Adding pids which are not being tracked
for pid, np := range pids {
if _, ok := e.pids[pid]; !ok {
e.pids[pid] = np
}
}
// Removing pids which are no longer present
for pid := range e.pids {
if _, ok := pids[pid]; !ok {
delete(e.pids, pid)
}
}
e.pidLock.Unlock()
timer.Reset(pidScanInterval)
case <-e.processExited:
return
}
}
}
// scanPids scans all the pids on the machine running the current executor and
// returns the child processes of the executor.
func (e *UniversalExecutor) scanPids(parentPid int, allPids []ps.Process) (map[int]*nomadPid, error) {
processFamily := make(map[int]struct{})
processFamily[parentPid] = struct{}{}
// A buffer for holding pids which haven't matched with any parent pid
var pidsRemaining []ps.Process
for {
// flag to indicate if we have found a match
foundNewPid := false
for _, pid := range allPids {
_, childPid := processFamily[pid.PPid()]
// checking if the pid is a child of any of the parents
if childPid {
processFamily[pid.Pid()] = struct{}{}
foundNewPid = true
} else {
// if it is not, then we add the pid to the buffer
pidsRemaining = append(pidsRemaining, pid)
}
// scan only the pids which are left in the buffer
allPids = pidsRemaining
}
// not scanning anymore if we couldn't find a single match
if !foundNewPid {
break
}
}
res := make(map[int]*nomadPid)
for pid := range processFamily {
np := nomadPid{
pid: pid,
cpuStatsTotal: stats.NewCpuStats(),
cpuStatsUser: stats.NewCpuStats(),
cpuStatsSys: stats.NewCpuStats(),
}
res[pid] = &np
}
return res, nil
}
// aggregatedResourceUsage aggregates the resource usage of all the pids and
// returns a TaskResourceUsage data point
func (e *UniversalExecutor) aggregatedResourceUsage(pidStats map[string]*cstructs.ResourceUsage) *cstructs.TaskResourceUsage {
ts := time.Now().UTC().UnixNano()
var (
systemModeCPU, userModeCPU, percent float64
totalRSS, totalSwap uint64
)
for _, pidStat := range pidStats {
systemModeCPU += pidStat.CpuStats.SystemMode
userModeCPU += pidStat.CpuStats.UserMode
percent += pidStat.CpuStats.Percent
totalRSS += pidStat.MemoryStats.RSS
totalSwap += pidStat.MemoryStats.Swap
}
totalCPU := &cstructs.CpuStats{
SystemMode: systemModeCPU,
UserMode: userModeCPU,
Percent: percent,
Measured: ExecutorBasicMeasuredCpuStats,
TotalTicks: e.systemCpuStats.TicksConsumed(percent),
}
totalMemory := &cstructs.MemoryStats{
RSS: totalRSS,
Swap: totalSwap,
Measured: ExecutorBasicMeasuredMemStats,
}
resourceUsage := cstructs.ResourceUsage{
MemoryStats: totalMemory,
CpuStats: totalCPU,
}
return &cstructs.TaskResourceUsage{
ResourceUsage: &resourceUsage,
Timestamp: ts,
Pids: pidStats,
}
}

View File

@@ -0,0 +1,46 @@
// +build darwin dragonfly freebsd netbsd openbsd solaris windows
package executor
import (
"os"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/mitchellh/go-ps"
)
func (e *UniversalExecutor) configureChroot() error {
return nil
}
func (e *UniversalExecutor) removeChrootMounts() error {
return nil
}
func (e *UniversalExecutor) runAs(userid string) error {
return nil
}
func (e *UniversalExecutor) applyLimits(pid int) error {
return nil
}
func (e *UniversalExecutor) configureIsolation() error {
return nil
}
func (e *UniversalExecutor) Stats() (*cstructs.TaskResourceUsage, error) {
pidStats, err := e.pidStats()
if err != nil {
return nil, err
}
return e.aggregatedResourceUsage(pidStats), nil
}
func (e *UniversalExecutor) getAllPids() (map[int]*nomadPid, error) {
allProcesses, err := ps.Processes()
if err != nil {
return nil, err
}
return e.scanPids(os.Getpid(), allProcesses)
}

View File

@@ -0,0 +1,373 @@
package executor
import (
"fmt"
"os"
"os/user"
"path/filepath"
"strconv"
"strings"
"syscall"
"time"
"github.com/hashicorp/go-multierror"
"github.com/mitchellh/go-ps"
"github.com/opencontainers/runc/libcontainer/cgroups"
cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
"github.com/opencontainers/runc/libcontainer/system"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/stats"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/nomad/structs"
)
var (
// A mapping of directories on the host OS to attempt to embed inside each
// task's chroot.
chrootEnv = map[string]string{
"/bin": "/bin",
"/etc": "/etc",
"/lib": "/lib",
"/lib32": "/lib32",
"/lib64": "/lib64",
"/run/resolvconf": "/run/resolvconf",
"/sbin": "/sbin",
"/usr": "/usr",
}
// clockTicks is the clocks per second of the machine
clockTicks = uint64(system.GetClockTicks())
// The statistics the executor exposes when using cgroups
ExecutorCgroupMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Max Usage", "Kernel Usage", "Kernel Max Usage"}
ExecutorCgroupMeasuredCpuStats = []string{"System Mode", "User Mode", "Throttled Periods", "Throttled Time", "Percent"}
)
// configureIsolation configures chroot and creates cgroups
func (e *UniversalExecutor) configureIsolation() error {
if e.command.FSIsolation {
if err := e.configureChroot(); err != nil {
return err
}
}
if e.command.ResourceLimits {
if err := e.configureCgroups(e.ctx.Task.Resources); err != nil {
return fmt.Errorf("error creating cgroups: %v", err)
}
}
return nil
}
// applyLimits puts a process in a pre-configured cgroup
func (e *UniversalExecutor) applyLimits(pid int) error {
if !e.command.ResourceLimits {
return nil
}
// Entering the process in the cgroup
manager := getCgroupManager(e.resConCtx.groups, nil)
if err := manager.Apply(pid); err != nil {
e.logger.Printf("[ERR] executor: error applying pid to cgroup: %v", err)
if er := e.removeChrootMounts(); er != nil {
e.logger.Printf("[ERR] executor: error removing chroot: %v", er)
}
return err
}
e.resConCtx.cgPaths = manager.GetPaths()
cgConfig := cgroupConfig.Config{Cgroups: e.resConCtx.groups}
if err := manager.Set(&cgConfig); err != nil {
e.logger.Printf("[ERR] executor: error setting cgroup config: %v", err)
if er := DestroyCgroup(e.resConCtx.groups, e.resConCtx.cgPaths, os.Getpid()); er != nil {
e.logger.Printf("[ERR] executor: error destroying cgroup: %v", er)
}
if er := e.removeChrootMounts(); er != nil {
e.logger.Printf("[ERR] executor: error removing chroot: %v", er)
}
return err
}
return nil
}
// configureCgroups converts a Nomad Resources specification into the equivalent
// cgroup configuration. It returns an error if the resources are invalid.
func (e *UniversalExecutor) configureCgroups(resources *structs.Resources) error {
e.resConCtx.groups = &cgroupConfig.Cgroup{}
e.resConCtx.groups.Resources = &cgroupConfig.Resources{}
cgroupName := structs.GenerateUUID()
e.resConCtx.groups.Path = filepath.Join("/nomad", cgroupName)
// TODO: verify this is needed for things like network access
e.resConCtx.groups.Resources.AllowAllDevices = true
if resources.MemoryMB > 0 {
// Total amount of memory allowed to consume
e.resConCtx.groups.Resources.Memory = int64(resources.MemoryMB * 1024 * 1024)
// Disable swap to avoid issues on the machine
e.resConCtx.groups.Resources.MemorySwap = int64(-1)
}
if resources.CPU < 2 {
return fmt.Errorf("resources.CPU must be equal to or greater than 2: %v", resources.CPU)
}
// Set the relative CPU shares for this cgroup.
e.resConCtx.groups.Resources.CpuShares = int64(resources.CPU)
if resources.IOPS != 0 {
// Validate it is in an acceptable range.
if resources.IOPS < 10 || resources.IOPS > 1000 {
return fmt.Errorf("resources.IOPS must be between 10 and 1000: %d", resources.IOPS)
}
e.resConCtx.groups.Resources.BlkioWeight = uint16(resources.IOPS)
}
return nil
}
// Stats reports the resource utilization of the cgroup. If there is no resource
// isolation we aggregate the resource utilization of all the pids launched by
// the executor.
func (e *UniversalExecutor) Stats() (*cstructs.TaskResourceUsage, error) {
if !e.command.ResourceLimits {
pidStats, err := e.pidStats()
if err != nil {
return nil, err
}
return e.aggregatedResourceUsage(pidStats), nil
}
ts := time.Now()
manager := getCgroupManager(e.resConCtx.groups, e.resConCtx.cgPaths)
stats, err := manager.GetStats()
if err != nil {
return nil, err
}
// Memory Related Stats
swap := stats.MemoryStats.SwapUsage
maxUsage := stats.MemoryStats.Usage.MaxUsage
rss := stats.MemoryStats.Stats["rss"]
cache := stats.MemoryStats.Stats["cache"]
ms := &cstructs.MemoryStats{
RSS: rss,
Cache: cache,
Swap: swap.Usage,
MaxUsage: maxUsage,
KernelUsage: stats.MemoryStats.KernelUsage.Usage,
KernelMaxUsage: stats.MemoryStats.KernelUsage.MaxUsage,
Measured: ExecutorCgroupMeasuredMemStats,
}
// CPU Related Stats
totalProcessCPUUsage := float64(stats.CpuStats.CpuUsage.TotalUsage)
userModeTime := float64(stats.CpuStats.CpuUsage.UsageInUsermode)
kernelModeTime := float64(stats.CpuStats.CpuUsage.UsageInKernelmode)
totalPercent := e.totalCpuStats.Percent(totalProcessCPUUsage)
cs := &cstructs.CpuStats{
SystemMode: e.systemCpuStats.Percent(kernelModeTime),
UserMode: e.userCpuStats.Percent(userModeTime),
Percent: totalPercent,
ThrottledPeriods: stats.CpuStats.ThrottlingData.ThrottledPeriods,
ThrottledTime: stats.CpuStats.ThrottlingData.ThrottledTime,
TotalTicks: e.systemCpuStats.TicksConsumed(totalPercent),
Measured: ExecutorCgroupMeasuredCpuStats,
}
taskResUsage := cstructs.TaskResourceUsage{
ResourceUsage: &cstructs.ResourceUsage{
MemoryStats: ms,
CpuStats: cs,
},
Timestamp: ts.UTC().UnixNano(),
}
if pidStats, err := e.pidStats(); err == nil {
taskResUsage.Pids = pidStats
}
return &taskResUsage, nil
}
// runAs takes a user id as a string and looks up the user, and sets the command
// to execute as that user.
func (e *UniversalExecutor) runAs(userid string) error {
u, err := user.Lookup(userid)
if err != nil {
return fmt.Errorf("Failed to identify user %v: %v", userid, err)
}
// Convert the uid and gid
uid, err := strconv.ParseUint(u.Uid, 10, 32)
if err != nil {
return fmt.Errorf("Unable to convert userid to uint32: %s", err)
}
gid, err := strconv.ParseUint(u.Gid, 10, 32)
if err != nil {
return fmt.Errorf("Unable to convert groupid to uint32: %s", err)
}
// Set the command to run as that user and group.
if e.cmd.SysProcAttr == nil {
e.cmd.SysProcAttr = &syscall.SysProcAttr{}
}
if e.cmd.SysProcAttr.Credential == nil {
e.cmd.SysProcAttr.Credential = &syscall.Credential{}
}
e.cmd.SysProcAttr.Credential.Uid = uint32(uid)
e.cmd.SysProcAttr.Credential.Gid = uint32(gid)
return nil
}
// configureChroot configures a chroot
func (e *UniversalExecutor) configureChroot() error {
allocDir := e.ctx.AllocDir
if err := allocDir.MountSharedDir(e.ctx.Task.Name); err != nil {
return err
}
chroot := chrootEnv
if len(e.ctx.ChrootEnv) > 0 {
chroot = e.ctx.ChrootEnv
}
if err := allocDir.Embed(e.ctx.Task.Name, chroot); err != nil {
return err
}
// Set the tasks AllocDir environment variable.
e.ctx.TaskEnv.
SetAllocDir(filepath.Join("/", allocdir.SharedAllocName)).
SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal)).
Build()
if e.cmd.SysProcAttr == nil {
e.cmd.SysProcAttr = &syscall.SysProcAttr{}
}
e.cmd.SysProcAttr.Chroot = e.taskDir
e.cmd.Dir = "/"
if err := allocDir.MountSpecialDirs(e.taskDir); err != nil {
return err
}
e.fsIsolationEnforced = true
return nil
}
// cleanTaskDir is an idempotent operation to clean the task directory and
// should be called when tearing down the task.
func (e *UniversalExecutor) removeChrootMounts() error {
// Prevent a race between Wait/ForceStop
e.resConCtx.cgLock.Lock()
defer e.resConCtx.cgLock.Unlock()
return e.ctx.AllocDir.UnmountAll()
}
// getAllPids returns the pids of all the processes spun up by the executor. We
// use the libcontainer apis to get the pids when the user is using cgroup
// isolation and we scan the entire process table if the user is not using any
// isolation
func (e *UniversalExecutor) getAllPids() (map[int]*nomadPid, error) {
if e.command.ResourceLimits {
manager := getCgroupManager(e.resConCtx.groups, e.resConCtx.cgPaths)
pids, err := manager.GetAllPids()
if err != nil {
return nil, err
}
np := make(map[int]*nomadPid, len(pids))
for _, pid := range pids {
np[pid] = &nomadPid{
pid: pid,
cpuStatsTotal: stats.NewCpuStats(),
cpuStatsSys: stats.NewCpuStats(),
cpuStatsUser: stats.NewCpuStats(),
}
}
return np, nil
}
allProcesses, err := ps.Processes()
if err != nil {
return nil, err
}
return e.scanPids(os.Getpid(), allProcesses)
}
// destroyCgroup kills all processes in the cgroup and removes the cgroup
// configuration from the host. This function is idempotent.
func DestroyCgroup(groups *cgroupConfig.Cgroup, cgPaths map[string]string, executorPid int) error {
mErrs := new(multierror.Error)
if groups == nil {
return fmt.Errorf("Can't destroy: cgroup configuration empty")
}
// Move the executor into the global cgroup so that the task specific
// cgroup can be destroyed.
nilGroup := &cgroupConfig.Cgroup{}
nilGroup.Path = "/"
nilGroup.Resources = groups.Resources
nilManager := getCgroupManager(nilGroup, nil)
err := nilManager.Apply(executorPid)
if err != nil && !strings.Contains(err.Error(), "no such process") {
return fmt.Errorf("failed to remove executor pid %d: %v", executorPid, err)
}
// Freeze the Cgroup so that it can not continue to fork/exec.
manager := getCgroupManager(groups, cgPaths)
err = manager.Freeze(cgroupConfig.Frozen)
if err != nil && !strings.Contains(err.Error(), "no such file or directory") {
return fmt.Errorf("failed to freeze cgroup: %v", err)
}
var procs []*os.Process
pids, err := manager.GetAllPids()
if err != nil {
multierror.Append(mErrs, fmt.Errorf("error getting pids: %v", err))
// Unfreeze the cgroup.
err = manager.Freeze(cgroupConfig.Thawed)
if err != nil && !strings.Contains(err.Error(), "no such file or directory") {
multierror.Append(mErrs, fmt.Errorf("failed to unfreeze cgroup: %v", err))
}
return mErrs.ErrorOrNil()
}
// Kill the processes in the cgroup
for _, pid := range pids {
proc, err := os.FindProcess(pid)
if err != nil {
multierror.Append(mErrs, fmt.Errorf("error finding process %v: %v", pid, err))
continue
}
procs = append(procs, proc)
if e := proc.Kill(); e != nil {
multierror.Append(mErrs, fmt.Errorf("error killing process %v: %v", pid, e))
}
}
// Unfreeze the cgroug so we can wait.
err = manager.Freeze(cgroupConfig.Thawed)
if err != nil && !strings.Contains(err.Error(), "no such file or directory") {
multierror.Append(mErrs, fmt.Errorf("failed to unfreeze cgroup: %v", err))
}
// Wait on the killed processes to ensure they are cleaned up.
for _, proc := range procs {
// Don't capture the error because we expect this to fail for
// processes we didn't fork.
proc.Wait()
}
// Remove the cgroup.
if err := manager.Destroy(); err != nil {
multierror.Append(mErrs, fmt.Errorf("failed to delete the cgroup directories: %v", err))
}
return mErrs.ErrorOrNil()
}
// getCgroupManager returns the correct libcontainer cgroup manager.
func getCgroupManager(groups *cgroupConfig.Cgroup, paths map[string]string) cgroups.Manager {
return &cgroupFs.Manager{Cgroups: groups, Paths: paths}
}

View File

@@ -0,0 +1,50 @@
// +build darwin dragonfly freebsd linux netbsd openbsd solaris
package executor
import (
"fmt"
"io"
"log/syslog"
"github.com/hashicorp/nomad/client/driver/logging"
)
func (e *UniversalExecutor) LaunchSyslogServer(ctx *ExecutorContext) (*SyslogServerState, error) {
e.ctx = ctx
// configuring the task dir
if err := e.configureTaskDir(); err != nil {
return nil, err
}
e.syslogChan = make(chan *logging.SyslogMessage, 2048)
l, err := e.getListener(e.ctx.PortLowerBound, e.ctx.PortUpperBound)
if err != nil {
return nil, err
}
e.logger.Printf("[DEBUG] sylog-server: launching syslog server on addr: %v", l.Addr().String())
if err := e.configureLoggers(); err != nil {
return nil, err
}
e.syslogServer = logging.NewSyslogServer(l, e.syslogChan, e.logger)
go e.syslogServer.Start()
go e.collectLogs(e.lre, e.lro)
syslogAddr := fmt.Sprintf("%s://%s", l.Addr().Network(), l.Addr().String())
return &SyslogServerState{Addr: syslogAddr}, nil
}
func (e *UniversalExecutor) collectLogs(we io.Writer, wo io.Writer) {
for logParts := range e.syslogChan {
// If the severity of the log line is err then we write to stderr
// otherwise all messages go to stdout
if logParts.Severity == syslog.LOG_ERR {
e.lre.Write(logParts.Message)
e.lre.Write([]byte{'\n'})
} else {
e.lro.Write(logParts.Message)
e.lro.Write([]byte{'\n'})
}
}
}

View File

@@ -0,0 +1,5 @@
package executor
func (e *UniversalExecutor) LaunchSyslogServer(ctx *ExecutorContext) (*SyslogServerState, error) {
return nil, nil
}

View File

@@ -0,0 +1,24 @@
// +build darwin dragonfly freebsd netbsd openbsd solaris windows
package executor
import (
dstructs "github.com/hashicorp/nomad/client/driver/structs"
)
// resourceContainerContext is a platform-specific struct for managing a
// resource container.
type resourceContainerContext struct {
}
func clientCleanup(ic *dstructs.IsolationConfig, pid int) error {
return nil
}
func (rc *resourceContainerContext) executorCleanup() error {
return nil
}
func (rc *resourceContainerContext) getIsolationConfig() *dstructs.IsolationConfig {
return nil
}

View File

@@ -0,0 +1,42 @@
package executor
import (
"os"
"sync"
dstructs "github.com/hashicorp/nomad/client/driver/structs"
cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
)
// resourceContainerContext is a platform-specific struct for managing a
// resource container. In the case of Linux, this is used to control Cgroups.
type resourceContainerContext struct {
groups *cgroupConfig.Cgroup
cgPaths map[string]string
cgLock sync.Mutex
}
// clientCleanup remoevs this host's Cgroup from the Nomad Client's context
func clientCleanup(ic *dstructs.IsolationConfig, pid int) error {
if err := DestroyCgroup(ic.Cgroup, ic.CgroupPaths, pid); err != nil {
return err
}
return nil
}
// cleanup removes this host's Cgroup from within an Executor's context
func (rc *resourceContainerContext) executorCleanup() error {
rc.cgLock.Lock()
defer rc.cgLock.Unlock()
if err := DestroyCgroup(rc.groups, rc.cgPaths, os.Getpid()); err != nil {
return err
}
return nil
}
func (rc *resourceContainerContext) getIsolationConfig() *dstructs.IsolationConfig {
return &dstructs.IsolationConfig{
Cgroup: rc.groups,
CgroupPaths: rc.cgPaths,
}
}

View File

@@ -0,0 +1,181 @@
package driver
import (
"encoding/gob"
"log"
"net/rpc"
"github.com/hashicorp/go-plugin"
"github.com/hashicorp/nomad/client/driver/executor"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/nomad/structs"
)
// Registering these types since we have to serialize and de-serialize the Task
// structs over the wire between drivers and the executor.
func init() {
gob.Register([]interface{}{})
gob.Register(map[string]interface{}{})
gob.Register([]map[string]string{})
gob.Register([]map[string]int{})
}
type ExecutorRPC struct {
client *rpc.Client
logger *log.Logger
}
// LaunchCmdArgs wraps a user command and the args for the purposes of RPC
type LaunchCmdArgs struct {
Cmd *executor.ExecCommand
Ctx *executor.ExecutorContext
}
// LaunchSyslogServerArgs wraps the executor context for the purposes of RPC
type LaunchSyslogServerArgs struct {
Ctx *executor.ExecutorContext
}
// SyncServicesArgs wraps the consul context for the purposes of RPC
type SyncServicesArgs struct {
Ctx *executor.ConsulContext
}
func (e *ExecutorRPC) LaunchCmd(cmd *executor.ExecCommand, ctx *executor.ExecutorContext) (*executor.ProcessState, error) {
var ps *executor.ProcessState
err := e.client.Call("Plugin.LaunchCmd", LaunchCmdArgs{Cmd: cmd, Ctx: ctx}, &ps)
return ps, err
}
func (e *ExecutorRPC) LaunchSyslogServer(ctx *executor.ExecutorContext) (*executor.SyslogServerState, error) {
var ss *executor.SyslogServerState
err := e.client.Call("Plugin.LaunchSyslogServer", LaunchSyslogServerArgs{Ctx: ctx}, &ss)
return ss, err
}
func (e *ExecutorRPC) Wait() (*executor.ProcessState, error) {
var ps executor.ProcessState
err := e.client.Call("Plugin.Wait", new(interface{}), &ps)
return &ps, err
}
func (e *ExecutorRPC) ShutDown() error {
return e.client.Call("Plugin.ShutDown", new(interface{}), new(interface{}))
}
func (e *ExecutorRPC) Exit() error {
return e.client.Call("Plugin.Exit", new(interface{}), new(interface{}))
}
func (e *ExecutorRPC) UpdateLogConfig(logConfig *structs.LogConfig) error {
return e.client.Call("Plugin.UpdateLogConfig", logConfig, new(interface{}))
}
func (e *ExecutorRPC) UpdateTask(task *structs.Task) error {
return e.client.Call("Plugin.UpdateTask", task, new(interface{}))
}
func (e *ExecutorRPC) SyncServices(ctx *executor.ConsulContext) error {
return e.client.Call("Plugin.SyncServices", SyncServicesArgs{Ctx: ctx}, new(interface{}))
}
func (e *ExecutorRPC) DeregisterServices() error {
return e.client.Call("Plugin.DeregisterServices", new(interface{}), new(interface{}))
}
func (e *ExecutorRPC) Version() (*executor.ExecutorVersion, error) {
var version executor.ExecutorVersion
err := e.client.Call("Plugin.Version", new(interface{}), &version)
return &version, err
}
func (e *ExecutorRPC) Stats() (*cstructs.TaskResourceUsage, error) {
var resourceUsage cstructs.TaskResourceUsage
err := e.client.Call("Plugin.Stats", new(interface{}), &resourceUsage)
return &resourceUsage, err
}
type ExecutorRPCServer struct {
Impl executor.Executor
logger *log.Logger
}
func (e *ExecutorRPCServer) LaunchCmd(args LaunchCmdArgs, ps *executor.ProcessState) error {
state, err := e.Impl.LaunchCmd(args.Cmd, args.Ctx)
if state != nil {
*ps = *state
}
return err
}
func (e *ExecutorRPCServer) LaunchSyslogServer(args LaunchSyslogServerArgs, ss *executor.SyslogServerState) error {
state, err := e.Impl.LaunchSyslogServer(args.Ctx)
if state != nil {
*ss = *state
}
return err
}
func (e *ExecutorRPCServer) Wait(args interface{}, ps *executor.ProcessState) error {
state, err := e.Impl.Wait()
if state != nil {
*ps = *state
}
return err
}
func (e *ExecutorRPCServer) ShutDown(args interface{}, resp *interface{}) error {
return e.Impl.ShutDown()
}
func (e *ExecutorRPCServer) Exit(args interface{}, resp *interface{}) error {
return e.Impl.Exit()
}
func (e *ExecutorRPCServer) UpdateLogConfig(args *structs.LogConfig, resp *interface{}) error {
return e.Impl.UpdateLogConfig(args)
}
func (e *ExecutorRPCServer) UpdateTask(args *structs.Task, resp *interface{}) error {
return e.Impl.UpdateTask(args)
}
func (e *ExecutorRPCServer) SyncServices(args SyncServicesArgs, resp *interface{}) error {
return e.Impl.SyncServices(args.Ctx)
}
func (e *ExecutorRPCServer) DeregisterServices(args interface{}, resp *interface{}) error {
return e.Impl.DeregisterServices()
}
func (e *ExecutorRPCServer) Version(args interface{}, version *executor.ExecutorVersion) error {
ver, err := e.Impl.Version()
if ver != nil {
*version = *ver
}
return err
}
func (e *ExecutorRPCServer) Stats(args interface{}, resourceUsage *cstructs.TaskResourceUsage) error {
ru, err := e.Impl.Stats()
if ru != nil {
*resourceUsage = *ru
}
return err
}
type ExecutorPlugin struct {
logger *log.Logger
Impl *ExecutorRPCServer
}
func (p *ExecutorPlugin) Server(*plugin.MuxBroker) (interface{}, error) {
if p.Impl == nil {
p.Impl = &ExecutorRPCServer{Impl: executor.NewExecutor(p.logger), logger: p.logger}
}
return p.Impl, nil
}
func (p *ExecutorPlugin) Client(b *plugin.MuxBroker, c *rpc.Client) (interface{}, error) {
return &ExecutorRPC{client: c, logger: p.logger}, nil
}

416
vendor/github.com/hashicorp/nomad/client/driver/java.go generated vendored Normal file
View File

@@ -0,0 +1,416 @@
package driver
import (
"bytes"
"encoding/json"
"fmt"
"log"
"os/exec"
"path/filepath"
"runtime"
"strings"
"syscall"
"time"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/go-plugin"
"github.com/mitchellh/mapstructure"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/client/driver/executor"
dstructs "github.com/hashicorp/nomad/client/driver/structs"
"github.com/hashicorp/nomad/client/fingerprint"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/helper/discover"
"github.com/hashicorp/nomad/helper/fields"
"github.com/hashicorp/nomad/nomad/structs"
)
const (
// The key populated in Node Attributes to indicate presence of the Java
// driver
javaDriverAttr = "driver.java"
)
// JavaDriver is a simple driver to execute applications packaged in Jars.
// It literally just fork/execs tasks with the java command.
type JavaDriver struct {
DriverContext
fingerprint.StaticFingerprinter
}
type JavaDriverConfig struct {
JarPath string `mapstructure:"jar_path"`
JvmOpts []string `mapstructure:"jvm_options"`
Args []string `mapstructure:"args"`
}
// javaHandle is returned from Start/Open as a handle to the PID
type javaHandle struct {
pluginClient *plugin.Client
userPid int
executor executor.Executor
isolationConfig *dstructs.IsolationConfig
taskDir string
allocDir *allocdir.AllocDir
killTimeout time.Duration
maxKillTimeout time.Duration
version string
logger *log.Logger
waitCh chan *dstructs.WaitResult
doneCh chan struct{}
}
// NewJavaDriver is used to create a new exec driver
func NewJavaDriver(ctx *DriverContext) Driver {
return &JavaDriver{DriverContext: *ctx}
}
// Validate is used to validate the driver configuration
func (d *JavaDriver) Validate(config map[string]interface{}) error {
fd := &fields.FieldData{
Raw: config,
Schema: map[string]*fields.FieldSchema{
"jar_path": &fields.FieldSchema{
Type: fields.TypeString,
Required: true,
},
"jvm_options": &fields.FieldSchema{
Type: fields.TypeArray,
},
"args": &fields.FieldSchema{
Type: fields.TypeArray,
},
},
}
if err := fd.Validate(); err != nil {
return err
}
return nil
}
func (d *JavaDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
// Get the current status so that we can log any debug messages only if the
// state changes
_, currentlyEnabled := node.Attributes[javaDriverAttr]
// Only enable if we are root and cgroups are mounted when running on linux systems.
if runtime.GOOS == "linux" && (syscall.Geteuid() != 0 || !d.cgroupsMounted(node)) {
if currentlyEnabled {
d.logger.Printf("[DEBUG] driver.java: root priviledges and mounted cgroups required on linux, disabling")
}
delete(node.Attributes, "driver.java")
return false, nil
}
// Find java version
var out bytes.Buffer
var erOut bytes.Buffer
cmd := exec.Command("java", "-version")
cmd.Stdout = &out
cmd.Stderr = &erOut
err := cmd.Run()
if err != nil {
// assume Java wasn't found
delete(node.Attributes, javaDriverAttr)
return false, nil
}
// 'java -version' returns output on Stderr typically.
// Check stdout, but it's probably empty
var infoString string
if out.String() != "" {
infoString = out.String()
}
if erOut.String() != "" {
infoString = erOut.String()
}
if infoString == "" {
if currentlyEnabled {
d.logger.Println("[WARN] driver.java: error parsing Java version information, aborting")
}
delete(node.Attributes, javaDriverAttr)
return false, nil
}
// Assume 'java -version' returns 3 lines:
// java version "1.6.0_36"
// OpenJDK Runtime Environment (IcedTea6 1.13.8) (6b36-1.13.8-0ubuntu1~12.04)
// OpenJDK 64-Bit Server VM (build 23.25-b01, mixed mode)
// Each line is terminated by \n
info := strings.Split(infoString, "\n")
versionString := info[0]
versionString = strings.TrimPrefix(versionString, "java version ")
versionString = strings.Trim(versionString, "\"")
node.Attributes[javaDriverAttr] = "1"
node.Attributes["driver.java.version"] = versionString
node.Attributes["driver.java.runtime"] = info[1]
node.Attributes["driver.java.vm"] = info[2]
return true, nil
}
func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
var driverConfig JavaDriverConfig
if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
return nil, err
}
// Set the host environment variables.
filter := strings.Split(d.config.ReadDefault("env.blacklist", config.DefaultEnvBlacklist), ",")
d.taskEnv.AppendHostEnvvars(filter)
taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName]
if !ok {
return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName)
}
if driverConfig.JarPath == "" {
return nil, fmt.Errorf("jar_path must be specified")
}
args := []string{}
// Look for jvm options
if len(driverConfig.JvmOpts) != 0 {
d.logger.Printf("[DEBUG] driver.java: found JVM options: %s", driverConfig.JvmOpts)
args = append(args, driverConfig.JvmOpts...)
}
// Build the argument list.
args = append(args, "-jar", driverConfig.JarPath)
if len(driverConfig.Args) != 0 {
args = append(args, driverConfig.Args...)
}
bin, err := discover.NomadExecutable()
if err != nil {
return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
}
pluginLogFile := filepath.Join(taskDir, fmt.Sprintf("%s-executor.out", task.Name))
pluginConfig := &plugin.ClientConfig{
Cmd: exec.Command(bin, "executor", pluginLogFile),
}
execIntf, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
if err != nil {
return nil, err
}
executorCtx := &executor.ExecutorContext{
TaskEnv: d.taskEnv,
Driver: "java",
AllocDir: ctx.AllocDir,
AllocID: ctx.AllocID,
ChrootEnv: d.config.ChrootEnv,
Task: task,
}
absPath, err := GetAbsolutePath("java")
if err != nil {
return nil, err
}
ps, err := execIntf.LaunchCmd(&executor.ExecCommand{
Cmd: absPath,
Args: args,
FSIsolation: true,
ResourceLimits: true,
User: getExecutorUser(task),
}, executorCtx)
if err != nil {
pluginClient.Kill()
return nil, err
}
d.logger.Printf("[DEBUG] driver.java: started process with pid: %v", ps.Pid)
// Return a driver handle
maxKill := d.DriverContext.config.MaxKillTimeout
h := &javaHandle{
pluginClient: pluginClient,
executor: execIntf,
userPid: ps.Pid,
isolationConfig: ps.IsolationConfig,
taskDir: taskDir,
allocDir: ctx.AllocDir,
killTimeout: GetKillTimeout(task.KillTimeout, maxKill),
maxKillTimeout: maxKill,
version: d.config.Version,
logger: d.logger,
doneCh: make(chan struct{}),
waitCh: make(chan *dstructs.WaitResult, 1),
}
if err := h.executor.SyncServices(consulContext(d.config, "")); err != nil {
d.logger.Printf("[ERR] driver.java: error registering services with consul for task: %q: %v", task.Name, err)
}
go h.run()
return h, nil
}
// cgroupsMounted returns true if the cgroups are mounted on a system otherwise
// returns false
func (d *JavaDriver) cgroupsMounted(node *structs.Node) bool {
_, ok := node.Attributes["unique.cgroup.mountpoint"]
return ok
}
type javaId struct {
Version string
KillTimeout time.Duration
MaxKillTimeout time.Duration
PluginConfig *PluginReattachConfig
IsolationConfig *dstructs.IsolationConfig
TaskDir string
AllocDir *allocdir.AllocDir
UserPid int
}
func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
id := &javaId{}
if err := json.Unmarshal([]byte(handleID), id); err != nil {
return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
}
pluginConfig := &plugin.ClientConfig{
Reattach: id.PluginConfig.PluginConfig(),
}
exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
if err != nil {
merrs := new(multierror.Error)
merrs.Errors = append(merrs.Errors, err)
d.logger.Println("[ERR] driver.java: error connecting to plugin so destroying plugin pid and user pid")
if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
merrs.Errors = append(merrs.Errors, fmt.Errorf("error destroying plugin and userpid: %v", e))
}
if id.IsolationConfig != nil {
ePid := pluginConfig.Reattach.Pid
if e := executor.ClientCleanup(id.IsolationConfig, ePid); e != nil {
merrs.Errors = append(merrs.Errors, fmt.Errorf("destroying resource container failed: %v", e))
}
}
if e := ctx.AllocDir.UnmountAll(); e != nil {
merrs.Errors = append(merrs.Errors, e)
}
return nil, fmt.Errorf("error connecting to plugin: %v", merrs.ErrorOrNil())
}
ver, _ := exec.Version()
d.logger.Printf("[DEBUG] driver.java: version of executor: %v", ver.Version)
// Return a driver handle
h := &javaHandle{
pluginClient: pluginClient,
executor: exec,
userPid: id.UserPid,
isolationConfig: id.IsolationConfig,
taskDir: id.TaskDir,
allocDir: id.AllocDir,
logger: d.logger,
version: id.Version,
killTimeout: id.KillTimeout,
maxKillTimeout: id.MaxKillTimeout,
doneCh: make(chan struct{}),
waitCh: make(chan *dstructs.WaitResult, 1),
}
if err := h.executor.SyncServices(consulContext(d.config, "")); err != nil {
d.logger.Printf("[ERR] driver.java: error registering services with consul: %v", err)
}
go h.run()
return h, nil
}
func (h *javaHandle) ID() string {
id := javaId{
Version: h.version,
KillTimeout: h.killTimeout,
MaxKillTimeout: h.maxKillTimeout,
PluginConfig: NewPluginReattachConfig(h.pluginClient.ReattachConfig()),
UserPid: h.userPid,
TaskDir: h.taskDir,
AllocDir: h.allocDir,
IsolationConfig: h.isolationConfig,
}
data, err := json.Marshal(id)
if err != nil {
h.logger.Printf("[ERR] driver.java: failed to marshal ID to JSON: %s", err)
}
return string(data)
}
func (h *javaHandle) WaitCh() chan *dstructs.WaitResult {
return h.waitCh
}
func (h *javaHandle) Update(task *structs.Task) error {
// Store the updated kill timeout.
h.killTimeout = GetKillTimeout(task.KillTimeout, h.maxKillTimeout)
h.executor.UpdateTask(task)
// Update is not possible
return nil
}
func (h *javaHandle) Kill() error {
if err := h.executor.ShutDown(); err != nil {
if h.pluginClient.Exited() {
return nil
}
return fmt.Errorf("executor Shutdown failed: %v", err)
}
select {
case <-h.doneCh:
return nil
case <-time.After(h.killTimeout):
if h.pluginClient.Exited() {
return nil
}
if err := h.executor.Exit(); err != nil {
return fmt.Errorf("executor Exit failed: %v", err)
}
return nil
}
}
func (h *javaHandle) Stats() (*cstructs.TaskResourceUsage, error) {
return h.executor.Stats()
}
func (h *javaHandle) run() {
ps, err := h.executor.Wait()
close(h.doneCh)
if ps.ExitCode == 0 && err != nil {
if h.isolationConfig != nil {
ePid := h.pluginClient.ReattachConfig().Pid
if e := executor.ClientCleanup(h.isolationConfig, ePid); e != nil {
h.logger.Printf("[ERR] driver.java: destroying resource container failed: %v", e)
}
} else {
if e := killProcess(h.userPid); e != nil {
h.logger.Printf("[ERR] driver.java: error killing user process: %v", e)
}
}
if e := h.allocDir.UnmountAll(); e != nil {
h.logger.Printf("[ERR] driver.java: unmounting dev,proc and alloc dirs failed: %v", e)
}
}
h.waitCh <- &dstructs.WaitResult{ExitCode: ps.ExitCode, Signal: ps.Signal, Err: err}
close(h.waitCh)
// Remove services
if err := h.executor.DeregisterServices(); err != nil {
h.logger.Printf("[ERR] driver.java: failed to kill the deregister services: %v", err)
}
h.executor.Exit()
h.pluginClient.Kill()
}

View File

@@ -0,0 +1,71 @@
package logging
import (
"log"
"github.com/hashicorp/nomad/client/allocdir"
cstructs "github.com/hashicorp/nomad/client/driver/structs"
"github.com/hashicorp/nomad/nomad/structs"
)
// LogCollectorContext holds context to configure the syslog server
type LogCollectorContext struct {
// TaskName is the name of the Task
TaskName string
// AllocDir is the handle to do operations on the alloc dir of
// the task
AllocDir *allocdir.AllocDir
// LogConfig provides configuration related to log rotation
LogConfig *structs.LogConfig
// PortUpperBound is the upper bound of the ports that we can use to start
// the syslog server
PortUpperBound uint
// PortLowerBound is the lower bound of the ports that we can use to start
// the syslog server
PortLowerBound uint
}
// SyslogCollectorState holds the address and islation information of a launched
// syslog server
type SyslogCollectorState struct {
IsolationConfig *cstructs.IsolationConfig
Addr string
}
// LogCollector is an interface which allows a driver to launch a log server
// and update log configuration
type LogCollector interface {
LaunchCollector(ctx *LogCollectorContext) (*SyslogCollectorState, error)
Exit() error
UpdateLogConfig(logConfig *structs.LogConfig) error
}
// SyslogCollector is a LogCollector which starts a syslog server and does
// rotation to incoming stream
type SyslogCollector struct {
}
// NewSyslogCollector returns an implementation of the SyslogCollector
func NewSyslogCollector(logger *log.Logger) *SyslogCollector {
return &SyslogCollector{}
}
// LaunchCollector launches a new syslog server and starts writing log lines to
// files and rotates them
func (s *SyslogCollector) LaunchCollector(ctx *LogCollectorContext) (*SyslogCollectorState, error) {
return nil, nil
}
// Exit kills the syslog server
func (s *SyslogCollector) Exit() error {
return nil
}
// UpdateLogConfig updates the log configuration
func (s *SyslogCollector) UpdateLogConfig(logConfig *structs.LogConfig) error {
return nil
}

View File

@@ -0,0 +1,285 @@
package logging
import (
"bufio"
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"time"
)
const (
bufSize = 32768
flushDur = 100 * time.Millisecond
)
// FileRotator writes bytes to a rotated set of files
type FileRotator struct {
MaxFiles int // MaxFiles is the maximum number of rotated files allowed in a path
FileSize int64 // FileSize is the size a rotated file is allowed to grow
path string // path is the path on the file system where the rotated set of files are opened
baseFileName string // baseFileName is the base file name of the rotated files
logFileIdx int // logFileIdx is the current index of the rotated files
oldestLogFileIdx int // oldestLogFileIdx is the index of the oldest log file in a path
currentFile *os.File // currentFile is the file that is currently getting written
currentWr int64 // currentWr is the number of bytes written to the current file
bufw *bufio.Writer
bufLock sync.Mutex
flushTicker *time.Ticker
logger *log.Logger
purgeCh chan struct{}
doneCh chan struct{}
closed bool
closedLock sync.Mutex
}
// NewFileRotator returns a new file rotator
func NewFileRotator(path string, baseFile string, maxFiles int,
fileSize int64, logger *log.Logger) (*FileRotator, error) {
rotator := &FileRotator{
MaxFiles: maxFiles,
FileSize: fileSize,
path: path,
baseFileName: baseFile,
flushTicker: time.NewTicker(flushDur),
logger: logger,
purgeCh: make(chan struct{}, 1),
doneCh: make(chan struct{}, 1),
}
if err := rotator.lastFile(); err != nil {
return nil, err
}
go rotator.purgeOldFiles()
go rotator.flushPeriodically()
return rotator, nil
}
// Write writes a byte array to a file and rotates the file if it's size becomes
// equal to the maximum size the user has defined.
func (f *FileRotator) Write(p []byte) (n int, err error) {
n = 0
var nw int
for n < len(p) {
// Check if we still have space in the current file, otherwise close and
// open the next file
if f.currentWr >= f.FileSize {
f.flushBuffer()
f.currentFile.Close()
if err := f.nextFile(); err != nil {
f.logger.Printf("[ERROR] driver.rotator: error creating next file: %v", err)
return 0, err
}
}
// Calculate the remaining size on this file
remainingSize := f.FileSize - f.currentWr
// Check if the number of bytes that we have to write is less than the
// remaining size of the file
if remainingSize < int64(len(p[n:])) {
// Write the number of bytes that we can write on the current file
li := int64(n) + remainingSize
nw, err = f.writeToBuffer(p[n:li])
} else {
// Write all the bytes in the current file
nw, err = f.writeToBuffer(p[n:])
}
// Increment the number of bytes written so far in this method
// invocation
n += nw
// Increment the total number of bytes in the file
f.currentWr += int64(n)
if err != nil {
f.logger.Printf("[ERROR] driver.rotator: error writing to file: %v", err)
return
}
}
return
}
// nextFile opens the next file and purges older files if the number of rotated
// files is larger than the maximum files configured by the user
func (f *FileRotator) nextFile() error {
nextFileIdx := f.logFileIdx
for {
nextFileIdx += 1
logFileName := filepath.Join(f.path, fmt.Sprintf("%s.%d", f.baseFileName, nextFileIdx))
if fi, err := os.Stat(logFileName); err == nil {
if fi.IsDir() || fi.Size() >= f.FileSize {
continue
}
}
f.logFileIdx = nextFileIdx
if err := f.createFile(); err != nil {
return err
}
break
}
// Purge old files if we have more files than MaxFiles
f.closedLock.Lock()
defer f.closedLock.Unlock()
if f.logFileIdx-f.oldestLogFileIdx >= f.MaxFiles && !f.closed {
select {
case f.purgeCh <- struct{}{}:
default:
}
}
return nil
}
// lastFile finds out the rotated file with the largest index in a path.
func (f *FileRotator) lastFile() error {
finfos, err := ioutil.ReadDir(f.path)
if err != nil {
return err
}
prefix := fmt.Sprintf("%s.", f.baseFileName)
for _, fi := range finfos {
if fi.IsDir() {
continue
}
if strings.HasPrefix(fi.Name(), prefix) {
fileIdx := strings.TrimPrefix(fi.Name(), prefix)
n, err := strconv.Atoi(fileIdx)
if err != nil {
continue
}
if n > f.logFileIdx {
f.logFileIdx = n
}
}
}
if err := f.createFile(); err != nil {
return err
}
return nil
}
// createFile opens a new or existing file for writing
func (f *FileRotator) createFile() error {
logFileName := filepath.Join(f.path, fmt.Sprintf("%s.%d", f.baseFileName, f.logFileIdx))
cFile, err := os.OpenFile(logFileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
return err
}
f.currentFile = cFile
fi, err := f.currentFile.Stat()
if err != nil {
return err
}
f.currentWr = fi.Size()
f.createOrResetBuffer()
return nil
}
// flushPeriodically flushes the buffered writer every 100ms to the underlying
// file
func (f *FileRotator) flushPeriodically() {
for _ = range f.flushTicker.C {
f.flushBuffer()
}
}
func (f *FileRotator) Close() {
f.closedLock.Lock()
defer f.closedLock.Unlock()
// Stop the ticker and flush for one last time
f.flushTicker.Stop()
f.flushBuffer()
// Stop the purge go routine
if !f.closed {
f.doneCh <- struct{}{}
close(f.purgeCh)
f.closed = true
}
}
// purgeOldFiles removes older files and keeps only the last N files rotated for
// a file
func (f *FileRotator) purgeOldFiles() {
for {
select {
case <-f.purgeCh:
var fIndexes []int
files, err := ioutil.ReadDir(f.path)
if err != nil {
return
}
// Inserting all the rotated files in a slice
for _, fi := range files {
if strings.HasPrefix(fi.Name(), f.baseFileName) {
fileIdx := strings.TrimPrefix(fi.Name(), fmt.Sprintf("%s.", f.baseFileName))
n, err := strconv.Atoi(fileIdx)
if err != nil {
continue
}
fIndexes = append(fIndexes, n)
}
}
// Not continuing to delete files if the number of files is not more
// than MaxFiles
if len(fIndexes) <= f.MaxFiles {
continue
}
// Sorting the file indexes so that we can purge the older files and keep
// only the number of files as configured by the user
sort.Sort(sort.IntSlice(fIndexes))
toDelete := fIndexes[0 : len(fIndexes)-f.MaxFiles]
for _, fIndex := range toDelete {
fname := filepath.Join(f.path, fmt.Sprintf("%s.%d", f.baseFileName, fIndex))
os.RemoveAll(fname)
}
f.oldestLogFileIdx = fIndexes[0]
case <-f.doneCh:
return
}
}
}
// flushBuffer flushes the buffer
func (f *FileRotator) flushBuffer() error {
f.bufLock.Lock()
defer f.bufLock.Unlock()
if f.bufw != nil {
return f.bufw.Flush()
}
return nil
}
// writeToBuffer writes the byte array to buffer
func (f *FileRotator) writeToBuffer(p []byte) (int, error) {
f.bufLock.Lock()
defer f.bufLock.Unlock()
return f.bufw.Write(p)
}
// createOrResetBuffer creates a new buffer if we don't have one otherwise
// resets the buffer
func (f *FileRotator) createOrResetBuffer() {
f.bufLock.Lock()
defer f.bufLock.Unlock()
if f.bufw == nil {
f.bufw = bufio.NewWriterSize(f.currentFile, bufSize)
} else {
f.bufw.Reset(f.currentFile)
}
}

View File

@@ -0,0 +1,158 @@
// +build darwin dragonfly freebsd linux netbsd openbsd solaris
package logging
import (
"fmt"
"log"
"log/syslog"
"strconv"
)
// Errors related to parsing priority
var (
ErrPriorityNoStart = fmt.Errorf("No start char found for priority")
ErrPriorityEmpty = fmt.Errorf("Priority field empty")
ErrPriorityNoEnd = fmt.Errorf("No end char found for priority")
ErrPriorityTooShort = fmt.Errorf("Priority field too short")
ErrPriorityTooLong = fmt.Errorf("Priority field too long")
ErrPriorityNonDigit = fmt.Errorf("Non digit found in priority")
)
// Priority header and ending characters
const (
PRI_PART_START = '<'
PRI_PART_END = '>'
)
// SyslogMessage represents a log line received
type SyslogMessage struct {
Message []byte
Severity syslog.Priority
}
// Priority holds all the priority bits in a syslog log line
type Priority struct {
Pri int
Facility syslog.Priority
Severity syslog.Priority
}
// DockerLogParser parses a line of log message that the docker daemon ships
type DockerLogParser struct {
logger *log.Logger
}
// NewDockerLogParser creates a new DockerLogParser
func NewDockerLogParser(logger *log.Logger) *DockerLogParser {
return &DockerLogParser{logger: logger}
}
// Parse parses a syslog log line
func (d *DockerLogParser) Parse(line []byte) *SyslogMessage {
pri, _, _ := d.parsePriority(line)
msgIdx := d.logContentIndex(line)
// Create a copy of the line so that subsequent Scans do not override the
// message
lineCopy := make([]byte, len(line[msgIdx:]))
copy(lineCopy, line[msgIdx:])
return &SyslogMessage{
Severity: pri.Severity,
Message: lineCopy,
}
}
// logContentIndex finds out the index of the start index of the content in a
// syslog line
func (d *DockerLogParser) logContentIndex(line []byte) int {
cursor := 0
numSpace := 0
numColons := 0
// first look for at least 2 colons. This matches into the date that has no more spaces in it
// DefaultFormatter log line look: '<30>2016-07-06T15:13:11Z00:00 hostname docker/9648c64f5037[16200]'
// UnixFormatter log line look: '<30>Jul 6 15:13:11 docker/9648c64f5037[16200]'
for i := 0; i < len(line); i++ {
if line[i] == ':' {
numColons += 1
if numColons == 2 {
cursor = i
break
}
}
}
// then look for the next space
for i := cursor; i < len(line); i++ {
if line[i] == ' ' {
numSpace += 1
if numSpace == 1 {
cursor = i
break
}
}
}
// then the colon is what seperates it, followed by a space
for i := cursor; i < len(line); i++ {
if line[i] == ':' && i+1 < len(line) && line[i+1] == ' ' {
cursor = i + 1
break
}
}
// return the cursor to the next character
return cursor + 1
}
// parsePriority parses the priority in a syslog message
func (d *DockerLogParser) parsePriority(line []byte) (Priority, int, error) {
cursor := 0
pri := d.newPriority(0)
if len(line) <= 0 {
return pri, cursor, ErrPriorityEmpty
}
if line[cursor] != PRI_PART_START {
return pri, cursor, ErrPriorityNoStart
}
i := 1
priDigit := 0
for i < len(line) {
if i >= 5 {
return pri, cursor, ErrPriorityTooLong
}
c := line[i]
if c == PRI_PART_END {
if i == 1 {
return pri, cursor, ErrPriorityTooShort
}
cursor = i + 1
return d.newPriority(priDigit), cursor, nil
}
if d.isDigit(c) {
v, e := strconv.Atoi(string(c))
if e != nil {
return pri, cursor, e
}
priDigit = (priDigit * 10) + v
} else {
return pri, cursor, ErrPriorityNonDigit
}
i++
}
return pri, cursor, ErrPriorityNoEnd
}
// isDigit checks if a byte is a numeric char
func (d *DockerLogParser) isDigit(c byte) bool {
return c >= '0' && c <= '9'
}
// newPriority creates a new default priority
func (d *DockerLogParser) newPriority(p int) Priority {
// The Priority value is calculated by first multiplying the Facility
// number by 8 and then adding the numerical value of the Severity.
return Priority{
Pri: p,
Facility: syslog.Priority(p / 8),
Severity: syslog.Priority(p % 8),
}
}

View File

@@ -0,0 +1,86 @@
// +build !windows
package logging
import (
"bufio"
"log"
"net"
"sync"
)
// SyslogServer is a server which listens to syslog messages and parses them
type SyslogServer struct {
listener net.Listener
messages chan *SyslogMessage
parser *DockerLogParser
doneCh chan interface{}
done bool
doneLock sync.Mutex
logger *log.Logger
}
// NewSyslogServer creates a new syslog server
func NewSyslogServer(l net.Listener, messages chan *SyslogMessage, logger *log.Logger) *SyslogServer {
parser := NewDockerLogParser(logger)
return &SyslogServer{
listener: l,
messages: messages,
parser: parser,
logger: logger,
doneCh: make(chan interface{}),
}
}
// Start starts accepting syslog connections
func (s *SyslogServer) Start() {
for {
select {
case <-s.doneCh:
s.listener.Close()
return
default:
connection, err := s.listener.Accept()
if err != nil {
s.logger.Printf("[ERR] logcollector.server: error in accepting connection: %v", err)
continue
}
go s.read(connection)
}
}
}
// read reads the bytes from a connection
func (s *SyslogServer) read(connection net.Conn) {
defer connection.Close()
scanner := bufio.NewScanner(bufio.NewReader(connection))
for {
select {
case <-s.doneCh:
return
default:
}
if scanner.Scan() {
b := scanner.Bytes()
msg := s.parser.Parse(b)
s.messages <- msg
} else {
return
}
}
}
// Shutdown shutsdown the syslog server
func (s *SyslogServer) Shutdown() {
s.doneLock.Lock()
s.doneLock.Unlock()
if !s.done {
close(s.doneCh)
close(s.messages)
s.done = true
}
}

View File

@@ -0,0 +1,10 @@
package logging
type SyslogServer struct {
}
func (s *SyslogServer) Shutdown() {
}
type SyslogMessage struct {
}

View File

@@ -0,0 +1,207 @@
// +build darwin dragonfly freebsd linux netbsd openbsd solaris
package logging
import (
"fmt"
"io"
"io/ioutil"
"log"
"log/syslog"
"net"
"os"
"runtime"
"github.com/hashicorp/nomad/client/allocdir"
cstructs "github.com/hashicorp/nomad/client/driver/structs"
"github.com/hashicorp/nomad/nomad/structs"
)
// LogCollectorContext holds context to configure the syslog server
type LogCollectorContext struct {
// TaskName is the name of the Task
TaskName string
// AllocDir is the handle to do operations on the alloc dir of
// the task
AllocDir *allocdir.AllocDir
// LogConfig provides configuration related to log rotation
LogConfig *structs.LogConfig
// PortUpperBound is the upper bound of the ports that we can use to start
// the syslog server
PortUpperBound uint
// PortLowerBound is the lower bound of the ports that we can use to start
// the syslog server
PortLowerBound uint
}
// SyslogCollectorState holds the address and islation information of a launched
// syslog server
type SyslogCollectorState struct {
IsolationConfig *cstructs.IsolationConfig
Addr string
}
// LogCollector is an interface which allows a driver to launch a log server
// and update log configuration
type LogCollector interface {
LaunchCollector(ctx *LogCollectorContext) (*SyslogCollectorState, error)
Exit() error
UpdateLogConfig(logConfig *structs.LogConfig) error
}
// SyslogCollector is a LogCollector which starts a syslog server and does
// rotation to incoming stream
type SyslogCollector struct {
addr net.Addr
logConfig *structs.LogConfig
ctx *LogCollectorContext
lro *FileRotator
lre *FileRotator
server *SyslogServer
syslogChan chan *SyslogMessage
taskDir string
logger *log.Logger
}
// NewSyslogCollector returns an implementation of the SyslogCollector
func NewSyslogCollector(logger *log.Logger) *SyslogCollector {
return &SyslogCollector{logger: logger, syslogChan: make(chan *SyslogMessage, 2048)}
}
// LaunchCollector launches a new syslog server and starts writing log lines to
// files and rotates them
func (s *SyslogCollector) LaunchCollector(ctx *LogCollectorContext) (*SyslogCollectorState, error) {
l, err := s.getListener(ctx.PortLowerBound, ctx.PortUpperBound)
if err != nil {
return nil, err
}
s.logger.Printf("[DEBUG] sylog-server: launching syslog server on addr: %v", l.Addr().String())
s.ctx = ctx
// configuring the task dir
if err := s.configureTaskDir(); err != nil {
return nil, err
}
s.server = NewSyslogServer(l, s.syslogChan, s.logger)
go s.server.Start()
logFileSize := int64(ctx.LogConfig.MaxFileSizeMB * 1024 * 1024)
lro, err := NewFileRotator(ctx.AllocDir.LogDir(), fmt.Sprintf("%v.stdout", ctx.TaskName),
ctx.LogConfig.MaxFiles, logFileSize, s.logger)
if err != nil {
return nil, err
}
s.lro = lro
lre, err := NewFileRotator(ctx.AllocDir.LogDir(), fmt.Sprintf("%v.stderr", ctx.TaskName),
ctx.LogConfig.MaxFiles, logFileSize, s.logger)
if err != nil {
return nil, err
}
s.lre = lre
go s.collectLogs(lre, lro)
syslogAddr := fmt.Sprintf("%s://%s", l.Addr().Network(), l.Addr().String())
return &SyslogCollectorState{Addr: syslogAddr}, nil
}
func (s *SyslogCollector) collectLogs(we io.Writer, wo io.Writer) {
for logParts := range s.syslogChan {
// If the severity of the log line is err then we write to stderr
// otherwise all messages go to stdout
if logParts.Severity == syslog.LOG_ERR {
s.lre.Write(logParts.Message)
s.lre.Write([]byte{'\n'})
} else {
s.lro.Write(logParts.Message)
s.lro.Write([]byte{'\n'})
}
}
}
// Exit kills the syslog server
func (s *SyslogCollector) Exit() error {
s.server.Shutdown()
s.lre.Close()
s.lro.Close()
return nil
}
// UpdateLogConfig updates the log configuration
func (s *SyslogCollector) UpdateLogConfig(logConfig *structs.LogConfig) error {
s.ctx.LogConfig = logConfig
if s.lro == nil {
return fmt.Errorf("log rotator for stdout doesn't exist")
}
s.lro.MaxFiles = logConfig.MaxFiles
s.lro.FileSize = int64(logConfig.MaxFileSizeMB * 1024 * 1024)
if s.lre == nil {
return fmt.Errorf("log rotator for stderr doesn't exist")
}
s.lre.MaxFiles = logConfig.MaxFiles
s.lre.FileSize = int64(logConfig.MaxFileSizeMB * 1024 * 1024)
return nil
}
// configureTaskDir sets the task dir in the SyslogCollector
func (s *SyslogCollector) configureTaskDir() error {
taskDir, ok := s.ctx.AllocDir.TaskDirs[s.ctx.TaskName]
if !ok {
return fmt.Errorf("couldn't find task directory for task %v", s.ctx.TaskName)
}
s.taskDir = taskDir
return nil
}
// getFreePort returns a free port ready to be listened on between upper and
// lower bounds
func (s *SyslogCollector) getListener(lowerBound uint, upperBound uint) (net.Listener, error) {
if runtime.GOOS == "windows" {
return s.listenerTCP(lowerBound, upperBound)
}
return s.listenerUnix()
}
// listenerTCP creates a TCP listener using an unused port between an upper and
// lower bound
func (s *SyslogCollector) listenerTCP(lowerBound uint, upperBound uint) (net.Listener, error) {
for i := lowerBound; i <= upperBound; i++ {
addr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("localhost:%v", i))
if err != nil {
return nil, err
}
l, err := net.ListenTCP("tcp", addr)
if err != nil {
continue
}
return l, nil
}
return nil, fmt.Errorf("No free port found")
}
// listenerUnix creates a Unix domain socket
func (s *SyslogCollector) listenerUnix() (net.Listener, error) {
f, err := ioutil.TempFile("", "plugin")
if err != nil {
return nil, err
}
path := f.Name()
if err := f.Close(); err != nil {
return nil, err
}
if err := os.Remove(path); err != nil {
return nil, err
}
return net.Listen("unix", path)
}

View File

@@ -0,0 +1,51 @@
package driver
import (
"io"
"log"
"net"
"github.com/hashicorp/go-plugin"
)
var HandshakeConfig = plugin.HandshakeConfig{
ProtocolVersion: 1,
MagicCookieKey: "NOMAD_PLUGIN_MAGIC_COOKIE",
MagicCookieValue: "e4327c2e01eabfd75a8a67adb114fb34a757d57eee7728d857a8cec6e91a7255",
}
func GetPluginMap(w io.Writer) map[string]plugin.Plugin {
e := new(ExecutorPlugin)
e.logger = log.New(w, "", log.LstdFlags)
s := new(SyslogCollectorPlugin)
s.logger = log.New(w, "", log.LstdFlags)
return map[string]plugin.Plugin{
"executor": e,
"syslogcollector": s,
}
}
// ExecutorReattachConfig is the config that we seralize and de-serialize and
// store in disk
type PluginReattachConfig struct {
Pid int
AddrNet string
AddrName string
}
// PluginConfig returns a config from an ExecutorReattachConfig
func (c *PluginReattachConfig) PluginConfig() *plugin.ReattachConfig {
var addr net.Addr
switch c.AddrNet {
case "unix", "unixgram", "unixpacket":
addr, _ = net.ResolveUnixAddr(c.AddrNet, c.AddrName)
case "tcp", "tcp4", "tcp6":
addr, _ = net.ResolveTCPAddr(c.AddrNet, c.AddrName)
}
return &plugin.ReattachConfig{Pid: c.Pid, Addr: addr}
}
func NewPluginReattachConfig(c *plugin.ReattachConfig) *PluginReattachConfig {
return &PluginReattachConfig{Pid: c.Pid, AddrNet: c.Addr.Network(), AddrName: c.Addr.String()}
}

412
vendor/github.com/hashicorp/nomad/client/driver/qemu.go generated vendored Normal file
View File

@@ -0,0 +1,412 @@
package driver
import (
"encoding/json"
"fmt"
"log"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"strings"
"time"
"github.com/hashicorp/go-plugin"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/client/driver/executor"
dstructs "github.com/hashicorp/nomad/client/driver/structs"
"github.com/hashicorp/nomad/client/fingerprint"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/helper/discover"
"github.com/hashicorp/nomad/helper/fields"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/mitchellh/mapstructure"
)
var (
reQemuVersion = regexp.MustCompile(`version (\d[\.\d+]+)`)
)
const (
// The key populated in Node Attributes to indicate presence of the Qemu
// driver
qemuDriverAttr = "driver.qemu"
)
// QemuDriver is a driver for running images via Qemu
// We attempt to chose sane defaults for now, with more configuration available
// planned in the future
type QemuDriver struct {
DriverContext
fingerprint.StaticFingerprinter
}
type QemuDriverConfig struct {
ImagePath string `mapstructure:"image_path"`
Accelerator string `mapstructure:"accelerator"`
PortMap []map[string]int `mapstructure:"port_map"` // A map of host port labels and to guest ports.
Args []string `mapstructure:"args"` // extra arguments to qemu executable
}
// qemuHandle is returned from Start/Open as a handle to the PID
type qemuHandle struct {
pluginClient *plugin.Client
userPid int
executor executor.Executor
allocDir *allocdir.AllocDir
killTimeout time.Duration
maxKillTimeout time.Duration
logger *log.Logger
version string
waitCh chan *dstructs.WaitResult
doneCh chan struct{}
}
// NewQemuDriver is used to create a new exec driver
func NewQemuDriver(ctx *DriverContext) Driver {
return &QemuDriver{DriverContext: *ctx}
}
// Validate is used to validate the driver configuration
func (d *QemuDriver) Validate(config map[string]interface{}) error {
fd := &fields.FieldData{
Raw: config,
Schema: map[string]*fields.FieldSchema{
"image_path": &fields.FieldSchema{
Type: fields.TypeString,
Required: true,
},
"accelerator": &fields.FieldSchema{
Type: fields.TypeString,
},
"port_map": &fields.FieldSchema{
Type: fields.TypeArray,
},
"args": &fields.FieldSchema{
Type: fields.TypeArray,
},
},
}
if err := fd.Validate(); err != nil {
return err
}
return nil
}
func (d *QemuDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
// Get the current status so that we can log any debug messages only if the
// state changes
_, currentlyEnabled := node.Attributes[qemuDriverAttr]
bin := "qemu-system-x86_64"
if runtime.GOOS == "windows" {
// On windows, the "qemu-system-x86_64" command does not respond to the
// version flag.
bin = "qemu-img"
}
outBytes, err := exec.Command(bin, "--version").Output()
if err != nil {
delete(node.Attributes, qemuDriverAttr)
return false, nil
}
out := strings.TrimSpace(string(outBytes))
matches := reQemuVersion.FindStringSubmatch(out)
if len(matches) != 2 {
delete(node.Attributes, qemuDriverAttr)
return false, fmt.Errorf("Unable to parse Qemu version string: %#v", matches)
}
if !currentlyEnabled {
d.logger.Printf("[DEBUG] driver.qemu: enabling driver")
}
node.Attributes[qemuDriverAttr] = "1"
node.Attributes["driver.qemu.version"] = matches[1]
return true, nil
}
// Run an existing Qemu image. Start() will pull down an existing, valid Qemu
// image and save it to the Drivers Allocation Dir
func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
var driverConfig QemuDriverConfig
if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
return nil, err
}
if len(driverConfig.PortMap) > 1 {
return nil, fmt.Errorf("Only one port_map block is allowed in the qemu driver config")
}
// Get the image source
vmPath := driverConfig.ImagePath
if vmPath == "" {
return nil, fmt.Errorf("image_path must be set")
}
vmID := filepath.Base(vmPath)
// Get the tasks local directory.
taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName]
if !ok {
return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName)
}
// Parse configuration arguments
// Create the base arguments
accelerator := "tcg"
if driverConfig.Accelerator != "" {
accelerator = driverConfig.Accelerator
}
// TODO: Check a lower bounds, e.g. the default 128 of Qemu
mem := fmt.Sprintf("%dM", task.Resources.MemoryMB)
absPath, err := GetAbsolutePath("qemu-system-x86_64")
if err != nil {
return nil, err
}
args := []string{
absPath,
"-machine", "type=pc,accel=" + accelerator,
"-name", vmID,
"-m", mem,
"-drive", "file=" + vmPath,
"-nographic",
}
// Add pass through arguments to qemu executable. A user can specify
// these arguments in driver task configuration. These arguments are
// passed directly to the qemu driver as command line options.
// For example, args = [ "-nodefconfig", "-nodefaults" ]
// This will allow a VM with embedded configuration to boot successfully.
args = append(args, driverConfig.Args...)
// Check the Resources required Networks to add port mappings. If no resources
// are required, we assume the VM is a purely compute job and does not require
// the outside world to be able to reach it. VMs ran without port mappings can
// still reach out to the world, but without port mappings it is effectively
// firewalled
protocols := []string{"udp", "tcp"}
if len(task.Resources.Networks) > 0 && len(driverConfig.PortMap) == 1 {
// Loop through the port map and construct the hostfwd string, to map
// reserved ports to the ports listenting in the VM
// Ex: hostfwd=tcp::22000-:22,hostfwd=tcp::80-:8080
var forwarding []string
taskPorts := task.Resources.Networks[0].MapLabelToValues(nil)
for label, guest := range driverConfig.PortMap[0] {
host, ok := taskPorts[label]
if !ok {
return nil, fmt.Errorf("Unknown port label %q", label)
}
for _, p := range protocols {
forwarding = append(forwarding, fmt.Sprintf("hostfwd=%s::%d-:%d", p, host, guest))
}
}
if len(forwarding) != 0 {
args = append(args,
"-netdev",
fmt.Sprintf("user,id=user.0,%s", strings.Join(forwarding, ",")),
"-device", "virtio-net,netdev=user.0",
)
}
}
// If using KVM, add optimization args
if accelerator == "kvm" {
args = append(args,
"-enable-kvm",
"-cpu", "host",
// Do we have cores information available to the Driver?
// "-smp", fmt.Sprintf("%d", cores),
)
}
d.logger.Printf("[DEBUG] Starting QemuVM command: %q", strings.Join(args, " "))
bin, err := discover.NomadExecutable()
if err != nil {
return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
}
pluginLogFile := filepath.Join(taskDir, fmt.Sprintf("%s-executor.out", task.Name))
pluginConfig := &plugin.ClientConfig{
Cmd: exec.Command(bin, "executor", pluginLogFile),
}
exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
if err != nil {
return nil, err
}
executorCtx := &executor.ExecutorContext{
TaskEnv: d.taskEnv,
Driver: "qemu",
AllocDir: ctx.AllocDir,
AllocID: ctx.AllocID,
Task: task,
}
ps, err := exec.LaunchCmd(&executor.ExecCommand{
Cmd: args[0],
Args: args[1:],
User: task.User,
}, executorCtx)
if err != nil {
pluginClient.Kill()
return nil, err
}
d.logger.Printf("[INFO] Started new QemuVM: %s", vmID)
// Create and Return Handle
maxKill := d.DriverContext.config.MaxKillTimeout
h := &qemuHandle{
pluginClient: pluginClient,
executor: exec,
userPid: ps.Pid,
allocDir: ctx.AllocDir,
killTimeout: GetKillTimeout(task.KillTimeout, maxKill),
maxKillTimeout: maxKill,
version: d.config.Version,
logger: d.logger,
doneCh: make(chan struct{}),
waitCh: make(chan *dstructs.WaitResult, 1),
}
if err := h.executor.SyncServices(consulContext(d.config, "")); err != nil {
h.logger.Printf("[ERR] driver.qemu: error registering services for task: %q: %v", task.Name, err)
}
go h.run()
return h, nil
}
type qemuId struct {
Version string
KillTimeout time.Duration
MaxKillTimeout time.Duration
UserPid int
PluginConfig *PluginReattachConfig
AllocDir *allocdir.AllocDir
}
func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
id := &qemuId{}
if err := json.Unmarshal([]byte(handleID), id); err != nil {
return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
}
pluginConfig := &plugin.ClientConfig{
Reattach: id.PluginConfig.PluginConfig(),
}
exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
if err != nil {
d.logger.Println("[ERR] driver.qemu: error connecting to plugin so destroying plugin pid and user pid")
if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
d.logger.Printf("[ERR] driver.qemu: error destroying plugin and userpid: %v", e)
}
return nil, fmt.Errorf("error connecting to plugin: %v", err)
}
ver, _ := exec.Version()
d.logger.Printf("[DEBUG] driver.qemu: version of executor: %v", ver.Version)
// Return a driver handle
h := &qemuHandle{
pluginClient: pluginClient,
executor: exec,
userPid: id.UserPid,
allocDir: id.AllocDir,
logger: d.logger,
killTimeout: id.KillTimeout,
maxKillTimeout: id.MaxKillTimeout,
version: id.Version,
doneCh: make(chan struct{}),
waitCh: make(chan *dstructs.WaitResult, 1),
}
if err := h.executor.SyncServices(consulContext(d.config, "")); err != nil {
h.logger.Printf("[ERR] driver.qemu: error registering services: %v", err)
}
go h.run()
return h, nil
}
func (h *qemuHandle) ID() string {
id := qemuId{
Version: h.version,
KillTimeout: h.killTimeout,
MaxKillTimeout: h.maxKillTimeout,
PluginConfig: NewPluginReattachConfig(h.pluginClient.ReattachConfig()),
UserPid: h.userPid,
AllocDir: h.allocDir,
}
data, err := json.Marshal(id)
if err != nil {
h.logger.Printf("[ERR] driver.qemu: failed to marshal ID to JSON: %s", err)
}
return string(data)
}
func (h *qemuHandle) WaitCh() chan *dstructs.WaitResult {
return h.waitCh
}
func (h *qemuHandle) Update(task *structs.Task) error {
// Store the updated kill timeout.
h.killTimeout = GetKillTimeout(task.KillTimeout, h.maxKillTimeout)
h.executor.UpdateTask(task)
// Update is not possible
return nil
}
// TODO: allow a 'shutdown_command' that can be executed over a ssh connection
// to the VM
func (h *qemuHandle) Kill() error {
if err := h.executor.ShutDown(); err != nil {
if h.pluginClient.Exited() {
return nil
}
return fmt.Errorf("executor Shutdown failed: %v", err)
}
select {
case <-h.doneCh:
return nil
case <-time.After(h.killTimeout):
if h.pluginClient.Exited() {
return nil
}
if err := h.executor.Exit(); err != nil {
return fmt.Errorf("executor Exit failed: %v", err)
}
return nil
}
}
func (h *qemuHandle) Stats() (*cstructs.TaskResourceUsage, error) {
return h.executor.Stats()
}
func (h *qemuHandle) run() {
ps, err := h.executor.Wait()
if ps.ExitCode == 0 && err != nil {
if e := killProcess(h.userPid); e != nil {
h.logger.Printf("[ERR] driver.qemu: error killing user process: %v", e)
}
if e := h.allocDir.UnmountAll(); e != nil {
h.logger.Printf("[ERR] driver.qemu: unmounting dev,proc and alloc dirs failed: %v", e)
}
}
close(h.doneCh)
h.waitCh <- &dstructs.WaitResult{ExitCode: ps.ExitCode, Signal: ps.Signal, Err: err}
close(h.waitCh)
// Remove services
if err := h.executor.DeregisterServices(); err != nil {
h.logger.Printf("[ERR] driver.qemu: failed to deregister services: %v", err)
}
h.executor.Exit()
h.pluginClient.Kill()
}

View File

@@ -0,0 +1,307 @@
package driver
import (
"encoding/json"
"fmt"
"log"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/hashicorp/go-plugin"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/client/driver/executor"
dstructs "github.com/hashicorp/nomad/client/driver/structs"
"github.com/hashicorp/nomad/client/fingerprint"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/helper/discover"
"github.com/hashicorp/nomad/helper/fields"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/mitchellh/mapstructure"
)
const (
// The option that enables this driver in the Config.Options map.
rawExecConfigOption = "driver.raw_exec.enable"
// The key populated in Node Attributes to indicate presence of the Raw Exec
// driver
rawExecDriverAttr = "driver.raw_exec"
)
// The RawExecDriver is a privileged version of the exec driver. It provides no
// resource isolation and just fork/execs. The Exec driver should be preferred
// and this should only be used when explicitly needed.
type RawExecDriver struct {
DriverContext
fingerprint.StaticFingerprinter
}
// rawExecHandle is returned from Start/Open as a handle to the PID
type rawExecHandle struct {
version string
pluginClient *plugin.Client
userPid int
executor executor.Executor
killTimeout time.Duration
maxKillTimeout time.Duration
allocDir *allocdir.AllocDir
logger *log.Logger
waitCh chan *dstructs.WaitResult
doneCh chan struct{}
}
// NewRawExecDriver is used to create a new raw exec driver
func NewRawExecDriver(ctx *DriverContext) Driver {
return &RawExecDriver{DriverContext: *ctx}
}
// Validate is used to validate the driver configuration
func (d *RawExecDriver) Validate(config map[string]interface{}) error {
fd := &fields.FieldData{
Raw: config,
Schema: map[string]*fields.FieldSchema{
"command": &fields.FieldSchema{
Type: fields.TypeString,
Required: true,
},
"args": &fields.FieldSchema{
Type: fields.TypeArray,
},
},
}
if err := fd.Validate(); err != nil {
return err
}
return nil
}
func (d *RawExecDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
// Get the current status so that we can log any debug messages only if the
// state changes
_, currentlyEnabled := node.Attributes[rawExecDriverAttr]
// Check that the user has explicitly enabled this executor.
enabled := cfg.ReadBoolDefault(rawExecConfigOption, false)
if enabled {
if currentlyEnabled {
d.logger.Printf("[WARN] driver.raw_exec: raw exec is enabled. Only enable if needed")
}
node.Attributes[rawExecDriverAttr] = "1"
return true, nil
}
delete(node.Attributes, rawExecDriverAttr)
return false, nil
}
func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
var driverConfig ExecDriverConfig
if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
return nil, err
}
// Get the tasks local directory.
taskName := d.DriverContext.taskName
taskDir, ok := ctx.AllocDir.TaskDirs[taskName]
if !ok {
return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName)
}
// Get the command to be ran
command := driverConfig.Command
if err := validateCommand(command, "args"); err != nil {
return nil, err
}
// Set the host environment variables.
filter := strings.Split(d.config.ReadDefault("env.blacklist", config.DefaultEnvBlacklist), ",")
d.taskEnv.AppendHostEnvvars(filter)
bin, err := discover.NomadExecutable()
if err != nil {
return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
}
pluginLogFile := filepath.Join(taskDir, fmt.Sprintf("%s-executor.out", task.Name))
pluginConfig := &plugin.ClientConfig{
Cmd: exec.Command(bin, "executor", pluginLogFile),
}
exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
if err != nil {
return nil, err
}
executorCtx := &executor.ExecutorContext{
TaskEnv: d.taskEnv,
Driver: "raw_exec",
AllocDir: ctx.AllocDir,
AllocID: ctx.AllocID,
Task: task,
}
ps, err := exec.LaunchCmd(&executor.ExecCommand{
Cmd: command,
Args: driverConfig.Args,
User: task.User,
}, executorCtx)
if err != nil {
pluginClient.Kill()
return nil, err
}
d.logger.Printf("[DEBUG] driver.raw_exec: started process with pid: %v", ps.Pid)
// Return a driver handle
maxKill := d.DriverContext.config.MaxKillTimeout
h := &rawExecHandle{
pluginClient: pluginClient,
executor: exec,
userPid: ps.Pid,
killTimeout: GetKillTimeout(task.KillTimeout, maxKill),
maxKillTimeout: maxKill,
allocDir: ctx.AllocDir,
version: d.config.Version,
logger: d.logger,
doneCh: make(chan struct{}),
waitCh: make(chan *dstructs.WaitResult, 1),
}
if err := h.executor.SyncServices(consulContext(d.config, "")); err != nil {
h.logger.Printf("[ERR] driver.raw_exec: error registering services with consul for task: %q: %v", task.Name, err)
}
go h.run()
return h, nil
}
type rawExecId struct {
Version string
KillTimeout time.Duration
MaxKillTimeout time.Duration
UserPid int
PluginConfig *PluginReattachConfig
AllocDir *allocdir.AllocDir
}
func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
id := &rawExecId{}
if err := json.Unmarshal([]byte(handleID), id); err != nil {
return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
}
pluginConfig := &plugin.ClientConfig{
Reattach: id.PluginConfig.PluginConfig(),
}
exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
if err != nil {
d.logger.Println("[ERR] driver.raw_exec: error connecting to plugin so destroying plugin pid and user pid")
if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
d.logger.Printf("[ERR] driver.raw_exec: error destroying plugin and userpid: %v", e)
}
return nil, fmt.Errorf("error connecting to plugin: %v", err)
}
ver, _ := exec.Version()
d.logger.Printf("[DEBUG] driver.raw_exec: version of executor: %v", ver.Version)
// Return a driver handle
h := &rawExecHandle{
pluginClient: pluginClient,
executor: exec,
userPid: id.UserPid,
logger: d.logger,
killTimeout: id.KillTimeout,
maxKillTimeout: id.MaxKillTimeout,
allocDir: id.AllocDir,
version: id.Version,
doneCh: make(chan struct{}),
waitCh: make(chan *dstructs.WaitResult, 1),
}
if err := h.executor.SyncServices(consulContext(d.config, "")); err != nil {
h.logger.Printf("[ERR] driver.raw_exec: error registering services with consul: %v", err)
}
go h.run()
return h, nil
}
func (h *rawExecHandle) ID() string {
id := rawExecId{
Version: h.version,
KillTimeout: h.killTimeout,
MaxKillTimeout: h.maxKillTimeout,
PluginConfig: NewPluginReattachConfig(h.pluginClient.ReattachConfig()),
UserPid: h.userPid,
AllocDir: h.allocDir,
}
data, err := json.Marshal(id)
if err != nil {
h.logger.Printf("[ERR] driver.raw_exec: failed to marshal ID to JSON: %s", err)
}
return string(data)
}
func (h *rawExecHandle) WaitCh() chan *dstructs.WaitResult {
return h.waitCh
}
func (h *rawExecHandle) Update(task *structs.Task) error {
// Store the updated kill timeout.
h.killTimeout = GetKillTimeout(task.KillTimeout, h.maxKillTimeout)
h.executor.UpdateTask(task)
// Update is not possible
return nil
}
func (h *rawExecHandle) Kill() error {
if err := h.executor.ShutDown(); err != nil {
if h.pluginClient.Exited() {
return nil
}
return fmt.Errorf("executor Shutdown failed: %v", err)
}
select {
case <-h.doneCh:
return nil
case <-time.After(h.killTimeout):
if h.pluginClient.Exited() {
return nil
}
if err := h.executor.Exit(); err != nil {
return fmt.Errorf("executor Exit failed: %v", err)
}
return nil
}
}
func (h *rawExecHandle) Stats() (*cstructs.TaskResourceUsage, error) {
return h.executor.Stats()
}
func (h *rawExecHandle) run() {
ps, err := h.executor.Wait()
close(h.doneCh)
if ps.ExitCode == 0 && err != nil {
if e := killProcess(h.userPid); e != nil {
h.logger.Printf("[ERR] driver.raw_exec: error killing user process: %v", e)
}
if e := h.allocDir.UnmountAll(); e != nil {
h.logger.Printf("[ERR] driver.raw_exec: unmounting dev,proc and alloc dirs failed: %v", e)
}
}
h.waitCh <- &dstructs.WaitResult{ExitCode: ps.ExitCode, Signal: ps.Signal, Err: err}
close(h.waitCh)
// Remove services
if err := h.executor.DeregisterServices(); err != nil {
h.logger.Printf("[ERR] driver.raw_exec: failed to deregister services: %v", err)
}
if err := h.executor.Exit(); err != nil {
h.logger.Printf("[ERR] driver.raw_exec: error killing executor: %v", err)
}
h.pluginClient.Kill()
}

436
vendor/github.com/hashicorp/nomad/client/driver/rkt.go generated vendored Normal file
View File

@@ -0,0 +1,436 @@
package driver
import (
"bytes"
"encoding/json"
"fmt"
"log"
"net"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"strings"
"syscall"
"time"
"github.com/hashicorp/go-plugin"
"github.com/hashicorp/go-version"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/client/driver/executor"
dstructs "github.com/hashicorp/nomad/client/driver/structs"
"github.com/hashicorp/nomad/client/fingerprint"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/helper/discover"
"github.com/hashicorp/nomad/helper/fields"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/mitchellh/mapstructure"
)
var (
reRktVersion = regexp.MustCompile(`rkt [vV]ersion[:]? (\d[.\d]+)`)
reAppcVersion = regexp.MustCompile(`appc [vV]ersion[:]? (\d[.\d]+)`)
)
const (
// minRktVersion is the earliest supported version of rkt. rkt added support
// for CPU and memory isolators in 0.14.0. We cannot support an earlier
// version to maintain an uniform interface across all drivers
minRktVersion = "0.14.0"
// The key populated in the Node Attributes to indicate the presence of the
// Rkt driver
rktDriverAttr = "driver.rkt"
)
// RktDriver is a driver for running images via Rkt
// We attempt to chose sane defaults for now, with more configuration available
// planned in the future
type RktDriver struct {
DriverContext
fingerprint.StaticFingerprinter
}
type RktDriverConfig struct {
ImageName string `mapstructure:"image"`
Command string `mapstructure:"command"`
Args []string `mapstructure:"args"`
TrustPrefix string `mapstructure:"trust_prefix"`
DNSServers []string `mapstructure:"dns_servers"` // DNS Server for containers
DNSSearchDomains []string `mapstructure:"dns_search_domains"` // DNS Search domains for containers
Debug bool `mapstructure:"debug"` // Enable debug option for rkt command
}
// rktHandle is returned from Start/Open as a handle to the PID
type rktHandle struct {
pluginClient *plugin.Client
executorPid int
executor executor.Executor
allocDir *allocdir.AllocDir
logger *log.Logger
killTimeout time.Duration
maxKillTimeout time.Duration
waitCh chan *dstructs.WaitResult
doneCh chan struct{}
}
// rktPID is a struct to map the pid running the process to the vm image on
// disk
type rktPID struct {
PluginConfig *PluginReattachConfig
AllocDir *allocdir.AllocDir
ExecutorPid int
KillTimeout time.Duration
MaxKillTimeout time.Duration
}
// NewRktDriver is used to create a new exec driver
func NewRktDriver(ctx *DriverContext) Driver {
return &RktDriver{DriverContext: *ctx}
}
// Validate is used to validate the driver configuration
func (d *RktDriver) Validate(config map[string]interface{}) error {
fd := &fields.FieldData{
Raw: config,
Schema: map[string]*fields.FieldSchema{
"image": &fields.FieldSchema{
Type: fields.TypeString,
Required: true,
},
"command": &fields.FieldSchema{
Type: fields.TypeString,
},
"args": &fields.FieldSchema{
Type: fields.TypeArray,
},
"trust_prefix": &fields.FieldSchema{
Type: fields.TypeString,
},
"dns_servers": &fields.FieldSchema{
Type: fields.TypeArray,
},
"dns_search_domains": &fields.FieldSchema{
Type: fields.TypeArray,
},
"debug": &fields.FieldSchema{
Type: fields.TypeBool,
},
},
}
if err := fd.Validate(); err != nil {
return err
}
return nil
}
func (d *RktDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
// Get the current status so that we can log any debug messages only if the
// state changes
_, currentlyEnabled := node.Attributes[rktDriverAttr]
// Only enable if we are root when running on non-windows systems.
if runtime.GOOS != "windows" && syscall.Geteuid() != 0 {
if currentlyEnabled {
d.logger.Printf("[DEBUG] driver.rkt: must run as root user, disabling")
}
delete(node.Attributes, rktDriverAttr)
return false, nil
}
outBytes, err := exec.Command("rkt", "version").Output()
if err != nil {
delete(node.Attributes, rktDriverAttr)
return false, nil
}
out := strings.TrimSpace(string(outBytes))
rktMatches := reRktVersion.FindStringSubmatch(out)
appcMatches := reAppcVersion.FindStringSubmatch(out)
if len(rktMatches) != 2 || len(appcMatches) != 2 {
delete(node.Attributes, rktDriverAttr)
return false, fmt.Errorf("Unable to parse Rkt version string: %#v", rktMatches)
}
node.Attributes[rktDriverAttr] = "1"
node.Attributes["driver.rkt.version"] = rktMatches[1]
node.Attributes["driver.rkt.appc.version"] = appcMatches[1]
minVersion, _ := version.NewVersion(minRktVersion)
currentVersion, _ := version.NewVersion(node.Attributes["driver.rkt.version"])
if currentVersion.LessThan(minVersion) {
// Do not allow rkt < 0.14.0
d.logger.Printf("[WARN] driver.rkt: please upgrade rkt to a version >= %s", minVersion)
node.Attributes[rktDriverAttr] = "0"
}
return true, nil
}
// Run an existing Rkt image.
func (d *RktDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
var driverConfig RktDriverConfig
if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
return nil, err
}
// ACI image
img := driverConfig.ImageName
// Get the tasks local directory.
taskName := d.DriverContext.taskName
taskDir, ok := ctx.AllocDir.TaskDirs[taskName]
if !ok {
return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName)
}
// Build the command.
var cmdArgs []string
// Add debug option to rkt command.
debug := driverConfig.Debug
// Add the given trust prefix
trustPrefix := driverConfig.TrustPrefix
insecure := false
if trustPrefix != "" {
var outBuf, errBuf bytes.Buffer
cmd := exec.Command("rkt", "trust", "--skip-fingerprint-review=true", fmt.Sprintf("--prefix=%s", trustPrefix), fmt.Sprintf("--debug=%t", debug))
cmd.Stdout = &outBuf
cmd.Stderr = &errBuf
if err := cmd.Run(); err != nil {
return nil, fmt.Errorf("Error running rkt trust: %s\n\nOutput: %s\n\nError: %s",
err, outBuf.String(), errBuf.String())
}
d.logger.Printf("[DEBUG] driver.rkt: added trust prefix: %q", trustPrefix)
} else {
// Disble signature verification if the trust command was not run.
insecure = true
}
cmdArgs = append(cmdArgs, "run")
cmdArgs = append(cmdArgs, fmt.Sprintf("--volume=%s,kind=host,source=%s", task.Name, ctx.AllocDir.SharedDir))
cmdArgs = append(cmdArgs, fmt.Sprintf("--mount=volume=%s,target=%s", task.Name, ctx.AllocDir.SharedDir))
cmdArgs = append(cmdArgs, img)
if insecure == true {
cmdArgs = append(cmdArgs, "--insecure-options=all")
}
cmdArgs = append(cmdArgs, fmt.Sprintf("--debug=%t", debug))
// Inject environment variables
for k, v := range d.taskEnv.EnvMap() {
cmdArgs = append(cmdArgs, fmt.Sprintf("--set-env=%v=%v", k, v))
}
// Check if the user has overridden the exec command.
if driverConfig.Command != "" {
cmdArgs = append(cmdArgs, fmt.Sprintf("--exec=%v", driverConfig.Command))
}
// Add memory isolator
cmdArgs = append(cmdArgs, fmt.Sprintf("--memory=%vM", int64(task.Resources.MemoryMB)))
// Add CPU isolator
cmdArgs = append(cmdArgs, fmt.Sprintf("--cpu=%vm", int64(task.Resources.CPU)))
// Add DNS servers
for _, ip := range driverConfig.DNSServers {
if err := net.ParseIP(ip); err == nil {
msg := fmt.Errorf("invalid ip address for container dns server %q", ip)
d.logger.Printf("[DEBUG] driver.rkt: %v", msg)
return nil, msg
} else {
cmdArgs = append(cmdArgs, fmt.Sprintf("--dns=%s", ip))
}
}
// set DNS search domains
for _, domain := range driverConfig.DNSSearchDomains {
cmdArgs = append(cmdArgs, fmt.Sprintf("--dns-search=%s", domain))
}
// Add user passed arguments.
if len(driverConfig.Args) != 0 {
parsed := d.taskEnv.ParseAndReplace(driverConfig.Args)
// Need to start arguments with "--"
if len(parsed) > 0 {
cmdArgs = append(cmdArgs, "--")
}
for _, arg := range parsed {
cmdArgs = append(cmdArgs, fmt.Sprintf("%v", arg))
}
}
// Set the host environment variables.
filter := strings.Split(d.config.ReadDefault("env.blacklist", config.DefaultEnvBlacklist), ",")
d.taskEnv.AppendHostEnvvars(filter)
bin, err := discover.NomadExecutable()
if err != nil {
return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
}
pluginLogFile := filepath.Join(taskDir, fmt.Sprintf("%s-executor.out", task.Name))
pluginConfig := &plugin.ClientConfig{
Cmd: exec.Command(bin, "executor", pluginLogFile),
}
execIntf, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
if err != nil {
return nil, err
}
executorCtx := &executor.ExecutorContext{
TaskEnv: d.taskEnv,
Driver: "rkt",
AllocDir: ctx.AllocDir,
AllocID: ctx.AllocID,
Task: task,
}
absPath, err := GetAbsolutePath("rkt")
if err != nil {
return nil, err
}
ps, err := execIntf.LaunchCmd(&executor.ExecCommand{
Cmd: absPath,
Args: cmdArgs,
User: task.User,
}, executorCtx)
if err != nil {
pluginClient.Kill()
return nil, err
}
d.logger.Printf("[DEBUG] driver.rkt: started ACI %q with: %v", img, cmdArgs)
maxKill := d.DriverContext.config.MaxKillTimeout
h := &rktHandle{
pluginClient: pluginClient,
executor: execIntf,
executorPid: ps.Pid,
allocDir: ctx.AllocDir,
logger: d.logger,
killTimeout: GetKillTimeout(task.KillTimeout, maxKill),
maxKillTimeout: maxKill,
doneCh: make(chan struct{}),
waitCh: make(chan *dstructs.WaitResult, 1),
}
if err := h.executor.SyncServices(consulContext(d.config, "")); err != nil {
h.logger.Printf("[ERR] driver.rkt: error registering services for task: %q: %v", task.Name, err)
}
go h.run()
return h, nil
}
func (d *RktDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
// Parse the handle
pidBytes := []byte(strings.TrimPrefix(handleID, "Rkt:"))
id := &rktPID{}
if err := json.Unmarshal(pidBytes, id); err != nil {
return nil, fmt.Errorf("failed to parse Rkt handle '%s': %v", handleID, err)
}
pluginConfig := &plugin.ClientConfig{
Reattach: id.PluginConfig.PluginConfig(),
}
exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
if err != nil {
d.logger.Println("[ERROR] driver.rkt: error connecting to plugin so destroying plugin pid and user pid")
if e := destroyPlugin(id.PluginConfig.Pid, id.ExecutorPid); e != nil {
d.logger.Printf("[ERROR] driver.rkt: error destroying plugin and executor pid: %v", e)
}
return nil, fmt.Errorf("error connecting to plugin: %v", err)
}
ver, _ := exec.Version()
d.logger.Printf("[DEBUG] driver.rkt: version of executor: %v", ver.Version)
// Return a driver handle
h := &rktHandle{
pluginClient: pluginClient,
executorPid: id.ExecutorPid,
allocDir: id.AllocDir,
executor: exec,
logger: d.logger,
killTimeout: id.KillTimeout,
maxKillTimeout: id.MaxKillTimeout,
doneCh: make(chan struct{}),
waitCh: make(chan *dstructs.WaitResult, 1),
}
if err := h.executor.SyncServices(consulContext(d.config, "")); err != nil {
h.logger.Printf("[ERR] driver.rkt: error registering services: %v", err)
}
go h.run()
return h, nil
}
func (h *rktHandle) ID() string {
// Return a handle to the PID
pid := &rktPID{
PluginConfig: NewPluginReattachConfig(h.pluginClient.ReattachConfig()),
KillTimeout: h.killTimeout,
MaxKillTimeout: h.maxKillTimeout,
ExecutorPid: h.executorPid,
AllocDir: h.allocDir,
}
data, err := json.Marshal(pid)
if err != nil {
h.logger.Printf("[ERR] driver.rkt: failed to marshal rkt PID to JSON: %s", err)
}
return fmt.Sprintf("Rkt:%s", string(data))
}
func (h *rktHandle) WaitCh() chan *dstructs.WaitResult {
return h.waitCh
}
func (h *rktHandle) Update(task *structs.Task) error {
// Store the updated kill timeout.
h.killTimeout = GetKillTimeout(task.KillTimeout, h.maxKillTimeout)
h.executor.UpdateTask(task)
// Update is not possible
return nil
}
// Kill is used to terminate the task. We send an Interrupt
// and then provide a 5 second grace period before doing a Kill.
func (h *rktHandle) Kill() error {
h.executor.ShutDown()
select {
case <-h.doneCh:
return nil
case <-time.After(h.killTimeout):
return h.executor.Exit()
}
}
func (h *rktHandle) Stats() (*cstructs.TaskResourceUsage, error) {
return nil, fmt.Errorf("stats not implemented for rkt")
}
func (h *rktHandle) run() {
ps, err := h.executor.Wait()
close(h.doneCh)
if ps.ExitCode == 0 && err != nil {
if e := killProcess(h.executorPid); e != nil {
h.logger.Printf("[ERROR] driver.rkt: error killing user process: %v", e)
}
if e := h.allocDir.UnmountAll(); e != nil {
h.logger.Printf("[ERROR] driver.rkt: unmounting dev,proc and alloc dirs failed: %v", e)
}
}
h.waitCh <- dstructs.NewWaitResult(ps.ExitCode, 0, err)
close(h.waitCh)
// Remove services
if err := h.executor.DeregisterServices(); err != nil {
h.logger.Printf("[ERR] driver.rkt: failed to deregister services: %v", err)
}
if err := h.executor.Exit(); err != nil {
h.logger.Printf("[ERR] driver.rkt: error killing executor: %v", err)
}
h.pluginClient.Kill()
}

View File

@@ -0,0 +1,77 @@
package structs
import (
"fmt"
"time"
)
const (
// The default user that the executor uses to run tasks
DefaultUnpriviledgedUser = "nobody"
// CheckBufSize is the size of the check output result
CheckBufSize = 4 * 1024
)
// WaitResult stores the result of a Wait operation.
type WaitResult struct {
ExitCode int
Signal int
Err error
}
func NewWaitResult(code, signal int, err error) *WaitResult {
return &WaitResult{
ExitCode: code,
Signal: signal,
Err: err,
}
}
func (r *WaitResult) Successful() bool {
return r.ExitCode == 0 && r.Signal == 0 && r.Err == nil
}
func (r *WaitResult) String() string {
return fmt.Sprintf("Wait returned exit code %v, signal %v, and error %v",
r.ExitCode, r.Signal, r.Err)
}
// RecoverableError wraps an error and marks whether it is recoverable and could
// be retried or it is fatal.
type RecoverableError struct {
Err error
Recoverable bool
}
// NewRecoverableError is used to wrap an error and mark it as recoverable or
// not.
func NewRecoverableError(e error, recoverable bool) *RecoverableError {
return &RecoverableError{
Err: e,
Recoverable: recoverable,
}
}
func (r *RecoverableError) Error() string {
return r.Err.Error()
}
// CheckResult encapsulates the result of a check
type CheckResult struct {
// ExitCode is the exit code of the check
ExitCode int
// Output is the output of the check script
Output string
// Timestamp is the time at which the check was executed
Timestamp time.Time
// Duration is the time it took the check to run
Duration time.Duration
// Err is the error that a check returned
Err error
}

View File

@@ -0,0 +1,12 @@
// +build darwin dragonfly freebsd netbsd openbsd solaris windows
package structs
// IsolationConfig has information about the isolation mechanism the executor
// uses to put resource constraints and isolation on the user process. The
// default implementation is empty. Platforms that support resource isolation
// (e.g. Linux's Cgroups) should build their own platform-specific copy. This
// information is transmitted via RPC so it is not permissable to change the
// API.
type IsolationConfig struct {
}

View File

@@ -0,0 +1,10 @@
package structs
import cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
// IsolationConfig has information about the isolation mechanism the executor
// uses to put resource constraints and isolation on the user process
type IsolationConfig struct {
Cgroup *cgroupConfig.Cgroup
CgroupPaths map[string]string
}

View File

@@ -0,0 +1,69 @@
package driver
import (
"log"
"net/rpc"
"github.com/hashicorp/go-plugin"
"github.com/hashicorp/nomad/client/driver/logging"
"github.com/hashicorp/nomad/nomad/structs"
)
type SyslogCollectorRPC struct {
client *rpc.Client
}
type LaunchCollectorArgs struct {
Ctx *logging.LogCollectorContext
}
func (e *SyslogCollectorRPC) LaunchCollector(ctx *logging.LogCollectorContext) (*logging.SyslogCollectorState, error) {
var ss *logging.SyslogCollectorState
err := e.client.Call("Plugin.LaunchCollector", LaunchCollectorArgs{Ctx: ctx}, &ss)
return ss, err
}
func (e *SyslogCollectorRPC) Exit() error {
return e.client.Call("Plugin.Exit", new(interface{}), new(interface{}))
}
func (e *SyslogCollectorRPC) UpdateLogConfig(logConfig *structs.LogConfig) error {
return e.client.Call("Plugin.UpdateLogConfig", logConfig, new(interface{}))
}
type SyslogCollectorRPCServer struct {
Impl logging.LogCollector
}
func (s *SyslogCollectorRPCServer) LaunchCollector(args LaunchCollectorArgs,
resp *logging.SyslogCollectorState) error {
ss, err := s.Impl.LaunchCollector(args.Ctx)
if ss != nil {
*resp = *ss
}
return err
}
func (s *SyslogCollectorRPCServer) Exit(args interface{}, resp *interface{}) error {
return s.Impl.Exit()
}
func (s *SyslogCollectorRPCServer) UpdateLogConfig(logConfig *structs.LogConfig, resp *interface{}) error {
return s.Impl.UpdateLogConfig(logConfig)
}
type SyslogCollectorPlugin struct {
logger *log.Logger
Impl *SyslogCollectorRPCServer
}
func (p *SyslogCollectorPlugin) Server(*plugin.MuxBroker) (interface{}, error) {
if p.Impl == nil {
p.Impl = &SyslogCollectorRPCServer{Impl: logging.NewSyslogCollector(p.logger)}
}
return p.Impl, nil
}
func (p *SyslogCollectorPlugin) Client(b *plugin.MuxBroker, c *rpc.Client) (interface{}, error) {
return &SyslogCollectorRPC{client: c}, nil
}

View File

@@ -0,0 +1,170 @@
package driver
import (
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/go-plugin"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/client/driver/executor"
"github.com/hashicorp/nomad/client/driver/logging"
cstructs "github.com/hashicorp/nomad/client/driver/structs"
"github.com/hashicorp/nomad/nomad/structs"
)
// createExecutor launches an executor plugin and returns an instance of the
// Executor interface
func createExecutor(config *plugin.ClientConfig, w io.Writer,
clientConfig *config.Config) (executor.Executor, *plugin.Client, error) {
config.HandshakeConfig = HandshakeConfig
config.Plugins = GetPluginMap(w)
config.MaxPort = clientConfig.ClientMaxPort
config.MinPort = clientConfig.ClientMinPort
// setting the setsid of the plugin process so that it doesn't get signals sent to
// the nomad client.
if config.Cmd != nil {
isolateCommand(config.Cmd)
}
executorClient := plugin.NewClient(config)
rpcClient, err := executorClient.Client()
if err != nil {
return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err)
}
raw, err := rpcClient.Dispense("executor")
if err != nil {
return nil, nil, fmt.Errorf("unable to dispense the executor plugin: %v", err)
}
executorPlugin := raw.(executor.Executor)
return executorPlugin, executorClient, nil
}
func createLogCollector(config *plugin.ClientConfig, w io.Writer,
clientConfig *config.Config) (logging.LogCollector, *plugin.Client, error) {
config.HandshakeConfig = HandshakeConfig
config.Plugins = GetPluginMap(w)
config.MaxPort = clientConfig.ClientMaxPort
config.MinPort = clientConfig.ClientMinPort
if config.Cmd != nil {
isolateCommand(config.Cmd)
}
syslogClient := plugin.NewClient(config)
rpcCLient, err := syslogClient.Client()
if err != nil {
return nil, nil, fmt.Errorf("error creating rpc client for syslog plugin: %v", err)
}
raw, err := rpcCLient.Dispense("syslogcollector")
if err != nil {
return nil, nil, fmt.Errorf("unable to dispense the syslog plugin: %v", err)
}
logCollector := raw.(logging.LogCollector)
return logCollector, syslogClient, nil
}
func consulContext(clientConfig *config.Config, containerID string) *executor.ConsulContext {
return &executor.ConsulContext{
ConsulConfig: clientConfig.ConsulConfig,
ContainerID: containerID,
DockerEndpoint: clientConfig.Read("docker.endpoint"),
TLSCa: clientConfig.Read("docker.tls.ca"),
TLSCert: clientConfig.Read("docker.tls.cert"),
TLSKey: clientConfig.Read("docker.tls.key"),
}
}
// killProcess kills a process with the given pid
func killProcess(pid int) error {
proc, err := os.FindProcess(pid)
if err != nil {
return err
}
return proc.Kill()
}
// destroyPlugin kills the plugin with the given pid and also kills the user
// process
func destroyPlugin(pluginPid int, userPid int) error {
var merr error
if err := killProcess(pluginPid); err != nil {
merr = multierror.Append(merr, err)
}
if err := killProcess(userPid); err != nil {
merr = multierror.Append(merr, err)
}
return merr
}
// validateCommand validates that the command only has a single value and
// returns a user friendly error message telling them to use the passed
// argField.
func validateCommand(command, argField string) error {
trimmed := strings.TrimSpace(command)
if len(trimmed) == 0 {
return fmt.Errorf("command empty: %q", command)
}
if len(trimmed) != len(command) {
return fmt.Errorf("command contains extra white space: %q", command)
}
split := strings.Split(trimmed, " ")
if len(split) != 1 {
return fmt.Errorf("command contained more than one input. Use %q field to pass arguments", argField)
}
return nil
}
// GetKillTimeout returns the kill timeout to use given the tasks desired kill
// timeout and the operator configured max kill timeout.
func GetKillTimeout(desired, max time.Duration) time.Duration {
maxNanos := max.Nanoseconds()
desiredNanos := desired.Nanoseconds()
// Make the minimum time between signal and kill, 1 second.
if desiredNanos <= 0 {
desiredNanos = (1 * time.Second).Nanoseconds()
}
// Protect against max not being set properly.
if maxNanos <= 0 {
maxNanos = (10 * time.Second).Nanoseconds()
}
if desiredNanos < maxNanos {
return time.Duration(desiredNanos)
}
return max
}
// GetAbsolutePath returns the absolute path of the passed binary by resolving
// it in the path and following symlinks.
func GetAbsolutePath(bin string) (string, error) {
lp, err := exec.LookPath(bin)
if err != nil {
return "", fmt.Errorf("failed to resolve path to %q executable: %v", bin, err)
}
return filepath.EvalSymlinks(lp)
}
// getExecutorUser returns the user of the task, defaulting to
// cstructs.DefaultUnprivilegedUser if none was given.
func getExecutorUser(task *structs.Task) string {
if task.User == "" {
return cstructs.DefaultUnpriviledgedUser
}
return task.User
}

View File

@@ -0,0 +1,18 @@
// +build darwin dragonfly freebsd linux netbsd openbsd solaris
package driver
import (
"os/exec"
"syscall"
)
// isolateCommand sets the setsid flag in exec.Cmd to true so that the process
// becomes the process leader in a new session and doesn't receive signals that
// are sent to the parent process.
func isolateCommand(cmd *exec.Cmd) {
if cmd.SysProcAttr == nil {
cmd.SysProcAttr = &syscall.SysProcAttr{}
}
cmd.SysProcAttr.Setsid = true
}

View File

@@ -0,0 +1,9 @@
package driver
import (
"os/exec"
)
// TODO Figure out if this is needed in Wondows
func isolateCommand(cmd *exec.Cmd) {
}

View File

@@ -0,0 +1,26 @@
package fingerprint
import (
"log"
"runtime"
client "github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/nomad/structs"
)
// ArchFingerprint is used to fingerprint the architecture
type ArchFingerprint struct {
StaticFingerprinter
logger *log.Logger
}
// NewArchFingerprint is used to create an OS fingerprint
func NewArchFingerprint(logger *log.Logger) Fingerprint {
f := &ArchFingerprint{logger: logger}
return f
}
func (f *ArchFingerprint) Fingerprint(config *client.Config, node *structs.Node) (bool, error) {
node.Attributes["arch"] = runtime.GOARCH
return true, nil
}

View File

@@ -0,0 +1,59 @@
// +build linux
package fingerprint
import (
"log"
"time"
"github.com/hashicorp/nomad/nomad/structs"
)
const (
cgroupAvailable = "available"
cgroupUnavailable = "unavailable"
interval = 15
)
type CGroupFingerprint struct {
logger *log.Logger
lastState string
mountPointDetector MountPointDetector
}
// An interface to isolate calls to the cgroup library
// This facilitates testing where we can implement
// fake mount points to test various code paths
type MountPointDetector interface {
MountPoint() (string, error)
}
// Implements the interface detector which calls the cgroups library directly
type DefaultMountPointDetector struct {
}
// Call out to the default cgroup library
func (b *DefaultMountPointDetector) MountPoint() (string, error) {
return FindCgroupMountpointDir()
}
// NewCGroupFingerprint returns a new cgroup fingerprinter
func NewCGroupFingerprint(logger *log.Logger) Fingerprint {
f := &CGroupFingerprint{
logger: logger,
lastState: cgroupUnavailable,
mountPointDetector: &DefaultMountPointDetector{},
}
return f
}
// clearCGroupAttributes clears any node attributes related to cgroups that might
// have been set in a previous fingerprint run.
func (f *CGroupFingerprint) clearCGroupAttributes(n *structs.Node) {
delete(n.Attributes, "unique.cgroup.mountpoint")
}
// Periodic determines the interval at which the periodic fingerprinter will run.
func (f *CGroupFingerprint) Periodic() (bool, time.Duration) {
return true, interval * time.Second
}

View File

@@ -0,0 +1,57 @@
// +build linux
package fingerprint
import (
"fmt"
client "github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/opencontainers/runc/libcontainer/cgroups"
)
// FindCgroupMountpointDir is used to find the cgroup mount point on a Linux
// system.
func FindCgroupMountpointDir() (string, error) {
mount, err := cgroups.FindCgroupMountpointDir()
if err != nil {
switch e := err.(type) {
case *cgroups.NotFoundError:
// It's okay if the mount point is not discovered
return "", nil
default:
// All other errors are passed back as is
return "", e
}
}
return mount, nil
}
// Fingerprint tries to find a valid cgroup moint point
func (f *CGroupFingerprint) Fingerprint(cfg *client.Config, node *structs.Node) (bool, error) {
mount, err := f.mountPointDetector.MountPoint()
if err != nil {
f.clearCGroupAttributes(node)
return false, fmt.Errorf("Failed to discover cgroup mount point: %s", err)
}
// Check if a cgroup mount point was found
if mount == "" {
// Clear any attributes from the previous fingerprint.
f.clearCGroupAttributes(node)
if f.lastState == cgroupAvailable {
f.logger.Printf("[INFO] fingerprint.cgroups: cgroups are unavailable")
}
f.lastState = cgroupUnavailable
return true, nil
}
node.Attributes["unique.cgroup.mountpoint"] = mount
if f.lastState == cgroupUnavailable {
f.logger.Printf("[INFO] fingerprint.cgroups: cgroups are available")
}
f.lastState = cgroupAvailable
return true, nil
}

View File

@@ -0,0 +1,100 @@
package fingerprint
import (
"fmt"
"log"
"strconv"
"time"
consul "github.com/hashicorp/consul/api"
client "github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/nomad/structs"
)
const (
consulAvailable = "available"
consulUnavailable = "unavailable"
)
// ConsulFingerprint is used to fingerprint the architecture
type ConsulFingerprint struct {
logger *log.Logger
client *consul.Client
lastState string
}
// NewConsulFingerprint is used to create an OS fingerprint
func NewConsulFingerprint(logger *log.Logger) Fingerprint {
return &ConsulFingerprint{logger: logger, lastState: consulUnavailable}
}
func (f *ConsulFingerprint) Fingerprint(config *client.Config, node *structs.Node) (bool, error) {
// Guard against uninitialized Links
if node.Links == nil {
node.Links = map[string]string{}
}
// Only create the client once to avoid creating too many connections to
// Consul.
if f.client == nil {
consulConfig, err := config.ConsulConfig.ApiConfig()
if err != nil {
return false, fmt.Errorf("Failed to initialize the Consul client config: %v", err)
}
f.client, err = consul.NewClient(consulConfig)
if err != nil {
return false, fmt.Errorf("Failed to initialize consul client: %s", err)
}
}
// We'll try to detect consul by making a query to to the agent's self API.
// If we can't hit this URL consul is probably not running on this machine.
info, err := f.client.Agent().Self()
if err != nil {
// Clear any attributes set by a previous fingerprint.
f.clearConsulAttributes(node)
// Print a message indicating that the Consul Agent is not available
// anymore
if f.lastState == consulAvailable {
f.logger.Printf("[INFO] fingerprint.consul: consul agent is unavailable")
}
f.lastState = consulUnavailable
return false, nil
}
node.Attributes["consul.server"] = strconv.FormatBool(info["Config"]["Server"].(bool))
node.Attributes["consul.version"] = info["Config"]["Version"].(string)
node.Attributes["consul.revision"] = info["Config"]["Revision"].(string)
node.Attributes["unique.consul.name"] = info["Config"]["NodeName"].(string)
node.Attributes["consul.datacenter"] = info["Config"]["Datacenter"].(string)
node.Links["consul"] = fmt.Sprintf("%s.%s",
node.Attributes["consul.datacenter"],
node.Attributes["unique.consul.name"])
// If the Consul Agent was previously unavailable print a message to
// indicate the Agent is available now
if f.lastState == consulUnavailable {
f.logger.Printf("[INFO] fingerprint.consul: consul agent is available")
}
f.lastState = consulAvailable
return true, nil
}
// clearConsulAttributes removes consul attributes and links from the passed
// Node.
func (f *ConsulFingerprint) clearConsulAttributes(n *structs.Node) {
delete(n.Attributes, "consul.server")
delete(n.Attributes, "consul.version")
delete(n.Attributes, "consul.revision")
delete(n.Attributes, "unique.consul.name")
delete(n.Attributes, "consul.datacenter")
delete(n.Links, "consul")
}
func (f *ConsulFingerprint) Periodic() (bool, time.Duration) {
return true, 15 * time.Second
}

View File

@@ -0,0 +1,52 @@
package fingerprint
import (
"fmt"
"log"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/helper/stats"
"github.com/hashicorp/nomad/nomad/structs"
)
// CPUFingerprint is used to fingerprint the CPU
type CPUFingerprint struct {
StaticFingerprinter
logger *log.Logger
}
// NewCPUFingerprint is used to create a CPU fingerprint
func NewCPUFingerprint(logger *log.Logger) Fingerprint {
f := &CPUFingerprint{logger: logger}
return f
}
func (f *CPUFingerprint) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
if err := stats.Init(); err != nil {
return false, fmt.Errorf("Unable to obtain CPU information: %v", err)
}
modelName := stats.CPUModelName()
if modelName != "" {
node.Attributes["cpu.modelname"] = modelName
}
mhz := stats.CPUMHzPerCore()
node.Attributes["cpu.frequency"] = fmt.Sprintf("%.0f", mhz)
f.logger.Printf("[DEBUG] fingerprint.cpu: frequency: %.0f MHz", mhz)
numCores := stats.CPUNumCores()
node.Attributes["cpu.numcores"] = fmt.Sprintf("%d", numCores)
f.logger.Printf("[DEBUG] fingerprint.cpu: core count: %d", numCores)
tt := stats.TotalTicksAvailable()
node.Attributes["cpu.totalcompute"] = fmt.Sprintf("%.0f", tt)
if node.Resources == nil {
node.Resources = &structs.Resources{}
}
node.Resources.CPU = int(tt)
return true, nil
}

View File

@@ -0,0 +1,250 @@
package fingerprint
import (
"fmt"
"io/ioutil"
"log"
"net/http"
"net/url"
"os"
"regexp"
"strings"
"time"
"github.com/hashicorp/go-cleanhttp"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/nomad/structs"
)
// This is where the AWS metadata server normally resides. We hardcode the
// "instance" path as well since it's the only one we access here.
const DEFAULT_AWS_URL = "http://169.254.169.254/latest/meta-data/"
// map of instance type to approximate speed, in Mbits/s
// http://serverfault.com/questions/324883/aws-bandwidth-and-content-delivery/326797#326797
// which itself cites these sources:
// - http://blog.rightscale.com/2007/10/28/network-performance-within-amazon-ec2-and-to-amazon-s3/
// - http://www.soc.napier.ac.uk/~bill/chris_p.pdf
//
// This data is meant for a loose approximation
var ec2InstanceSpeedMap = map[string]int{
"m4.large": 80,
"m3.medium": 80,
"m3.large": 80,
"c4.large": 80,
"c3.large": 80,
"c3.xlarge": 80,
"r3.large": 80,
"r3.xlarge": 80,
"i2.xlarge": 80,
"d2.xlarge": 80,
"t2.micro": 16,
"t2.small": 16,
"t2.medium": 16,
"t2.large": 16,
"m4.xlarge": 760,
"m4.2xlarge": 760,
"m4.4xlarge": 760,
"m3.xlarge": 760,
"m3.2xlarge": 760,
"c4.xlarge": 760,
"c4.2xlarge": 760,
"c4.4xlarge": 760,
"c3.2xlarge": 760,
"c3.4xlarge": 760,
"g2.2xlarge": 760,
"r3.2xlarge": 760,
"r3.4xlarge": 760,
"i2.2xlarge": 760,
"i2.4xlarge": 760,
"d2.2xlarge": 760,
"d2.4xlarge": 760,
"m4.10xlarge": 10000,
"c4.8xlarge": 10000,
"c3.8xlarge": 10000,
"g2.8xlarge": 10000,
"r3.8xlarge": 10000,
"i2.8xlarge": 10000,
"d2.8xlarge": 10000,
}
// EnvAWSFingerprint is used to fingerprint AWS metadata
type EnvAWSFingerprint struct {
StaticFingerprinter
logger *log.Logger
}
// NewEnvAWSFingerprint is used to create a fingerprint from AWS metadata
func NewEnvAWSFingerprint(logger *log.Logger) Fingerprint {
f := &EnvAWSFingerprint{logger: logger}
return f
}
func (f *EnvAWSFingerprint) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
if !f.isAWS() {
return false, nil
}
// newNetwork is populated and addded to the Nodes resources
newNetwork := &structs.NetworkResource{
Device: "eth0",
}
if node.Links == nil {
node.Links = make(map[string]string)
}
metadataURL := os.Getenv("AWS_ENV_URL")
if metadataURL == "" {
metadataURL = DEFAULT_AWS_URL
}
// assume 2 seconds is enough time for inside AWS network
client := &http.Client{
Timeout: 2 * time.Second,
Transport: cleanhttp.DefaultTransport(),
}
// Keys and whether they should be namespaced as unique. Any key whose value
// uniquely identifies a node, such as ip, should be marked as unique. When
// marked as unique, the key isn't included in the computed node class.
keys := map[string]bool{
"ami-id": true,
"hostname": true,
"instance-id": true,
"instance-type": false,
"local-hostname": true,
"local-ipv4": true,
"public-hostname": true,
"public-ipv4": true,
"placement/availability-zone": false,
}
for k, unique := range keys {
res, err := client.Get(metadataURL + k)
if res.StatusCode != http.StatusOK {
f.logger.Printf("[WARN]: fingerprint.env_aws: Could not read value for attribute %q", k)
continue
}
if err != nil {
// if it's a URL error, assume we're not in an AWS environment
// TODO: better way to detect AWS? Check xen virtualization?
if _, ok := err.(*url.Error); ok {
return false, nil
}
// not sure what other errors it would return
return false, err
}
resp, err := ioutil.ReadAll(res.Body)
res.Body.Close()
if err != nil {
f.logger.Printf("[ERR]: fingerprint.env_aws: Error reading response body for AWS %s", k)
}
// assume we want blank entries
key := "platform.aws." + strings.Replace(k, "/", ".", -1)
if unique {
key = structs.UniqueNamespace(key)
}
node.Attributes[key] = strings.Trim(string(resp), "\n")
}
// copy over network specific information
if val := node.Attributes["unique.platform.aws.local-ipv4"]; val != "" {
node.Attributes["unique.network.ip-address"] = val
newNetwork.IP = val
newNetwork.CIDR = newNetwork.IP + "/32"
}
// find LinkSpeed from lookup
if throughput := f.linkSpeed(); throughput > 0 {
newNetwork.MBits = throughput
}
if node.Resources == nil {
node.Resources = &structs.Resources{}
}
node.Resources.Networks = append(node.Resources.Networks, newNetwork)
// populate Node Network Resources
// populate Links
node.Links["aws.ec2"] = fmt.Sprintf("%s.%s",
node.Attributes["platform.aws.placement.availability-zone"],
node.Attributes["unique.platform.aws.instance-id"])
return true, nil
}
func (f *EnvAWSFingerprint) isAWS() bool {
// Read the internal metadata URL from the environment, allowing test files to
// provide their own
metadataURL := os.Getenv("AWS_ENV_URL")
if metadataURL == "" {
metadataURL = DEFAULT_AWS_URL
}
// assume 2 seconds is enough time for inside AWS network
client := &http.Client{
Timeout: 2 * time.Second,
Transport: cleanhttp.DefaultTransport(),
}
// Query the metadata url for the ami-id, to veryify we're on AWS
resp, err := client.Get(metadataURL + "ami-id")
if err != nil {
f.logger.Printf("[DEBUG] fingerprint.env_aws: Error querying AWS Metadata URL, skipping")
return false
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
// URL not found, which indicates that this isn't AWS
return false
}
instanceID, err := ioutil.ReadAll(resp.Body)
if err != nil {
f.logger.Printf("[DEBUG] fingerprint.env_aws: Error reading AWS Instance ID, skipping")
return false
}
match, err := regexp.MatchString("ami-*", string(instanceID))
if err != nil || !match {
return false
}
return true
}
// EnvAWSFingerprint uses lookup table to approximate network speeds
func (f *EnvAWSFingerprint) linkSpeed() int {
// Query the API for the instance type, and use the table above to approximate
// the network speed
metadataURL := os.Getenv("AWS_ENV_URL")
if metadataURL == "" {
metadataURL = DEFAULT_AWS_URL
}
// assume 2 seconds is enough time for inside AWS network
client := &http.Client{
Timeout: 2 * time.Second,
Transport: cleanhttp.DefaultTransport(),
}
res, err := client.Get(metadataURL + "instance-type")
body, err := ioutil.ReadAll(res.Body)
res.Body.Close()
if err != nil {
f.logger.Printf("[ERR]: fingerprint.env_aws: Error reading response body for instance-type")
return 0
}
key := strings.Trim(string(body), "\n")
v, ok := ec2InstanceSpeedMap[key]
if !ok {
return 0
}
return v
}

View File

@@ -0,0 +1,270 @@
package fingerprint
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"net/url"
"os"
"regexp"
"strconv"
"strings"
"time"
"github.com/hashicorp/go-cleanhttp"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/nomad/structs"
)
// This is where the GCE metadata server normally resides. We hardcode the
// "instance" path as well since it's the only one we access here.
const DEFAULT_GCE_URL = "http://169.254.169.254/computeMetadata/v1/instance/"
type GCEMetadataNetworkInterface struct {
AccessConfigs []struct {
ExternalIp string
Type string
}
ForwardedIps []string
Ip string
Network string
}
type ReqError struct {
StatusCode int
}
func (e ReqError) Error() string {
return http.StatusText(e.StatusCode)
}
func lastToken(s string) string {
index := strings.LastIndex(s, "/")
return s[index+1:]
}
// EnvGCEFingerprint is used to fingerprint GCE metadata
type EnvGCEFingerprint struct {
StaticFingerprinter
client *http.Client
logger *log.Logger
metadataURL string
}
// NewEnvGCEFingerprint is used to create a fingerprint from GCE metadata
func NewEnvGCEFingerprint(logger *log.Logger) Fingerprint {
// Read the internal metadata URL from the environment, allowing test files to
// provide their own
metadataURL := os.Getenv("GCE_ENV_URL")
if metadataURL == "" {
metadataURL = DEFAULT_GCE_URL
}
// assume 2 seconds is enough time for inside GCE network
client := &http.Client{
Timeout: 2 * time.Second,
Transport: cleanhttp.DefaultTransport(),
}
return &EnvGCEFingerprint{
client: client,
logger: logger,
metadataURL: metadataURL,
}
}
func (f *EnvGCEFingerprint) Get(attribute string, recursive bool) (string, error) {
reqUrl := f.metadataURL + attribute
if recursive {
reqUrl = reqUrl + "?recursive=true"
}
parsedUrl, err := url.Parse(reqUrl)
if err != nil {
return "", err
}
req := &http.Request{
Method: "GET",
URL: parsedUrl,
Header: http.Header{
"Metadata-Flavor": []string{"Google"},
},
}
res, err := f.client.Do(req)
if err != nil || res.StatusCode != http.StatusOK {
f.logger.Printf("[DEBUG] fingerprint.env_gce: Could not read value for attribute %q", attribute)
return "", err
}
resp, err := ioutil.ReadAll(res.Body)
res.Body.Close()
if err != nil {
f.logger.Printf("[ERR] fingerprint.env_gce: Error reading response body for GCE %s", attribute)
return "", err
}
if res.StatusCode >= 400 {
return "", ReqError{res.StatusCode}
}
return string(resp), nil
}
func checkError(err error, logger *log.Logger, desc string) error {
// If it's a URL error, assume we're not actually in an GCE environment.
// To the outer layers, this isn't an error so return nil.
if _, ok := err.(*url.Error); ok {
logger.Printf("[DEBUG] fingerprint.env_gce: Error querying GCE " + desc + ", skipping")
return nil
}
// Otherwise pass the error through.
return err
}
func (f *EnvGCEFingerprint) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
if !f.isGCE() {
return false, nil
}
if node.Links == nil {
node.Links = make(map[string]string)
}
// Keys and whether they should be namespaced as unique. Any key whose value
// uniquely identifies a node, such as ip, should be marked as unique. When
// marked as unique, the key isn't included in the computed node class.
keys := map[string]bool{
"hostname": true,
"id": true,
"cpu-platform": false,
"scheduling/automatic-restart": false,
"scheduling/on-host-maintenance": false,
}
for k, unique := range keys {
value, err := f.Get(k, false)
if err != nil {
return false, checkError(err, f.logger, k)
}
// assume we want blank entries
key := "platform.gce." + strings.Replace(k, "/", ".", -1)
if unique {
key = structs.UniqueNamespace(key)
}
node.Attributes[key] = strings.Trim(string(value), "\n")
}
// These keys need everything before the final slash removed to be usable.
keys = map[string]bool{
"machine-type": false,
"zone": false,
}
for k, unique := range keys {
value, err := f.Get(k, false)
if err != nil {
return false, checkError(err, f.logger, k)
}
key := "platform.gce." + k
if unique {
key = structs.UniqueNamespace(key)
}
node.Attributes[key] = strings.Trim(lastToken(value), "\n")
}
// Get internal and external IPs (if they exist)
value, err := f.Get("network-interfaces/", true)
var interfaces []GCEMetadataNetworkInterface
if err := json.Unmarshal([]byte(value), &interfaces); err != nil {
f.logger.Printf("[WARN] fingerprint.env_gce: Error decoding network interface information: %s", err.Error())
}
for _, intf := range interfaces {
prefix := "platform.gce.network." + lastToken(intf.Network)
uniquePrefix := "unique." + prefix
node.Attributes[prefix] = "true"
node.Attributes[uniquePrefix+".ip"] = strings.Trim(intf.Ip, "\n")
for index, accessConfig := range intf.AccessConfigs {
node.Attributes[uniquePrefix+".external-ip."+strconv.Itoa(index)] = accessConfig.ExternalIp
}
}
var tagList []string
value, err = f.Get("tags", false)
if err != nil {
return false, checkError(err, f.logger, "tags")
}
if err := json.Unmarshal([]byte(value), &tagList); err != nil {
f.logger.Printf("[WARN] fingerprint.env_gce: Error decoding instance tags: %s", err.Error())
}
for _, tag := range tagList {
attr := "platform.gce.tag."
var key string
// If the tag is namespaced as unique, we strip it from the tag and
// prepend to the whole attribute.
if structs.IsUniqueNamespace(tag) {
tag = strings.TrimPrefix(tag, structs.NodeUniqueNamespace)
key = fmt.Sprintf("%s%s%s", structs.NodeUniqueNamespace, attr, tag)
} else {
key = fmt.Sprintf("%s%s", attr, tag)
}
node.Attributes[key] = "true"
}
var attrDict map[string]string
value, err = f.Get("attributes/", true)
if err != nil {
return false, checkError(err, f.logger, "attributes/")
}
if err := json.Unmarshal([]byte(value), &attrDict); err != nil {
f.logger.Printf("[WARN] fingerprint.env_gce: Error decoding instance attributes: %s", err.Error())
}
for k, v := range attrDict {
attr := "platform.gce.attr."
var key string
// If the key is namespaced as unique, we strip it from the
// key and prepend to the whole attribute.
if structs.IsUniqueNamespace(k) {
k = strings.TrimPrefix(k, structs.NodeUniqueNamespace)
key = fmt.Sprintf("%s%s%s", structs.NodeUniqueNamespace, attr, k)
} else {
key = fmt.Sprintf("%s%s", attr, k)
}
node.Attributes[key] = strings.Trim(v, "\n")
}
// populate Links
node.Links["gce"] = node.Attributes["unique.platform.gce.id"]
return true, nil
}
func (f *EnvGCEFingerprint) isGCE() bool {
// TODO: better way to detect GCE?
// Query the metadata url for the machine type, to verify we're on GCE
machineType, err := f.Get("machine-type", false)
if err != nil {
if re, ok := err.(ReqError); !ok || re.StatusCode != 404 {
// If it wasn't a 404 error, print an error message.
f.logger.Printf("[DEBUG] fingerprint.env_gce: Error querying GCE Metadata URL, skipping")
}
return false
}
match, err := regexp.MatchString("projects/.+/machineTypes/.+", machineType)
if err != nil || !match {
return false
}
return true
}

View File

@@ -0,0 +1,87 @@
package fingerprint
import (
"fmt"
"log"
"sort"
"time"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/nomad/structs"
)
// EmptyDuration is to be used by fingerprinters that are not periodic.
const (
EmptyDuration = time.Duration(0)
)
func init() {
builtinFingerprintMap["arch"] = NewArchFingerprint
builtinFingerprintMap["cpu"] = NewCPUFingerprint
builtinFingerprintMap["env_aws"] = NewEnvAWSFingerprint
builtinFingerprintMap["env_gce"] = NewEnvGCEFingerprint
builtinFingerprintMap["host"] = NewHostFingerprint
builtinFingerprintMap["memory"] = NewMemoryFingerprint
builtinFingerprintMap["network"] = NewNetworkFingerprint
builtinFingerprintMap["nomad"] = NewNomadFingerprint
builtinFingerprintMap["storage"] = NewStorageFingerprint
// Initialize the list of available fingerprinters per platform. Each
// platform defines its own list of available fingerprinters.
initPlatformFingerprints(builtinFingerprintMap)
}
// builtinFingerprintMap contains the built in registered fingerprints which are
// available for a given platform.
var builtinFingerprintMap = make(map[string]Factory, 16)
// BuiltinFingerprints is a slice containing the key names of all registered
// fingerprints available, to provided an ordered iteration
func BuiltinFingerprints() []string {
fingerprints := make([]string, 0, len(builtinFingerprintMap))
for k := range builtinFingerprintMap {
fingerprints = append(fingerprints, k)
}
sort.Strings(fingerprints)
return fingerprints
}
// NewFingerprint is used to instantiate and return a new fingerprint
// given the name and a logger
func NewFingerprint(name string, logger *log.Logger) (Fingerprint, error) {
// Lookup the factory function
factory, ok := builtinFingerprintMap[name]
if !ok {
return nil, fmt.Errorf("unknown fingerprint '%s'", name)
}
// Instantiate the fingerprint
f := factory(logger)
return f, nil
}
// Factory is used to instantiate a new Fingerprint
type Factory func(*log.Logger) Fingerprint
// Fingerprint is used for doing "fingerprinting" of the
// host to automatically determine attributes, resources,
// and metadata about it. Each of these is a heuristic, and
// many of them can be applied on a particular host.
type Fingerprint interface {
// Fingerprint is used to update properties of the Node,
// and returns if the fingerprint was applicable and a potential error.
Fingerprint(*config.Config, *structs.Node) (bool, error)
// Periodic is a mechanism for the fingerprinter to indicate that it should
// be run periodically. The return value is a boolean indicating if it
// should be periodic, and if true, a duration.
Periodic() (bool, time.Duration)
}
// StaticFingerprinter can be embedded in a struct that has a Fingerprint method
// to make it non-periodic.
type StaticFingerprinter struct{}
func (s *StaticFingerprinter) Periodic() (bool, time.Duration) {
return false, EmptyDuration
}

View File

@@ -0,0 +1,6 @@
// +build darwin dragonfly freebsd netbsd openbsd solaris windows
package fingerprint
func initPlatformFingerprints(fps map[string]Factory) {
}

View File

@@ -0,0 +1,5 @@
package fingerprint
func initPlatformFingerprints(fps map[string]Factory) {
fps["cgroup"] = NewCGroupFingerprint
}

View File

@@ -0,0 +1,51 @@
package fingerprint
import (
"fmt"
"log"
"os/exec"
"runtime"
"strings"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/shirou/gopsutil/host"
)
// HostFingerprint is used to fingerprint the host
type HostFingerprint struct {
StaticFingerprinter
logger *log.Logger
}
// NewHostFingerprint is used to create a Host fingerprint
func NewHostFingerprint(logger *log.Logger) Fingerprint {
f := &HostFingerprint{logger: logger}
return f
}
func (f *HostFingerprint) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
hostInfo, err := host.Info()
if err != nil {
f.logger.Println("[WARN] Error retrieving host information: ", err)
return false, err
}
node.Attributes["os.name"] = hostInfo.Platform
node.Attributes["os.version"] = hostInfo.PlatformVersion
node.Attributes["kernel.name"] = runtime.GOOS
node.Attributes["kernel.version"] = ""
if runtime.GOOS != "windows" {
out, err := exec.Command("uname", "-r").Output()
if err != nil {
return false, fmt.Errorf("Failed to run uname: %s", err)
}
node.Attributes["kernel.version"] = strings.Trim(string(out), "\n")
}
node.Attributes["unique.hostname"] = hostInfo.Hostname
return true, nil
}

View File

@@ -0,0 +1,43 @@
package fingerprint
import (
"fmt"
"log"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/shirou/gopsutil/mem"
)
// MemoryFingerprint is used to fingerprint the available memory on the node
type MemoryFingerprint struct {
StaticFingerprinter
logger *log.Logger
}
// NewMemoryFingerprint is used to create a Memory fingerprint
func NewMemoryFingerprint(logger *log.Logger) Fingerprint {
f := &MemoryFingerprint{
logger: logger,
}
return f
}
func (f *MemoryFingerprint) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
memInfo, err := mem.VirtualMemory()
if err != nil {
f.logger.Printf("[WARN] Error reading memory information: %s", err)
return false, err
}
if memInfo.Total > 0 {
node.Attributes["memory.totalbytes"] = fmt.Sprintf("%d", memInfo.Total)
if node.Resources == nil {
node.Resources = &structs.Resources{}
}
node.Resources.MemoryMB = int(memInfo.Total / 1024 / 1024)
}
return true, nil
}

View File

@@ -0,0 +1,167 @@
package fingerprint
import (
"errors"
"fmt"
"log"
"net"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/nomad/structs"
)
// NetworkFingerprint is used to fingerprint the Network capabilities of a node
type NetworkFingerprint struct {
StaticFingerprinter
logger *log.Logger
interfaceDetector NetworkInterfaceDetector
}
// An interface to isolate calls to various api in net package
// This facilitates testing where we can implement
// fake interfaces and addresses to test varios code paths
type NetworkInterfaceDetector interface {
Interfaces() ([]net.Interface, error)
InterfaceByName(name string) (*net.Interface, error)
Addrs(intf *net.Interface) ([]net.Addr, error)
}
// Implements the interface detector which calls net directly
type DefaultNetworkInterfaceDetector struct {
}
func (b *DefaultNetworkInterfaceDetector) Interfaces() ([]net.Interface, error) {
return net.Interfaces()
}
func (b *DefaultNetworkInterfaceDetector) InterfaceByName(name string) (*net.Interface, error) {
return net.InterfaceByName(name)
}
func (b *DefaultNetworkInterfaceDetector) Addrs(intf *net.Interface) ([]net.Addr, error) {
return intf.Addrs()
}
// NewNetworkFingerprint returns a new NetworkFingerprinter with the given
// logger
func NewNetworkFingerprint(logger *log.Logger) Fingerprint {
f := &NetworkFingerprint{logger: logger, interfaceDetector: &DefaultNetworkInterfaceDetector{}}
return f
}
func (f *NetworkFingerprint) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
// newNetwork is populated and addded to the Nodes resources
newNetwork := &structs.NetworkResource{}
var ip string
intf, err := f.findInterface(cfg.NetworkInterface)
switch {
case err != nil:
return false, fmt.Errorf("Error while detecting network interface during fingerprinting: %v", err)
case intf == nil:
// No interface could be found
return false, nil
}
if ip, err = f.ipAddress(intf); err != nil {
return false, fmt.Errorf("Unable to find IP address of interface: %s, err: %v", intf.Name, err)
}
newNetwork.Device = intf.Name
node.Attributes["unique.network.ip-address"] = ip
newNetwork.IP = ip
newNetwork.CIDR = newNetwork.IP + "/32"
f.logger.Printf("[DEBUG] fingerprint.network: Detected interface %v with IP %v during fingerprinting", intf.Name, ip)
if throughput := f.linkSpeed(intf.Name); throughput > 0 {
newNetwork.MBits = throughput
f.logger.Printf("[DEBUG] fingerprint.network: link speed for %v set to %v", intf.Name, newNetwork.MBits)
} else {
f.logger.Printf("[DEBUG] fingerprint.network: Unable to read link speed; setting to default %v", cfg.NetworkSpeed)
newNetwork.MBits = cfg.NetworkSpeed
}
if node.Resources == nil {
node.Resources = &structs.Resources{}
}
node.Resources.Networks = append(node.Resources.Networks, newNetwork)
// return true, because we have a network connection
return true, nil
}
// Gets the ipv4 addr for a network interface
func (f *NetworkFingerprint) ipAddress(intf *net.Interface) (string, error) {
var addrs []net.Addr
var err error
if addrs, err = f.interfaceDetector.Addrs(intf); err != nil {
return "", err
}
if len(addrs) == 0 {
return "", errors.New(fmt.Sprintf("Interface %s has no IP address", intf.Name))
}
for _, addr := range addrs {
var ip net.IP
switch v := (addr).(type) {
case *net.IPNet:
ip = v.IP
case *net.IPAddr:
ip = v.IP
}
if ip.To4() != nil {
return ip.String(), nil
}
}
return "", fmt.Errorf("Couldn't parse IP address for interface %s", intf.Name)
}
// Checks if the device is marked UP by the operator
func (f *NetworkFingerprint) isDeviceEnabled(intf *net.Interface) bool {
return intf.Flags&net.FlagUp != 0
}
// Checks if the device has any IP address configured
func (f *NetworkFingerprint) deviceHasIpAddress(intf *net.Interface) bool {
_, err := f.ipAddress(intf)
return err == nil
}
func (n *NetworkFingerprint) isDeviceLoopBackOrPointToPoint(intf *net.Interface) bool {
return intf.Flags&(net.FlagLoopback|net.FlagPointToPoint) != 0
}
// Returns the interface with the name passed by user
// If the name is blank then it iterates through all the devices
// and finds one which is routable and marked as UP
// It excludes PPP and lo devices unless they are specifically asked
func (f *NetworkFingerprint) findInterface(deviceName string) (*net.Interface, error) {
var interfaces []net.Interface
var err error
if deviceName != "" {
return f.interfaceDetector.InterfaceByName(deviceName)
}
var intfs []net.Interface
if intfs, err = f.interfaceDetector.Interfaces(); err != nil {
return nil, err
}
for _, intf := range intfs {
if f.isDeviceEnabled(&intf) && !f.isDeviceLoopBackOrPointToPoint(&intf) && f.deviceHasIpAddress(&intf) {
interfaces = append(interfaces, intf)
}
}
if len(interfaces) == 0 {
return nil, nil
}
return &interfaces[0], nil
}

View File

@@ -0,0 +1,8 @@
// +build !linux,!windows
package fingerprint
// linkSpeed returns the default link speed
func (f *NetworkFingerprint) linkSpeed(device string) int {
return 0
}

View File

@@ -0,0 +1,78 @@
package fingerprint
import (
"fmt"
"io/ioutil"
"os/exec"
"regexp"
"strconv"
"strings"
)
// linkSpeedSys parses link speed in Mb/s from /sys.
func (f *NetworkFingerprint) linkSpeedSys(device string) int {
path := fmt.Sprintf("/sys/class/net/%s/speed", device)
// Read contents of the device/speed file
content, err := ioutil.ReadFile(path)
if err != nil {
f.logger.Printf("[DEBUG] fingerprint.network: Unable to read link speed from %s", path)
return 0
}
lines := strings.Split(string(content), "\n")
mbs, err := strconv.Atoi(lines[0])
if err != nil || mbs <= 0 {
f.logger.Printf("[DEBUG] fingerprint.network: Unable to parse link speed from %s", path)
return 0
}
return mbs
}
// linkSpeed returns link speed in Mb/s, or 0 when unable to determine it.
func (f *NetworkFingerprint) linkSpeed(device string) int {
// Use LookPath to find the ethtool in the systems $PATH
// If it's not found or otherwise errors, LookPath returns and empty string
// and an error we can ignore for our purposes
ethtoolPath, _ := exec.LookPath("ethtool")
if ethtoolPath != "" {
if speed := f.linkSpeedEthtool(ethtoolPath, device); speed > 0 {
return speed
}
}
// Fall back on checking a system file for link speed.
return f.linkSpeedSys(device)
}
// linkSpeedEthtool determines link speed in Mb/s with 'ethtool'.
func (f *NetworkFingerprint) linkSpeedEthtool(path, device string) int {
outBytes, err := exec.Command(path, device).Output()
if err != nil {
f.logger.Printf("[WARN] fingerprint.network: Error calling ethtool (%s %s): %v", path, device, err)
return 0
}
output := strings.TrimSpace(string(outBytes))
re := regexp.MustCompile("Speed: [0-9]+[a-zA-Z]+/s")
m := re.FindString(output)
if m == "" {
// no matches found, output may be in a different format
f.logger.Printf("[WARN] fingerprint.network: Unable to parse Speed in output of '%s %s'", path, device)
return 0
}
// Split and trim the Mb/s unit from the string output
args := strings.Split(m, ": ")
raw := strings.TrimSuffix(args[1], "Mb/s")
// convert to Mb/s
mbs, err := strconv.Atoi(raw)
if err != nil || mbs <= 0 {
f.logger.Printf("[WARN] fingerprint.network: Unable to parse Mb/s in output of '%s %s'", path, device)
return 0
}
return mbs
}

View File

@@ -0,0 +1,52 @@
package fingerprint
import (
"fmt"
"os/exec"
"strconv"
"strings"
)
// linkSpeed returns link speed in Mb/s, or 0 when unable to determine it.
func (f *NetworkFingerprint) linkSpeed(device string) int {
command := fmt.Sprintf("Get-NetAdapter -IncludeHidden | Where name -eq '%s' | Select -ExpandProperty LinkSpeed", device)
path := "powershell.exe"
outBytes, err := exec.Command(path, command).Output()
if err != nil {
f.logger.Printf("[WARN] fingerprint.network: Error calling %s (%s): %v", path, command, err)
return 0
}
output := strings.TrimSpace(string(outBytes))
return f.parseLinkSpeed(output)
}
func (f *NetworkFingerprint) parseLinkSpeed(commandOutput string) int {
args := strings.Split(commandOutput, " ")
if len(args) != 2 {
f.logger.Printf("[WARN] fingerprint.network: Couldn't split LinkSpeed (%s)", commandOutput)
return 0
}
unit := strings.Replace(args[1], "\r\n", "", -1)
value, err := strconv.Atoi(args[0])
if err != nil {
f.logger.Printf("[WARN] fingerprint.network: Unable to parse LinkSpeed value (%s)", commandOutput)
return 0
}
switch unit {
case "Mbps":
return value
case "Kbps":
return value / 1000
case "Gbps":
return value * 1000
case "bps":
return value / 1000000
}
return 0
}

Some files were not shown because too many files have changed in this diff Show More