jobqueue: Introduce jobqueue backed by a postgres database

Co-authored-by: sanne <sanne.raymaekers@gmail.com>

vendor/github.com/jackc/puddle/.travis.yml

@@ -0,0 +1,16 @@
+language: go
+
+go:
+  - 1.x
+  - tip
+
+env:
+  global:
+    - STRESS_TEST_DURATION=15s
+
+script:
+  - go test -v -race
+
+matrix:
+  allow_failures:
+    - go: tip

vendor/github.com/jackc/puddle/CHANGELOG.md

@@ -0,0 +1,22 @@
+# 1.1.3 (December 3, 2020)
+
+* Fix: Failed resource creation could cause concurrent Acquire to hang. (Evgeny Vanslov)
+
+# 1.1.2 (September 26, 2020)
+
+* Fix: Resource.Destroy no longer removes itself from the pool before its destructor has completed.
+* Fix: Prevent crash when pool is closed while resource is being created.
+
+# 1.1.1 (April 2, 2020)
+
+* Pool.Close can be safely called multiple times
+* AcquireAllIDle immediately returns nil if pool is closed
+* CreateResource checks if pool is closed before taking any action
+* Fix potential race condition when CreateResource and Close are called concurrently. CreateResource now checks if pool is closed before adding newly created resource to pool.
+
+# 1.1.0 (February 5, 2020)
+
+* Use runtime.nanotime for faster tracking of acquire time and last usage time.
+* Track resource idle time to enable client health check logic. (Patrick Ellul)
+* Add CreateResource to construct a new resource without acquiring it. (Patrick Ellul)
+* Fix deadlock race when acquire is cancelled. (Michael Tharp)

vendor/github.com/jackc/puddle/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2018 Jack Christensen
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/jackc/puddle/README.md

@@ -0,0 +1,54 @@
+[![](https://godoc.org/github.com/jackc/puddle?status.svg)](https://godoc.org/github.com/jackc/puddle)
+[![Build Status](https://travis-ci.org/jackc/puddle.svg)](https://travis-ci.org/jackc/puddle)
+
+# Puddle
+
+Puddle is a tiny generic resource pool library for Go that uses the standard
+context library to signal cancellation of acquires. It is designed to contain
+the minimum functionality required for a resource pool. It can be used directly
+or it can be used as the base for a domain specific resource pool. For example,
+a database connection pool may use puddle internally and implement health checks
+and keep-alive behavior without needing to implement any concurrent code of its
+own.
+
+## Features
+
+* Acquire cancellation via context standard library
+* Statistics API for monitoring pool pressure
+* No dependencies outside of standard library
+* High performance
+* 100% test coverage
+
+## Example Usage
+
+```go
+constructor := func(context.Context) (interface{}, error) {
+  return net.Dial("tcp", "127.0.0.1:8080")
+}
+destructor := func(value interface{}) {
+  value.(net.Conn).Close()
+}
+maxPoolSize := 10
+
+pool := puddle.NewPool(constructor, destructor, maxPoolSize)
+
+// Acquire resource from the pool.
+res, err := pool.Acquire(context.Background())
+if err != nil {
+  // ...
+}
+
+// Use resource.
+_, err = res.Value().(net.Conn).Write([]byte{1})
+if err != nil {
+  // ...
+}
+
+// Release when done.
+res.Release()
+
+```
+
+## License
+
+MIT

vendor/github.com/jackc/puddle/doc.go

@@ -0,0 +1,11 @@
+// Package puddle is a generic resource pool.
+/*
+
+Puddle is a tiny generic resource pool library for Go that uses the standard
+context library to signal cancellation of acquires. It is designed to contain
+the minimum functionality a resource pool needs that cannot be implemented
+without concurrency concerns. For example, a database connection pool may use
+puddle internally and implement health checks and keep-alive behavior without
+needing to implement any concurrent code of its own.
+*/
+package puddle

vendor/github.com/jackc/puddle/go.mod

@@ -0,0 +1,5 @@
+module github.com/jackc/puddle
+
+go 1.12
+
+require github.com/stretchr/testify v1.3.0

vendor/github.com/jackc/puddle/go.sum

@@ -0,0 +1,7 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=

vendor/github.com/jackc/puddle/nanotime_time.go

@@ -0,0 +1,13 @@
+// +build purego appengine js
+
+// This file contains the safe implementation of nanotime using time.Now().
+
+package puddle
+
+import (
+	"time"
+)
+
+func nanotime() int64 {
+	return time.Now().UnixNano()
+}

vendor/github.com/jackc/puddle/nanotime_unsafe.go

@@ -0,0 +1,12 @@
+// +build !purego,!appengine,!js
+
+// This file contains the implementation of nanotime using runtime.nanotime.
+
+package puddle
+
+import "unsafe"
+
+var _ = unsafe.Sizeof(0)
+
+//go:linkname nanotime runtime.nanotime
+func nanotime() int64

vendor/github.com/jackc/puddle/pool.go

@@ -0,0 +1,477 @@
+package puddle
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"time"
+)
+
+const (
+	resourceStatusConstructing = 0
+	resourceStatusIdle         = iota
+	resourceStatusAcquired     = iota
+	resourceStatusHijacked     = iota
+)
+
+// ErrClosedPool occurs on an attempt to acquire a connection from a closed pool
+// or a pool that is closed while the acquire is waiting.
+var ErrClosedPool = errors.New("closed pool")
+
+// Constructor is a function called by the pool to construct a resource.
+type Constructor func(ctx context.Context) (res interface{}, err error)
+
+// Destructor is a function called by the pool to destroy a resource.
+type Destructor func(res interface{})
+
+// Resource is the resource handle returned by acquiring from the pool.
+type Resource struct {
+	value        interface{}
+	pool         *Pool
+	creationTime time.Time
+	lastUsedNano int64
+	status       byte
+}
+
+// Value returns the resource value.
+func (res *Resource) Value() interface{} {
+	if !(res.status == resourceStatusAcquired || res.status == resourceStatusHijacked) {
+		panic("tried to access resource that is not acquired or hijacked")
+	}
+	return res.value
+}
+
+// Release returns the resource to the pool. res must not be subsequently used.
+func (res *Resource) Release() {
+	if res.status != resourceStatusAcquired {
+		panic("tried to release resource that is not acquired")
+	}
+	res.pool.releaseAcquiredResource(res, nanotime())
+}
+
+// ReleaseUnused returns the resource to the pool without updating when it was last used used. i.e. LastUsedNanotime
+// will not change. res must not be subsequently used.
+func (res *Resource) ReleaseUnused() {
+	if res.status != resourceStatusAcquired {
+		panic("tried to release resource that is not acquired")
+	}
+	res.pool.releaseAcquiredResource(res, res.lastUsedNano)
+}
+
+// Destroy returns the resource to the pool for destruction. res must not be
+// subsequently used.
+func (res *Resource) Destroy() {
+	if res.status != resourceStatusAcquired {
+		panic("tried to destroy resource that is not acquired")
+	}
+	go res.pool.destroyAcquiredResource(res)
+}
+
+// Hijack assumes ownership of the resource from the pool. Caller is responsible
+// for cleanup of resource value.
+func (res *Resource) Hijack() {
+	if res.status != resourceStatusAcquired {
+		panic("tried to hijack resource that is not acquired")
+	}
+	res.pool.hijackAcquiredResource(res)
+}
+
+// CreationTime returns when the resource was created by the pool.
+func (res *Resource) CreationTime() time.Time {
+	if !(res.status == resourceStatusAcquired || res.status == resourceStatusHijacked) {
+		panic("tried to access resource that is not acquired or hijacked")
+	}
+	return res.creationTime
+}
+
+// LastUsedNanotime returns when Release was last called on the resource measured in nanoseconds from an arbitrary time
+// (a monotonic time). Returns creation time if Release has never been called. This is only useful to compare with
+// other calls to LastUsedNanotime. In almost all cases, IdleDuration should be used instead.
+func (res *Resource) LastUsedNanotime() int64 {
+	if !(res.status == resourceStatusAcquired || res.status == resourceStatusHijacked) {
+		panic("tried to access resource that is not acquired or hijacked")
+	}
+
+	return res.lastUsedNano
+}
+
+// IdleDuration returns the duration since Release was last called on the resource. This is equivalent to subtracting
+// LastUsedNanotime to the current nanotime.
+func (res *Resource) IdleDuration() time.Duration {
+	if !(res.status == resourceStatusAcquired || res.status == resourceStatusHijacked) {
+		panic("tried to access resource that is not acquired or hijacked")
+	}
+
+	return time.Duration(nanotime() - res.lastUsedNano)
+}
+
+// Pool is a concurrency-safe resource pool.
+type Pool struct {
+	cond       *sync.Cond
+	destructWG *sync.WaitGroup
+
+	allResources  []*Resource
+	idleResources []*Resource
+
+	constructor Constructor
+	destructor  Destructor
+	maxSize     int32
+
+	acquireCount         int64
+	acquireDuration      time.Duration
+	emptyAcquireCount    int64
+	canceledAcquireCount int64
+
+	closed bool
+}
+
+// NewPool creates a new pool. Panics if maxSize is less than 1.
+func NewPool(constructor Constructor, destructor Destructor, maxSize int32) *Pool {
+	if maxSize < 1 {
+		panic("maxSize is less than 1")
+	}
+
+	return &Pool{
+		cond:        sync.NewCond(new(sync.Mutex)),
+		destructWG:  &sync.WaitGroup{},
+		maxSize:     maxSize,
+		constructor: constructor,
+		destructor:  destructor,
+	}
+}
+
+// Close destroys all resources in the pool and rejects future Acquire calls.
+// Blocks until all resources are returned to pool and destroyed.
+func (p *Pool) Close() {
+	p.cond.L.Lock()
+	if p.closed {
+		p.cond.L.Unlock()
+		return
+	}
+	p.closed = true
+
+	for _, res := range p.idleResources {
+		p.allResources = removeResource(p.allResources, res)
+		go p.destructResourceValue(res.value)
+	}
+	p.idleResources = nil
+	p.cond.L.Unlock()
+
+	// Wake up all go routines waiting for a resource to be returned so they can terminate.
+	p.cond.Broadcast()
+
+	p.destructWG.Wait()
+}
+
+// Stat is a snapshot of Pool statistics.
+type Stat struct {
+	constructingResources int32
+	acquiredResources     int32
+	idleResources         int32
+	maxResources          int32
+	acquireCount          int64
+	acquireDuration       time.Duration
+	emptyAcquireCount     int64
+	canceledAcquireCount  int64
+}
+
+// TotalResource returns the total number of resources currently in the pool.
+// The value is the sum of ConstructingResources, AcquiredResources, and
+// IdleResources.
+func (s *Stat) TotalResources() int32 {
+	return s.constructingResources + s.acquiredResources + s.idleResources
+}
+
+// ConstructingResources returns the number of resources with construction in progress in
+// the pool.
+func (s *Stat) ConstructingResources() int32 {
+	return s.constructingResources
+}
+
+// AcquiredResources returns the number of currently acquired resources in the pool.
+func (s *Stat) AcquiredResources() int32 {
+	return s.acquiredResources
+}
+
+// IdleResources returns the number of currently idle resources in the pool.
+func (s *Stat) IdleResources() int32 {
+	return s.idleResources
+}
+
+// MaxResources returns the maximum size of the pool.
+func (s *Stat) MaxResources() int32 {
+	return s.maxResources
+}
+
+// AcquireCount returns the cumulative count of successful acquires from the pool.
+func (s *Stat) AcquireCount() int64 {
+	return s.acquireCount
+}
+
+// AcquireDuration returns the total duration of all successful acquires from
+// the pool.
+func (s *Stat) AcquireDuration() time.Duration {
+	return s.acquireDuration
+}
+
+// EmptyAcquireCount returns the cumulative count of successful acquires from the pool
+// that waited for a resource to be released or constructed because the pool was
+// empty.
+func (s *Stat) EmptyAcquireCount() int64 {
+	return s.emptyAcquireCount
+}
+
+// CanceledAcquireCount returns the cumulative count of acquires from the pool
+// that were canceled by a context.
+func (s *Stat) CanceledAcquireCount() int64 {
+	return s.canceledAcquireCount
+}
+
+// Stat returns the current pool statistics.
+func (p *Pool) Stat() *Stat {
+	p.cond.L.Lock()
+	s := &Stat{
+		maxResources:         p.maxSize,
+		acquireCount:         p.acquireCount,
+		emptyAcquireCount:    p.emptyAcquireCount,
+		canceledAcquireCount: p.canceledAcquireCount,
+		acquireDuration:      p.acquireDuration,
+	}
+
+	for _, res := range p.allResources {
+		switch res.status {
+		case resourceStatusConstructing:
+			s.constructingResources += 1
+		case resourceStatusIdle:
+			s.idleResources += 1
+		case resourceStatusAcquired:
+			s.acquiredResources += 1
+		}
+	}
+
+	p.cond.L.Unlock()
+	return s
+}
+
+// Acquire gets a resource from the pool. If no resources are available and the pool
+// is not at maximum capacity it will create a new resource. If the pool is at
+// maximum capacity it will block until a resource is available. ctx can be used
+// to cancel the Acquire.
+func (p *Pool) Acquire(ctx context.Context) (*Resource, error) {
+	startNano := nanotime()
+	p.cond.L.Lock()
+	if doneChan := ctx.Done(); doneChan != nil {
+		select {
+		case <-ctx.Done():
+			p.canceledAcquireCount += 1
+			p.cond.L.Unlock()
+			return nil, ctx.Err()
+		default:
+		}
+	}
+
+	emptyAcquire := false
+
+	for {
+		if p.closed {
+			p.cond.L.Unlock()
+			return nil, ErrClosedPool
+		}
+
+		// If a resource is available now
+		if len(p.idleResources) > 0 {
+			res := p.idleResources[len(p.idleResources)-1]
+			p.idleResources = p.idleResources[:len(p.idleResources)-1]
+			res.status = resourceStatusAcquired
+			if emptyAcquire {
+				p.emptyAcquireCount += 1
+			}
+			p.acquireCount += 1
+			p.acquireDuration += time.Duration(nanotime() - startNano)
+			p.cond.L.Unlock()
+			return res, nil
+		}
+
+		emptyAcquire = true
+
+		// If there is room to create a resource do so
+		if len(p.allResources) < int(p.maxSize) {
+			res := &Resource{pool: p, creationTime: time.Now(), lastUsedNano: nanotime(), status: resourceStatusConstructing}
+			p.allResources = append(p.allResources, res)
+			p.destructWG.Add(1)
+			p.cond.L.Unlock()
+
+			value, err := p.constructResourceValue(ctx)
+			p.cond.L.Lock()
+			if err != nil {
+				p.allResources = removeResource(p.allResources, res)
+				p.destructWG.Done()
+
+				select {
+				case <-ctx.Done():
+					if err == ctx.Err() {
+						p.canceledAcquireCount += 1
+					}
+				default:
+				}
+
+				p.cond.L.Unlock()
+				p.cond.Signal()
+				return nil, err
+			}
+
+			res.value = value
+			res.status = resourceStatusAcquired
+			p.emptyAcquireCount += 1
+			p.acquireCount += 1
+			p.acquireDuration += time.Duration(nanotime() - startNano)
+			p.cond.L.Unlock()
+			return res, nil
+		}
+
+		if ctx.Done() == nil {
+			p.cond.Wait()
+		} else {
+			// Convert p.cond.Wait into a channel
+			waitChan := make(chan struct{}, 1)
+			go func() {
+				p.cond.Wait()
+				waitChan <- struct{}{}
+			}()
+
+			select {
+			case <-ctx.Done():
+				// Allow goroutine waiting for signal to exit. Re-signal since we couldn't
+				// do anything with it. Another goroutine might be waiting.
+				go func() {
+					<-waitChan
+					p.cond.Signal()
+					p.cond.L.Unlock()
+				}()
+
+				p.cond.L.Lock()
+				p.canceledAcquireCount += 1
+				p.cond.L.Unlock()
+				return nil, ctx.Err()
+			case <-waitChan:
+			}
+		}
+	}
+}
+
+// AcquireAllIdle atomically acquires all currently idle resources. Its intended
+// use is for health check and keep-alive functionality. It does not update pool
+// statistics.
+func (p *Pool) AcquireAllIdle() []*Resource {
+	p.cond.L.Lock()
+	if p.closed {
+		p.cond.L.Unlock()
+		return nil
+	}
+
+	for _, res := range p.idleResources {
+		res.status = resourceStatusAcquired
+	}
+	resources := make([]*Resource, len(p.idleResources))
+	copy(resources, p.idleResources)
+	p.idleResources = p.idleResources[0:0]
+
+	p.cond.L.Unlock()
+	return resources
+}
+
+// CreateResource constructs a new resource without acquiring it.
+// It goes straight in the IdlePool. It does not check against maxSize.
+// It can be useful to maintain warm resources under little load.
+func (p *Pool) CreateResource(ctx context.Context) error {
+	p.cond.L.Lock()
+	if p.closed {
+		p.cond.L.Unlock()
+		return ErrClosedPool
+	}
+	p.cond.L.Unlock()
+
+	value, err := p.constructResourceValue(ctx)
+	if err != nil {
+		return err
+	}
+
+	res := &Resource{
+		pool:         p,
+		creationTime: time.Now(),
+		status:       resourceStatusIdle,
+		value:        value,
+		lastUsedNano: nanotime(),
+	}
+	p.destructWG.Add(1)
+
+	p.cond.L.Lock()
+	// If closed while constructing resource then destroy it and return an error
+	if p.closed {
+		go p.destructResourceValue(res.value)
+		return ErrClosedPool
+	}
+	p.allResources = append(p.allResources, res)
+	p.idleResources = append(p.idleResources, res)
+	p.cond.L.Unlock()
+
+	return nil
+}
+
+// releaseAcquiredResource returns res to the the pool.
+func (p *Pool) releaseAcquiredResource(res *Resource, lastUsedNano int64) {
+	p.cond.L.Lock()
+
+	if !p.closed {
+		res.lastUsedNano = lastUsedNano
+		res.status = resourceStatusIdle
+		p.idleResources = append(p.idleResources, res)
+	} else {
+		p.allResources = removeResource(p.allResources, res)
+		go p.destructResourceValue(res.value)
+	}
+
+	p.cond.L.Unlock()
+	p.cond.Signal()
+}
+
+// Remove removes res from the pool and closes it. If res is not part of the
+// pool Remove will panic.
+func (p *Pool) destroyAcquiredResource(res *Resource) {
+	p.destructResourceValue(res.value)
+	p.cond.L.Lock()
+	p.allResources = removeResource(p.allResources, res)
+	p.cond.L.Unlock()
+	p.cond.Signal()
+}
+
+func (p *Pool) hijackAcquiredResource(res *Resource) {
+	p.cond.L.Lock()
+
+	p.allResources = removeResource(p.allResources, res)
+	res.status = resourceStatusHijacked
+	p.destructWG.Done() // not responsible for destructing hijacked resources
+
+	p.cond.L.Unlock()
+	p.cond.Signal()
+}
+
+func removeResource(slice []*Resource, res *Resource) []*Resource {
+	for i := range slice {
+		if slice[i] == res {
+			slice[i] = slice[len(slice)-1]
+			return slice[:len(slice)-1]
+		}
+	}
+
+	panic("BUG: removeResource could not find res in slice")
+}
+
+func (p *Pool) constructResourceValue(ctx context.Context) (interface{}, error) {
+	return p.constructor(ctx)
+}
+
+func (p *Pool) destructResourceValue(value interface{}) {
+	p.destructor(value)
+	p.destructWG.Done()
+}