-
-
Notifications
You must be signed in to change notification settings - Fork 16
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
10 changed files
with
322 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
/test |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,69 @@ | ||
package indir | ||
|
||
import ( | ||
"context" | ||
"github.com/reddec/dfq" | ||
"github.com/reddec/trusted-cgi/types" | ||
"github.com/tinylib/msgp/msgp" | ||
"io" | ||
) | ||
|
||
func New(directory string) (*inDirQueue, error) { | ||
back, err := dfq.Open(directory) | ||
if err != nil { | ||
return nil, err | ||
} | ||
return &inDirQueue{backend: back}, nil | ||
} | ||
|
||
type inDirQueue struct { | ||
backend dfq.Queue | ||
} | ||
|
||
func (queue *inDirQueue) Put(ctx context.Context, request *types.Request) error { | ||
defer request.Body.Close() | ||
return queue.backend.Stream(func(out io.Writer) error { | ||
w := msgp.NewWriter(out) | ||
err := request.EncodeMsg(w) | ||
if err != nil { | ||
return err | ||
} | ||
_, err = io.Copy(w, request.Body) | ||
if err != nil { | ||
return err | ||
} | ||
return w.Flush() | ||
}) | ||
} | ||
|
||
func (queue *inDirQueue) Peek(ctx context.Context) (*types.Request, error) { | ||
in, err := queue.backend.Wait(ctx) | ||
if err != nil { | ||
return nil, err | ||
} | ||
reader := msgp.NewReader(in) | ||
var head types.Request | ||
err = head.DecodeMsg(reader) | ||
if err != nil { | ||
_ = in.Close() | ||
return nil, err | ||
} | ||
return head.WithBody(&readCloser{reader: reader.R, closer: in}), nil | ||
} | ||
|
||
func (queue *inDirQueue) Commit(ctx context.Context) error { | ||
return queue.backend.Commit() | ||
} | ||
|
||
type readCloser struct { | ||
reader io.Reader | ||
closer io.Closer | ||
} | ||
|
||
func (rc *readCloser) Read(p []byte) (n int, err error) { | ||
return rc.reader.Read(p) | ||
} | ||
|
||
func (rc *readCloser) Close() error { | ||
return rc.closer.Close() | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,109 @@ | ||
package inmemory | ||
|
||
import ( | ||
"bytes" | ||
"context" | ||
"fmt" | ||
"github.com/reddec/trusted-cgi/types" | ||
"io/ioutil" | ||
"sync" | ||
"sync/atomic" | ||
) | ||
|
||
type item struct { | ||
payload types.Request | ||
data []byte | ||
} | ||
|
||
func (it *item) makeRequest() *types.Request { | ||
cp := it.payload // shallow copy | ||
cp.Body = ioutil.NopCloser(bytes.NewReader(it.data)) | ||
return &cp | ||
} | ||
|
||
// Dummy implementation of in-memory queue with pre-allocated channel for buffering. | ||
// It's safe to call Close several times | ||
func New(size int) *memoryQueue { | ||
return &memoryQueue{ | ||
closed: make(chan struct{}), | ||
stream: make(chan item, size), | ||
} | ||
} | ||
|
||
type memoryQueue struct { | ||
closed chan struct{} | ||
stream chan item | ||
peeked struct { | ||
value item | ||
available bool | ||
} | ||
rlock sync.Mutex | ||
closing int32 | ||
} | ||
|
||
func (queue *memoryQueue) Put(ctx context.Context, request *types.Request) error { | ||
defer request.Body.Close() | ||
select { | ||
case <-queue.closed: | ||
return fmt.Errorf("put: queue is closed") | ||
default: | ||
} | ||
data, err := ioutil.ReadAll(request.Body) | ||
if err != nil { | ||
return fmt.Errorf("put: read body: %w", err) | ||
} | ||
select { | ||
case <-queue.closed: | ||
return fmt.Errorf("put: queue is closed") | ||
case <-ctx.Done(): | ||
return fmt.Errorf("put: context closed: %w", ctx.Err()) | ||
case queue.stream <- item{ | ||
payload: *request, | ||
data: data, | ||
}: | ||
return nil | ||
} | ||
} | ||
|
||
func (queue *memoryQueue) Peek(ctx context.Context) (*types.Request, error) { | ||
select { | ||
case <-queue.closed: | ||
return nil, fmt.Errorf("peek: queue is closed") | ||
default: | ||
} | ||
queue.rlock.Lock() | ||
defer queue.rlock.Unlock() | ||
if !queue.peeked.available { | ||
select { | ||
case <-queue.closed: | ||
return nil, fmt.Errorf("peek: queue is closed") | ||
case <-ctx.Done(): | ||
return nil, fmt.Errorf("peek: context closed: %w", ctx.Err()) | ||
case item := <-queue.stream: | ||
queue.peeked.value = item | ||
queue.peeked.available = true | ||
} | ||
} | ||
return queue.peeked.value.makeRequest(), nil | ||
} | ||
|
||
func (queue *memoryQueue) Commit(ctx context.Context) error { | ||
select { | ||
case <-queue.closed: | ||
return fmt.Errorf("commit: queue is closed") | ||
default: | ||
} | ||
queue.rlock.Lock() | ||
defer queue.rlock.Unlock() | ||
queue.peeked.available = false | ||
return nil | ||
} | ||
|
||
func (queue *memoryQueue) Done() <-chan struct{} { return queue.closed } | ||
|
||
func (queue *memoryQueue) Close() { | ||
if atomic.CompareAndSwapInt32(&queue.closing, 0, 1) { | ||
close(queue.closed) | ||
close(queue.stream) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
package queue | ||
|
||
import ( | ||
"context" | ||
"github.com/reddec/trusted-cgi/types" | ||
) | ||
|
||
// Thread-safe FIFO queue designed for one multiple concurrent writers and single consumer. | ||
// Queue should store somewhere request body. | ||
type Queue interface { | ||
// Put request to queue | ||
Put(ctx context.Context, request *types.Request) error | ||
// Peek oldest request or wait till new data arrived/context expiration | ||
Peek(ctx context.Context) (*types.Request, error) | ||
// Commit (remove) oldest record | ||
Commit(ctx context.Context) error | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,107 @@ | ||
package queue_test | ||
|
||
import ( | ||
"bytes" | ||
"context" | ||
"github.com/google/uuid" | ||
"github.com/reddec/trusted-cgi/queue" | ||
"github.com/reddec/trusted-cgi/queue/indir" | ||
"github.com/reddec/trusted-cgi/queue/inmemory" | ||
"github.com/reddec/trusted-cgi/types" | ||
"github.com/stretchr/testify/assert" | ||
"io/ioutil" | ||
"testing" | ||
) | ||
|
||
func testPutPeek(ctx context.Context, t *testing.T, queue queue.Queue) *types.Request { | ||
payload := uuid.New().String() | ||
|
||
req := &types.Request{ | ||
Method: "POST", | ||
URL: "http://example.com:8889/sample/" + payload, | ||
Path: "/sample/" + payload, | ||
RemoteAddress: "127.0.0.2:9992", | ||
Form: map[string]string{ | ||
"USER": "user1", | ||
}, | ||
Headers: map[string]string{ | ||
"Content-Type": "text/plain", | ||
}, | ||
Body: ioutil.NopCloser(bytes.NewBufferString(payload)), | ||
} | ||
err := queue.Put(ctx, req) | ||
if !assert.NoError(t, err) { | ||
return req | ||
} | ||
|
||
v, err := queue.Peek(ctx) | ||
if !assert.NoError(t, err) { | ||
return req | ||
} | ||
data, err := ioutil.ReadAll(v.Body) | ||
if !assert.NoError(t, err) { | ||
return req | ||
} | ||
assert.Equal(t, string(data), payload) | ||
assert.Equal(t, v.WithBody(nil), req.WithBody(nil)) | ||
return req | ||
} | ||
|
||
func TestInMemory(t *testing.T) { | ||
ctx := context.Background() | ||
q := inmemory.New(1) | ||
defer q.Close() | ||
|
||
req := testPutPeek(ctx, t, q) | ||
// try again - should be same result | ||
v2, err := q.Peek(ctx) | ||
if !assert.NoError(t, err) { | ||
return | ||
} | ||
assert.Equal(t, v2.WithBody(nil), req.WithBody(nil)) | ||
|
||
err = q.Commit(ctx) | ||
if !assert.NoError(t, err) { | ||
return | ||
} | ||
// put again | ||
testPutPeek(ctx, t, q) | ||
|
||
q.Close() | ||
|
||
_, err = q.Peek(ctx) | ||
assert.Error(t, err) | ||
|
||
var closed bool | ||
select { | ||
case <-q.Done(): | ||
closed = true | ||
default: | ||
|
||
} | ||
assert.True(t, closed, "should be closed") | ||
|
||
} | ||
|
||
func TestInDir(t *testing.T) { | ||
ctx := context.Background() | ||
q, err := indir.New("test/queue") | ||
if !assert.NoError(t, err) { | ||
return | ||
} | ||
|
||
req := testPutPeek(ctx, t, q) | ||
// try again - should be same result | ||
v2, err := q.Peek(ctx) | ||
if !assert.NoError(t, err) { | ||
return | ||
} | ||
assert.Equal(t, v2.WithBody(nil), req.WithBody(nil)) | ||
|
||
err = q.Commit(ctx) | ||
if !assert.NoError(t, err) { | ||
return | ||
} | ||
// put again | ||
testPutPeek(ctx, t, q) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters