ralsina/nano-run
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

use sequential ids

Browse Source
This commit is contained in:
Alexander Baryshnikov 2020-09-28 23:36:20 +08:00
parent 4698e52dca
commit 53e1acdbe8
2 changed files with 19 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
services/meta/micrometa/request_meta_storage.go
View File

@ -98,7 +98,9 @@ func (rms *MicroMeta) Close() error {
func (rms *MicroMeta) Iterate(handler func(id string, record meta.Request) error) error { func (rms *MicroMeta) Iterate(handler func(id string, record meta.Request) error) error {
return rms.db.View(func(txn *badger.Txn) error { return rms.db.View(func(txn *badger.Txn) error {
iter := txn.NewIterator(badger.DefaultIteratorOptions) cfg := badger.DefaultIteratorOptions
cfg.Reverse = true
iter := txn.NewIterator(cfg)
iter.Rewind() iter.Rewind()
defer iter.Close() defer iter.Close()
for iter.Valid() { for iter.Valid() {

22
worker/worker.go
View File

@ -9,13 +9,14 @@ import (
"net/http" "net/http"
"os" "os"
"path/filepath" "path/filepath"
"regexp"
"runtime" "runtime"
"strconv" "strconv"
"strings"
"sync" "sync"
"sync/atomic"
"time" "time"
"github.com/google/uuid"
"nano-run/services/blob" "nano-run/services/blob"
"nano-run/services/blob/fsblob" "nano-run/services/blob/fsblob"
"nano-run/services/meta" "nano-run/services/meta"
@ -41,9 +42,12 @@ func Default(location string) (*Worker, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
valid, err := regexp.Compile("^[a-zA-Z0-9-]+$")
if err != nil {
return nil, err
}
storage := fsblob.NewCheck(path, func(id string) bool { storage := fsblob.NewCheck(path, func(id string) bool {
_, err := uuid.Parse(id) return valid.MatchString(id)
return err == nil
}) })
taskQueue, err := microqueue.NewMicroQueue(filepath.Join(location, "queue")) taskQueue, err := microqueue.NewMicroQueue(filepath.Join(location, "queue"))
@ -110,10 +114,16 @@ type Worker struct {
concurrency int concurrency int
reloadMeta chan struct{} reloadMeta chan struct{}
interval time.Duration interval time.Duration
sequence int64
} }
func (mgr *Worker) init() error { func (mgr *Worker) init() error {
return mgr.meta.Iterate(func(id string, record meta.Request) error { return mgr.meta.Iterate(func(id string, record meta.Request) error {
if v, err := strconv.ParseInt(strings.TrimLeft(id, "0"), 10, 64); err == nil && v > mgr.sequence {
mgr.sequence = v
} else if err != nil {
log.Println("found broken id:", id, "-", err)
}
if !record.Complete { if !record.Complete {
log.Println("found incomplete job", id) log.Println("found incomplete job", id)
return mgr.queue.Push([]byte(id)) return mgr.queue.Push([]byte(id))
@ -276,7 +286,7 @@ func (mgr *Worker) call(ctx context.Context, requestID string, info *meta.Reques
return err return err
} }
defer req.Body.Close() defer req.Body.Close()
attemptID := uuid.New().String() attemptID := fmt.Sprintf("%016d", len(info.Attempts)+1)
req.Header.Set("X-Correlation-Id", requestID) req.Header.Set("X-Correlation-Id", requestID)
req.Header.Set("X-Attempt-Id", attemptID) req.Header.Set("X-Attempt-Id", attemptID)
@ -411,7 +421,7 @@ func (mgr *Worker) requeueItem(ctx context.Context, id string, info *meta.Reques
} }
func (mgr *Worker) saveRequest(req *http.Request) (string, error) { func (mgr *Worker) saveRequest(req *http.Request) (string, error) {
id := uuid.New().String() id := fmt.Sprintf("%016d", atomic.AddInt64(&mgr.sequence, 1))
err := mgr.blob.Push(id, func(out io.Writer) error { err := mgr.blob.Push(id, func(out io.Writer) error {
_, err := io.Copy(out, req.Body) _, err := io.Copy(out, req.Body)
return err return err