diff --git a/cmd/scrapyard/main.go b/cmd/scrapyard/main.go new file mode 100644 index 0000000..8eae1a5 --- /dev/null +++ b/cmd/scrapyard/main.go @@ -0,0 +1,134 @@ +package main + +import ( + "github.com/dgtized/scrapyard/internal/fileyard" + "github.com/dgtized/scrapyard/internal/parser" + "github.com/sirupsen/logrus" + "gopkg.in/alecthomas/kingpin.v2" + "os" +) + +func main() { + var ( + yard string + + log logrus.FieldLogger = &logrus.Logger{ + Formatter: &logrus.TextFormatter{ + DisableLevelTruncation: true, + FullTimestamp: true, + }, + Out: os.Stdout, + } + ) + + app := scrapyard(&log, &yard) + + crush(&app, &log, &yard) + junk(&app, &log, &yard) + search(&app, &log, &yard) + store(&app, &log, &yard) + + kingpin.MustParse(app.Parse(os.Args[1:])) +} + +func initialize(log *logrus.FieldLogger, verbose *bool, yard *string) kingpin.Action { + return func(_ *kingpin.ParseContext) error { + if *verbose { + (*log).(*logrus.Logger).Level = logrus.DebugLevel + } else { + (*log).(*logrus.Logger).Level = logrus.WarnLevel + } + *log = (*log).WithField("yard", *yard) + + (*log).Info("Creating scrapyard if it doesn't exist") + return os.MkdirAll(*yard, os.ModePerm) + } +} + +func scrapyard(log *logrus.FieldLogger, yard *string) kingpin.Application { + var verbose bool + + app := kingpin. + New("scrapyard", "Simple caching tool for faster CI builds"). + Action(initialize(log, &verbose, yard)) + app. + Flag("verbose", ""). + Short('v'). + BoolVar(&verbose) + app. + Flag("yard", "The directory the scrapyard is stored in."). + Short('y'). + Default("/tmp/scrapyard"). + StringVar(yard) + + return *app +} + +func crush(app *kingpin.Application, log *logrus.FieldLogger, yard *string) { + app. + Command("crush", "Crush a yard to scrap"). + Action(fileyard.Crush(log, yard)) +} + +func junk(app *kingpin.Application, log *logrus.FieldLogger, yard *string) { + var keys []string + + command := app. + Command("junk", "Junk keys in a scrapyard"). + Action(fileyard.Junk(&keys, log, yard)) + parser.CsvVar( + &keys, + command. + Flag("keys", "Keys to junk in the scrapyard"). + Short('k'). + Required(), + ) +} + +func search(app *kingpin.Application, log *logrus.FieldLogger, yard *string) { + var ( + keys []string + paths []string + ) + + command := app. + Command("search", "Search for paths in a scrapyard"). + Action(fileyard.Search(&keys, log, &paths, yard)) + parser.CsvVar( + &keys, + command. + Flag("keys", "Keys to search in order of preference the scrapyard"). + Short('k'). + Required(), + ) + parser.CsvVar( + &paths, + command. + Flag("paths", "Paths to search in the scrapyard"). + Short('p'). + Required(), + ) +} + +func store(app *kingpin.Application, log *logrus.FieldLogger, yard *string) { + var ( + key string + paths []string + ) + + command := app. + Command("store", "Store paths in a scrapyard"). + Action(fileyard.Store(&key, log, &paths, yard)) + command. + Flag("key", "Key to store in the scrapyard"). + Short('k'). + Required(). + StringVar(&key) + parser.CsvVar( + &paths, + command. + Flag("paths", "Paths to store in the scrapyard"). + Short('p'). + Required(), + ) +} diff --git a/internal/fileyard/crush.go b/internal/fileyard/crush.go new file mode 100644 index 0000000..54cd7b6 --- /dev/null +++ b/internal/fileyard/crush.go @@ -0,0 +1,40 @@ +package fileyard + +import ( + "github.com/sirupsen/logrus" + "gopkg.in/alecthomas/kingpin.v2" + "os" + "path/filepath" + "time" +) + +// Crush a yard to scrap. +func Crush(log *logrus.FieldLogger, yard *string) kingpin.Action { + return func(_ *kingpin.ParseContext) error { + (*log).Info("Crushing the yard to scrap!") + return filepath.Walk(*yard, func(tarball string, info os.FileInfo, err error) error { + if err != nil { + (*log).Debug("Yard does not exist") + return nil + } else if *yard == tarball { + // filepath.Walk also gives us the directory, ignore it. + return nil + } + + modtime := info.ModTime() + log := (*log).WithFields( + logrus.Fields{ + "modtime": modtime, + "tarball": tarball, + }, + ) + + if modtime.Before(time.Now().AddDate(0, 0, -20)) { + log.Info("Crushing tarball") + return os.Remove(tarball) + } + log.Debug("Keeping tarball") + return nil + }) + } +} diff --git a/internal/fileyard/junk.go b/internal/fileyard/junk.go new file mode 100644 index 0000000..b3bc425 --- /dev/null +++ b/internal/fileyard/junk.go @@ -0,0 +1,25 @@ +package fileyard + +import ( + "github.com/dgtized/scrapyard/internal/key" + "github.com/sirupsen/logrus" + "gopkg.in/alecthomas/kingpin.v2" + "os" +) + +// Junk keys in a scrapyard. +func Junk(keys *[]string, log *logrus.FieldLogger, yard *string) kingpin.Action { + return func(_ *kingpin.ParseContext) error { + log := (*log).WithField("keys", *keys) + + log.Info("Junking keys") + for _, k := range *keys { + path := key.ToPath(*yard, k, ".tgz", log) + log.WithField("path", path).Debug("Removing key") + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return err + } + } + return nil + } +} diff --git a/internal/fileyard/search.go b/internal/fileyard/search.go new file mode 100644 index 0000000..c567983 --- /dev/null +++ b/internal/fileyard/search.go @@ -0,0 +1,63 @@ +package fileyard + +import ( + "github.com/dgtized/scrapyard/internal/key" + "github.com/dgtized/scrapyard/internal/pack" + "github.com/sirupsen/logrus" + "gopkg.in/alecthomas/kingpin.v2" + "os" + "path/filepath" + "time" +) + +// Search for paths in a scrapyard. +func Search(keys *[]string, log *logrus.FieldLogger, paths *[]string, yard *string) kingpin.Action { + return func(_ *kingpin.ParseContext) error { + log := (*log).WithFields( + logrus.Fields{ + "keys": *keys, + "paths": *paths, + }, + ) + + log.Info("Searching for keys") + for _, k := range *keys { + path := key.ToPath(*yard, k, "*", log) + glob, errGlob := filepath.Glob(path) + if errGlob != nil { + return errGlob + } + if len(glob) == 0 { + continue + } + + var ( + latestCache string + latestTime time.Time + + scanLog = log.WithFields( + logrus.Fields{ + "glob": glob, + "path": path, + }, + ) + ) + + scanLog.Debug("Scanning") + for _, cache := range glob { + cacheInfo, err := os.Stat(cache) + if err != nil { + return err + } + if modTime := cacheInfo.ModTime(); modTime.After(latestTime) { + latestCache = cache + latestTime = modTime + } + } + return pack.Restore(scanLog, latestCache, *paths) + } + + log.Info("Unable to find key(s)") + return nil + } +} diff --git a/internal/fileyard/store.go b/internal/fileyard/store.go new file mode 100644 index 0000000..3fb5329 --- /dev/null +++ b/internal/fileyard/store.go @@ -0,0 +1,24 @@ +package fileyard + +import ( + "github.com/dgtized/scrapyard/internal/key" + "github.com/dgtized/scrapyard/internal/pack" + "github.com/sirupsen/logrus" + "gopkg.in/alecthomas/kingpin.v2" +) + +// Store paths in a scrapyard. +func Store(k *string, log *logrus.FieldLogger, paths *[]string, yard *string) kingpin.Action { + return func(_ *kingpin.ParseContext) error { + log := (*log).WithFields( + logrus.Fields{ + "key": *k, + "paths": *paths, + }, + ) + + log.Info("Storing keys") + keyPath := key.ToPath(*yard, *k, ".tgz", log) + return pack.Save(log, keyPath, *paths) + } +} diff --git a/internal/key/key.go b/internal/key/key.go new file mode 100644 index 0000000..f1cd0ec --- /dev/null +++ b/internal/key/key.go @@ -0,0 +1,39 @@ +package key + +import ( + "crypto/sha1" + "fmt" + "github.com/sirupsen/logrus" + "io/ioutil" + "path/filepath" + "regexp" + "strings" +) + +var interpolater = regexp.MustCompile(`#\([^}]+\)`) + +// checksum interpolates a template string with the checksums of files +// +// For example, `checksum("foo-#(main.go)")` would interpolate the checksum of +// the contents of `main.go` to make something like `"foo-123abc654"`. +func checksum(key string, log logrus.FieldLogger) string { + return interpolater.ReplaceAllStringFunc(key, func(m string) string { + match := strings.TrimSpace(m[2 : len(m)-1]) + matchLog := log.WithField("match", match) + + bytes, err := ioutil.ReadFile(match) + if err != nil { + matchLog.Debug("File does not exist, ignoring checksum") + return "" + } + matchLog.Debug("Including sha1 of file") + hash := sha1.New() + hash.Write(bytes) + return fmt.Sprintf("%x", hash.Sum(nil)) + }) +} + +// ToPath converts a Key into a file path by interpolating any checksums. +func ToPath(yard, key, extension string, log logrus.FieldLogger) string { + return filepath.Join(yard, checksum(key, log)+extension) +} diff --git a/internal/pack/pack.go b/internal/pack/pack.go new file mode 100644 index 0000000..de9cf50 --- /dev/null +++ b/internal/pack/pack.go @@ -0,0 +1,82 @@ +package pack + +import ( + "github.com/sirupsen/logrus" + "io/ioutil" + "os" + "os/exec" + "strings" +) + +// Restore the files in a tarball. +func Restore(log logrus.FieldLogger, cache string, paths []string) error { + duCommand := exec.Command("du", "-sh", strings.Join(paths, " ")) + tarCommand := exec.Command("tar", "zxf", cache) + restoreLog := log.WithFields( + logrus.Fields{ + "cache": cache, + "duCommand": duCommand.Args, + "paths": paths, + "tarCommand": tarCommand.Args, + }, + ) + + restoreLog.Debug("Found scrap in cache") + restoreLog.Info("Executing tar command") + if err := tarCommand.Run(); err != nil { + return err + } + if len(paths) > 0 { + duOutput, err := duCommand.Output() + if err != nil { + return err + } + restoreLog. + WithField("duOutput", strings.TrimSpace(string(duOutput))). + Info("Restored cache") + } + return nil +} + +// Save the paths in the given tarball name. +func Save(log logrus.FieldLogger, cache string, paths []string) error { + file, errTemp := ioutil.TempFile("", "scrapyard") + if errTemp != nil { + return errTemp + } + fileName := file.Name() + defer os.Remove(fileName) + + lsCommand := exec.Command("ls", "-lah", cache) + mvCommand := exec.Command("mv", fileName, cache) + tarCommand := exec.Command("tar", append([]string{"czf", fileName}, paths...)...) + touchCommand := exec.Command("touch", cache) + saveLog := log.WithFields( + logrus.Fields{ + "cache": cache, + "tarCommand": tarCommand.Args, + "tempFile": fileName, + "touchCommand": touchCommand.Args, + }, + ) + + saveLog.Debug("Executing tar command") + if err := tarCommand.Run(); err != nil { + return err + } + if err := mvCommand.Run(); err != nil { + return err + } + if err := touchCommand.Run(); err != nil { + return err + } + lsOutput, err := lsCommand.Output() + if err != nil { + return err + } + + saveLog. + WithField("lsOutput", strings.TrimSpace(string(lsOutput))). + Info("Created cache") + return nil +} diff --git a/internal/parser/csv.go b/internal/parser/csv.go new file mode 100644 index 0000000..11cba21 --- /dev/null +++ b/internal/parser/csv.go @@ -0,0 +1,40 @@ +package parser + +import ( + "fmt" + "gopkg.in/alecthomas/kingpin.v2" + "strings" +) + +type csvValue struct { + values *[]string +} + +func (c csvValue) Get() interface{} { + return c.values +} + +// IsCumulative allows repeated flags. +// Without this, only one flag will work. +func (c csvValue) IsCumulative() bool { + return true +} + +func (c *csvValue) Set(value string) error { + *c.values = append(*c.values, strings.Split(value, ",")...) + + return nil +} + +func (c csvValue) String() string { + return fmt.Sprintf("%v", c.values) +} + +// CsvVar is a custom parser for strings variables +// It's effectively `kingpin.StringsVar` with support for comma-separated values. +// +// For example, if you have a flag `foo`, +// it will parse `--foo bar --foo baz,qux --foo gar` to `[bar baz qux gar]`. +func CsvVar(target *[]string, s kingpin.Settings) { + s.SetValue(&csvValue{values: target}) +} diff --git a/test.sh b/test.sh index 1bec22d..4891fe1 100755 --- a/test.sh +++ b/test.sh @@ -2,6 +2,7 @@ set -exuo pipefail +SCRAPYARD=${1:-./scrapyard.rb} VERBOSE="-v" rm -rf a_dir a_file scrapyard @@ -32,7 +33,7 @@ $SCRAPYARD $VERBOSE store -k "key-#(a_file)" -y $YARD -p a_dir && $SCRAPYARD $VERBOSE store -k "key" -y $YARD -p a_dir && echo "SUCCESS" -$SCRAPYARD $VERBOSE junk -k "key-#(a_file)" -y $YARD -p a_dir && +$SCRAPYARD $VERBOSE junk -k "key-#(a_file)" -y $YARD && echo "SUCCESS" $SCRAPYARD $VERBOSE search -k "key-#(a_file)","key","k" -y $YARD -p a_dir &&