Go rewrite

cmd/scrapyard/main.go

@@ -0,0 +1,134 @@
+package main
+
+import (
+	"github.com/dgtized/scrapyard/internal/fileyard"
+	"github.com/dgtized/scrapyard/internal/parser"
+	"github.com/sirupsen/logrus"
+	"gopkg.in/alecthomas/kingpin.v2"
+	"os"
+)
+
+func main() {
+	var (
+		yard string
+
+		log logrus.FieldLogger = &logrus.Logger{
+			Formatter: &logrus.TextFormatter{
+				DisableLevelTruncation: true,
+				FullTimestamp:          true,
+			},
+			Out: os.Stdout,
+		}
+	)
+
+	app := scrapyard(&log, &yard)
+
+	crush(&app, &log, &yard)
+	junk(&app, &log, &yard)
+	search(&app, &log, &yard)
+	store(&app, &log, &yard)
+
+	kingpin.MustParse(app.Parse(os.Args[1:]))
+}
+
+func initialize(log *logrus.FieldLogger, verbose *bool, yard *string) kingpin.Action {
+	return func(_ *kingpin.ParseContext) error {
+		if *verbose {
+			(*log).(*logrus.Logger).Level = logrus.DebugLevel
+		} else {
+			(*log).(*logrus.Logger).Level = logrus.WarnLevel
+		}
+		*log = (*log).WithField("yard", *yard)
+
+		(*log).Info("Creating scrapyard if it doesn't exist")
+		return os.MkdirAll(*yard, os.ModePerm)
+	}
+}
+
+func scrapyard(log *logrus.FieldLogger, yard *string) kingpin.Application {
+	var verbose bool
+
+	app := kingpin.
+		New("scrapyard", "Simple caching tool for faster CI builds").
+		Action(initialize(log, &verbose, yard))
+	app.
+		Flag("verbose", "").
+		Short('v').
+		BoolVar(&verbose)
+	app.
+		Flag("yard", "The directory the scrapyard is stored in.").
+		Short('y').
+		Default("/tmp/scrapyard").
+		StringVar(yard)
+
+	return *app
+}
+
+func crush(app *kingpin.Application, log *logrus.FieldLogger, yard *string) {
+	app.
+		Command("crush", "Crush a yard to scrap").
+		Action(fileyard.Crush(log, yard))
+}
+
+func junk(app *kingpin.Application, log *logrus.FieldLogger, yard *string) {
+	var keys []string
+
+	command := app.
+		Command("junk", "Junk keys in a scrapyard").
+		Action(fileyard.Junk(&keys, log, yard))
+	parser.CsvVar(
+		&keys,
+		command.
+			Flag("keys", "Keys to junk in the scrapyard").
+			Short('k').
+			Required(),
+	)
+}
+
+func search(app *kingpin.Application, log *logrus.FieldLogger, yard *string) {
+	var (
+		keys  []string
+		paths []string
+	)
+
+	command := app.
+		Command("search", "Search for paths in a scrapyard").
+		Action(fileyard.Search(&keys, log, &paths, yard))
+	parser.CsvVar(
+		&keys,
+		command.
+			Flag("keys", "Keys to search in order of preference the scrapyard").
+			Short('k').
+			Required(),
+	)
+	parser.CsvVar(
+		&paths,
+		command.
+			Flag("paths", "Paths to search in the scrapyard").
+			Short('p').
+			Required(),
+	)
+}
+
+func store(app *kingpin.Application, log *logrus.FieldLogger, yard *string) {
+	var (
+		key   string
+		paths []string
+	)
+
+	command := app.
+		Command("store", "Store paths in a scrapyard").
+		Action(fileyard.Store(&key, log, &paths, yard))
+	command.
+		Flag("key", "Key to store in the scrapyard").
+		Short('k').
+		Required().
+		StringVar(&key)
+	parser.CsvVar(
+		&paths,
+		command.
+			Flag("paths", "Paths to store in the scrapyard").
+			Short('p').
+			Required(),
+	)
+}

internal/fileyard/crush.go

@@ -0,0 +1,40 @@
+package fileyard
+
+import (
+	"github.com/sirupsen/logrus"
+	"gopkg.in/alecthomas/kingpin.v2"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+// Crush a yard to scrap.
+func Crush(log *logrus.FieldLogger, yard *string) kingpin.Action {
+	return func(_ *kingpin.ParseContext) error {
+		(*log).Info("Crushing the yard to scrap!")
+		return filepath.Walk(*yard, func(tarball string, info os.FileInfo, err error) error {
+			if err != nil {
+				(*log).Debug("Yard does not exist")
+				return nil
+			} else if *yard == tarball {
+				// filepath.Walk also gives us the directory, ignore it.
+				return nil
+			}
+
+			modtime := info.ModTime()
+			log := (*log).WithFields(
+				logrus.Fields{
+					"modtime": modtime,
+					"tarball": tarball,
+				},
+			)
+
+			if modtime.Before(time.Now().AddDate(0, 0, -20)) {
+				log.Info("Crushing tarball")
+				return os.Remove(tarball)
+			}
+			log.Debug("Keeping tarball")
+			return nil
+		})
+	}
+}

internal/fileyard/junk.go

@@ -0,0 +1,25 @@
+package fileyard
+
+import (
+	"github.com/dgtized/scrapyard/internal/key"
+	"github.com/sirupsen/logrus"
+	"gopkg.in/alecthomas/kingpin.v2"
+	"os"
+)
+
+// Junk keys in a scrapyard.
+func Junk(keys *[]string, log *logrus.FieldLogger, yard *string) kingpin.Action {
+	return func(_ *kingpin.ParseContext) error {
+		log := (*log).WithField("keys", *keys)
+
+		log.Info("Junking keys")
+		for _, k := range *keys {
+			path := key.ToPath(*yard, k, ".tgz", log)
+			log.WithField("path", path).Debug("Removing key")
+			if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+				return err
+			}
+		}
+		return nil
+	}
+}

internal/fileyard/search.go

@@ -0,0 +1,63 @@
+package fileyard
+
+import (
+	"github.com/dgtized/scrapyard/internal/key"
+	"github.com/dgtized/scrapyard/internal/pack"
+	"github.com/sirupsen/logrus"
+	"gopkg.in/alecthomas/kingpin.v2"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+// Search for paths in a scrapyard.
+func Search(keys *[]string, log *logrus.FieldLogger, paths *[]string, yard *string) kingpin.Action {
+	return func(_ *kingpin.ParseContext) error {
+		log := (*log).WithFields(
+			logrus.Fields{
+				"keys":  *keys,
+				"paths": *paths,
+			},
+		)
+
+		log.Info("Searching for keys")
+		for _, k := range *keys {
+			path := key.ToPath(*yard, k, "*", log)
+			glob, errGlob := filepath.Glob(path)
+			if errGlob != nil {
+				return errGlob
+			}
+			if len(glob) == 0 {
+				continue
+			}
+
+			var (
+				latestCache string
+				latestTime  time.Time
+
+				scanLog = log.WithFields(
+					logrus.Fields{
+						"glob": glob,
+						"path": path,
+					},
+				)
+			)
+
+			scanLog.Debug("Scanning")
+			for _, cache := range glob {
+				cacheInfo, err := os.Stat(cache)
+				if err != nil {
+					return err
+				}
+				if modTime := cacheInfo.ModTime(); modTime.After(latestTime) {
+					latestCache = cache
+					latestTime = modTime
+				}
+			}
+			return pack.Restore(scanLog, latestCache, *paths)
+		}
+
+		log.Info("Unable to find key(s)")
+		return nil
+	}
+}

internal/fileyard/store.go

@@ -0,0 +1,24 @@
+package fileyard
+
+import (
+	"github.com/dgtized/scrapyard/internal/key"
+	"github.com/dgtized/scrapyard/internal/pack"
+	"github.com/sirupsen/logrus"
+	"gopkg.in/alecthomas/kingpin.v2"
+)
+
+// Store paths in a scrapyard.
+func Store(k *string, log *logrus.FieldLogger, paths *[]string, yard *string) kingpin.Action {
+	return func(_ *kingpin.ParseContext) error {
+		log := (*log).WithFields(
+			logrus.Fields{
+				"key":   *k,
+				"paths": *paths,
+			},
+		)
+
+		log.Info("Storing keys")
+		keyPath := key.ToPath(*yard, *k, ".tgz", log)
+		return pack.Save(log, keyPath, *paths)
+	}
+}

internal/key/key.go

@@ -0,0 +1,39 @@
+package key
+
+import (
+	"crypto/sha1"
+	"fmt"
+	"github.com/sirupsen/logrus"
+	"io/ioutil"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+var interpolater = regexp.MustCompile(`#\([^}]+\)`)
+
+// checksum interpolates a template string with the checksums of files
+//
+// For example, `checksum("foo-#(main.go)")` would interpolate the checksum of
+// the contents of `main.go` to make something like `"foo-123abc654"`.
+func checksum(key string, log logrus.FieldLogger) string {
+	return interpolater.ReplaceAllStringFunc(key, func(m string) string {
+		match := strings.TrimSpace(m[2 : len(m)-1])
+		matchLog := log.WithField("match", match)
+
+		bytes, err := ioutil.ReadFile(match)
+		if err != nil {
+			matchLog.Debug("File does not exist, ignoring checksum")
+			return ""
+		}
+		matchLog.Debug("Including sha1 of file")
+		hash := sha1.New()
+		hash.Write(bytes)
+		return fmt.Sprintf("%x", hash.Sum(nil))
+	})
+}
+
+// ToPath converts a Key into a file path by interpolating any checksums.
+func ToPath(yard, key, extension string, log logrus.FieldLogger) string {
+	return filepath.Join(yard, checksum(key, log)+extension)
+}