crowdsec/pkg/hubops/download.go

package hubops

import (
	"context"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"time"

	"github.com/fatih/color"
	log "github.com/sirupsen/logrus"
	"gopkg.in/yaml.v3"

	"github.com/crowdsecurity/go-cs-lib/downloader"

	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
	"github.com/crowdsecurity/crowdsec/pkg/types"
)

// DownloadCommand handles the downloading of hub items.
// It ensures that items are fetched from the hub (or from the index file if it also has content)
// managing dependencies and verifying the integrity of downloaded content.
// This is used by "cscli install" and "cscli upgrade".
// Tainted items require the force parameter, local items are skipped.
type DownloadCommand struct {
	Item            *cwhub.Item
	Force           bool
	contentProvider cwhub.ContentProvider
}

func NewDownloadCommand(item *cwhub.Item, contentProvider cwhub.ContentProvider, force bool) *DownloadCommand {
	return &DownloadCommand{Item: item, Force: force, contentProvider: contentProvider}
}

func (c *DownloadCommand) Prepare(plan *ActionPlan) (bool, error) {
	i := c.Item

	if i.State.IsLocal() {
		log.Infof("%s - not downloading local item", i.FQName())
		return false, nil
	}

	// XXX: if it's tainted do we upgrade the dependencies anyway?
	if i.State.Tainted && !c.Force {
		log.Warnf("%s is tainted, use '--force' to overwrite", i.FQName())
		return false, nil
	}

	toDisable := make(map[*cwhub.Item]struct{})

	var disableKeys []*cwhub.Item

	if i.State.IsInstalled() {
		for sub := range i.CurrentDependencies().SubItems(plan.hub) {
			disableKeys = append(disableKeys, sub)
			toDisable[sub] = struct{}{}
		}
	}

	for sub := range i.LatestDependencies().SubItems(plan.hub) {
		if err := plan.AddCommand(NewDownloadCommand(sub, c.contentProvider, c.Force)); err != nil {
			return false, err
		}

		if i.State.IsInstalled() {
			// ensure the _new_ dependencies are installed too
			if err := plan.AddCommand(NewEnableCommand(sub, c.Force)); err != nil {
				return false, err
			}

			for _, sub2 := range disableKeys {
				if sub2 == sub {
					delete(toDisable, sub)
				}
			}
		}
	}

	for sub := range toDisable {
		if err := plan.AddCommand(NewDisableCommand(sub, c.Force)); err != nil {
			return false, err
		}
	}

	if i.State.IsDownloaded() && i.State.UpToDate {
		return false, nil
	}

	return true, nil
}

// The DataSet is a list of data sources required by an item (built from the data: section in the yaml).
type DataSet struct {
	Data []types.DataSource `yaml:"data,omitempty"`
}

// downloadDataSet downloads all the data files for an item.
func downloadDataSet(ctx context.Context, dataFolder string, force bool, reader io.Reader) (bool, error) {
	needReload := false

	dec := yaml.NewDecoder(reader)

	for {
		data := &DataSet{}

		if err := dec.Decode(data); err != nil {
			if errors.Is(err, io.EOF) {
				break
			}

			return needReload, fmt.Errorf("while reading file: %w", err)
		}

		for _, dataS := range data.Data {
			if dataS.SourceURL == "" {
				continue
			}

			// twopenny validation
			if u, err := url.Parse(dataS.SourceURL); err != nil {
				return false, err
			} else if u.Scheme == "" {
				return false, fmt.Errorf("a valid URL was expected (note: local items can download data too): %s", dataS.SourceURL)
			}

			// XXX: check context cancellation
			destPath, err := cwhub.SafePath(dataFolder, dataS.DestPath)
			if err != nil {
				return needReload, err
			}

			d := downloader.
				New().
				WithHTTPClient(cwhub.HubClient).
				ToFile(destPath).
				CompareContent().
				BeforeRequest(func(req *http.Request) {
					fmt.Printf("downloading %s\n", req.URL)
				}).
				WithLogger(log.WithField("url", dataS.SourceURL))

			if !force {
				d = d.WithLastModified().
					WithShelfLife(7 * 24 * time.Hour)
			}

			downloaded, err := d.Download(ctx, dataS.SourceURL)
			if err != nil {
				return needReload, fmt.Errorf("while getting data: %w", err)
			}

			needReload = needReload || downloaded
		}
	}

	return needReload, nil
}

func (c *DownloadCommand) Run(ctx context.Context, plan *ActionPlan) error {
	i := c.Item

	fmt.Printf("downloading %s\n", colorizeItemName(i.FQName()))

	// ensure that target file is within target dir
	finalPath, err := i.PathForDownload()
	if err != nil {
		return err
	}

	downloaded, _, err := i.FetchContentTo(ctx, c.contentProvider, finalPath)
	if err != nil {
		return fmt.Errorf("%s: %w", i.FQName(), err)
	}

	if downloaded {
		plan.ReloadNeeded = true
	}

	i.State.Tainted = false
	i.State.UpToDate = true

	// read content to get the list of data files
	reader, err := os.Open(finalPath)
	if err != nil {
		return fmt.Errorf("while opening %s: %w", finalPath, err)
	}

	defer reader.Close()

	needReload, err := downloadDataSet(ctx, plan.hub.GetDataDir(), c.Force, reader)
	if err != nil {
		return fmt.Errorf("while downloading data for %s: %w", i.FileName, err)
	}

	if needReload {
		plan.ReloadNeeded = true
	}

	return nil
}

func (c *DownloadCommand) OperationType() string {
	return "download"
}

func (c *DownloadCommand) ItemType() string {
	return c.Item.Type
}

func (c *DownloadCommand) Detail() string {
	i := c.Item

	version := color.YellowString(i.Version)

	if i.State.IsDownloaded() {
		version = c.Item.State.LocalVersion + " -> " + color.YellowString(i.Version)
	}

	return colorizeItemName(c.Item.Name) + " (" + version + ")"
}