Add 'downloads' field in pkg.info

src/bpmlib/compilation.go

@@ -1,11 +1,15 @@
 package bpmlib
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"errors"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path"
+	"slices"
 	"strconv"
 	"strings"
 	"syscall"
@@ -116,6 +120,12 @@ func CompileSourcePackage(archiveFilename, outputDirectory string, skipChecks bo
 		return nil, err
 	}
 
+	// Download files
+	err = downloadPackageFiles(bpmpkg.PkgInfo, tempDirectory)
+	if err != nil {
+		return nil, err
+	}
+
 	// Setup environment for commands
 	env := os.Environ()
 	env = append(env, "HOME="+tempDirectory)
@@ -251,8 +261,9 @@ func CompileSourcePackage(archiveFilename, outputDirectory string, skipChecks bo
 		pkgInfo.Arch = pkg.OutputArch
 		pkgInfo.OutputArch = ""
 
-		// Remove split package field
+		// Remove split package and downloads fields
 		pkgInfo.SplitPackages = nil
+		pkgInfo.Downloads = nil
 
 		// Marshal package info
 		pkgInfoBytes, err := yaml.Marshal(pkgInfo)
@@ -323,3 +334,143 @@ func CompileSourcePackage(archiveFilename, outputDirectory string, skipChecks bo
 
 	return outputBpmPackages, nil
 }
+
+func downloadPackageFiles(pkgInfo *PackageInfo, tempDirectory string) error {
+	for _, download := range pkgInfo.Downloads {
+		// Replace variables in download url
+		downloadUrl := download.Url
+		downloadUrl = os.Expand(downloadUrl, func(s string) string {
+			switch s {
+			case "BPM_PKG_VERSION":
+				return pkgInfo.Version
+			case "BPM_PKG_NAME":
+				return pkgInfo.Name
+			default:
+				return ""
+			}
+		})
+
+		switch download.Type {
+		case "", "file":
+			filepath := path.Join(tempDirectory, path.Base(downloadUrl))
+			if download.Filepath != "" {
+				filepath = download.Filepath
+			}
+
+			err := downloadFile(downloadUrl, filepath, 0644)
+			if err != nil {
+				return err
+			}
+
+			if download.Checksum != "skip" {
+				f, err := os.Open(filepath)
+				if err != nil {
+					return err
+				}
+				defer f.Close()
+
+				h := sha256.New()
+				_, err = io.Copy(h, f)
+				if err != nil {
+					return err
+				}
+
+				if hex.EncodeToString(h.Sum(nil)) != download.Checksum {
+					fmt.Printf("Downloaded file checksum: %x\n", h.Sum(nil))
+					return fmt.Errorf("downloaded file checksums did not match")
+				}
+			} else {
+				fmt.Println("Skipping checksum checking...")
+			}
+
+			if !download.NoExtract && (strings.Contains(filepath, ".tar") || strings.HasSuffix(filepath, ".tgz")) {
+				cmd := exec.Command("tar", "xvf", filepath, "--strip-components="+strconv.Itoa(download.ExtractStripComponents))
+				if download.ExtractToBPMSource {
+					cmd.Args = append(cmd.Args, "-C", path.Join(tempDirectory, "source"))
+				}
+
+				cmd.Stdout = os.Stdout
+				cmd.Stderr = os.Stderr
+
+				err := cmd.Run()
+				if err != nil {
+					return err
+				}
+			} else if !download.NoExtract && strings.HasSuffix(filepath, ".zip") {
+				cmd := exec.Command("unzip", filepath)
+				if download.ExtractToBPMSource {
+					cmd.Args = append(cmd.Args, "-d", path.Join(tempDirectory, "source"))
+				} else {
+					err := os.Mkdir(path.Join(tempDirectory, strings.TrimSuffix(path.Base(filepath), ".zip")), 0755)
+					if err != nil {
+						return err
+					}
+
+					cmd.Args = append(cmd.Args, "-d", path.Join(tempDirectory, strings.TrimSuffix(path.Base(filepath), ".zip")))
+				}
+
+				cmd.Stdout = os.Stdout
+				cmd.Stderr = os.Stderr
+
+				err := cmd.Run()
+				if err != nil {
+					return err
+				}
+			}
+		case "git":
+			// Replace variables in git branch
+			gitBranch := download.GitBranch
+			gitBranch = os.Expand(gitBranch, func(s string) string {
+				switch s {
+				case "BPM_PKG_VERSION":
+					return pkgInfo.Version
+				case "BPM_PKG_NAME":
+					return pkgInfo.Name
+				default:
+					return ""
+				}
+			})
+
+			cmd := exec.Command("git", "clone", "--depth=1", downloadUrl)
+			if gitBranch != "" {
+				cmd.Args = slices.Insert(cmd.Args, len(cmd.Args)-1, "--branch="+gitBranch)
+			}
+			if download.ExtractToBPMSource {
+				cmd.Args = append(cmd.Args, path.Join(tempDirectory, "source"))
+			}
+
+			cmd.Stdout = os.Stdout
+			cmd.Stderr = os.Stderr
+
+			err := cmd.Run()
+			if err != nil {
+				return err
+			}
+
+			if download.Checksum != "skip" {
+				cmd := exec.Command("git", "rev-parse", "HEAD")
+				if download.ExtractToBPMSource {
+					cmd.Dir = path.Join(tempDirectory, "source")
+				} else {
+					cmd.Dir = path.Join(tempDirectory, strings.TrimSuffix(path.Base(downloadUrl), ".git"))
+				}
+
+				branchChecksum, err := cmd.Output()
+				if err != nil {
+					return err
+				}
+
+				if strings.TrimSpace(string(branchChecksum)) != download.Checksum {
+					fmt.Printf("Git branch checksum: %s\n", strings.TrimSpace(string(branchChecksum)))
+					return fmt.Errorf("Cloned git repository checksum did not match")
+				}
+			} else {
+				fmt.Println("Skipping checksum checking...")
+			}
+		default:
+			return fmt.Errorf("unknown download type (%s)", download.Type)
+		}
+	}
+
+	return nil
+}

src/bpmlib/databases.go

@@ -3,8 +3,6 @@ package bpmlib
 import (
 	"errors"
 	"fmt"
-	"io"
-	"net/http"
 	"net/url"
 	"os"
 	"path"
@@ -132,14 +130,10 @@ func (db *configDatabase) SyncLocalDatabaseFile() error {
 	}
 
 	// Retrieve data from URL
-	resp, err := http.Get(u)
+	buffer, err := retrieveUrlData(u)
 	if err != nil {
 		return err
 	}
-	defer resp.Body.Close()
-
-	// Load data into byte buffer
-	buffer, err := io.ReadAll(resp.Body)
 
 	// Unmarshal data to ensure it is a valid BPM database
 	err = yaml.Unmarshal(buffer, &BPMDatabase{})
@@ -233,30 +227,23 @@ func ResolveVirtualPackage(vpkg string) *BPMDatabaseEntry {
 }
 
 func (db *BPMDatabase) FetchPackage(pkg string) (string, error) {
+	// Check if package exists in database
 	if !db.ContainsPackage(pkg) {
 		return "", errors.New("could not fetch package '" + pkg + "'")
 	}
+
+	// Get package url from database
 	entry := db.Entries[pkg]
-	URL, err := url.JoinPath(db.Source, entry.Filepath)
+	u, err := url.JoinPath(db.Source, entry.Filepath)
 	if err != nil {
 		return "", err
 	}
-	resp, err := http.Get(URL)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
 
-	err = os.MkdirAll("/var/cache/bpm/fetched/", 0755)
+	// Download package from url
+	err = downloadFile(u, path.Join("/var/cache/bpm/fetched/", path.Base(entry.Filepath)), 0644)
 	if err != nil {
 		return "", err
 	}
-	out, err := os.Create("/var/cache/bpm/fetched/" + path.Base(entry.Filepath))
-	if err != nil {
-		return "", err
-	}
-	defer out.Close()
 
-	_, err = io.Copy(out, resp.Body)
+	return path.Join("/var/cache/bpm/fetched/", path.Base(entry.Filepath)), nil
-	return "/var/cache/bpm/fetched/" + path.Base(entry.Filepath), nil
 }

src/bpmlib/net.go

@@ -0,0 +1,61 @@
+package bpmlib
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path"
+	"strings"
+)
+
+func retrieveUrlData(u string) ([]byte, error) {
+	resp, err := http.Get(u)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	// Load data into byte buffer
+	buffer, err := io.ReadAll(resp.Body)
+
+	return buffer, nil
+}
+
+func downloadFile(u, filepath string, perm os.FileMode) error {
+	if strings.HasSuffix(filepath, "/") {
+		return fmt.Errorf("Filepath must not end in '/'")
+	}
+
+	data, err := retrieveUrlData(u)
+	if err != nil {
+		return err
+	}
+
+	// Create parent directories
+	err = os.MkdirAll(path.Dir(filepath), 0755)
+	if err != nil {
+		return err
+	}
+
+	// Create file
+	file, err := os.Create(filepath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	// Copy data
+	_, err = file.Write(data)
+	if err != nil {
+		return err
+	}
+
+	// Set file permissions
+	err = file.Chmod(perm)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

src/bpmlib/packages.go

@@ -26,23 +26,35 @@ type BPMPackage struct {
 }
 
 type PackageInfo struct {
-	Name            string         `yaml:"name,omitempty"`
+	Name            string            `yaml:"name"`
-	Description     string         `yaml:"description,omitempty"`
+	Description     string            `yaml:"description"`
-	Version         string         `yaml:"version,omitempty"`
+	Version         string            `yaml:"version"`
-	Revision        int            `yaml:"revision,omitempty"`
+	Revision        int               `yaml:"revision,omitempty"`
-	Url             string         `yaml:"url,omitempty"`
+	Url             string            `yaml:"url,omitempty"`
-	License         string         `yaml:"license,omitempty"`
+	License         string            `yaml:"license,omitempty"`
-	Arch            string         `yaml:"architecture,omitempty"`
+	Arch            string            `yaml:"architecture,omitempty"`
-	OutputArch      string         `yaml:"output_architecture,omitempty"`
+	OutputArch      string            `yaml:"output_architecture,omitempty"`
-	Type            string         `yaml:"type,omitempty"`
+	Type            string            `yaml:"type,omitempty"`
-	Keep            []string       `yaml:"keep,omitempty"`
+	Keep            []string          `yaml:"keep,omitempty"`
-	Depends         []string       `yaml:"depends,omitempty"`
+	Depends         []string          `yaml:"depends,omitempty"`
-	MakeDepends     []string       `yaml:"make_depends,omitempty"`
+	MakeDepends     []string          `yaml:"make_depends,omitempty"`
-	OptionalDepends []string       `yaml:"optional_depends,omitempty"`
+	OptionalDepends []string          `yaml:"optional_depends,omitempty"`
-	Conflicts       []string       `yaml:"conflicts,omitempty"`
+	Conflicts       []string          `yaml:"conflicts,omitempty"`
-	Replaces        []string       `yaml:"replaces,omitempty"`
+	Replaces        []string          `yaml:"replaces,omitempty"`
-	Provides        []string       `yaml:"provides,omitempty"`
+	Provides        []string          `yaml:"provides,omitempty"`
-	SplitPackages   []*PackageInfo `yaml:"split_packages,omitempty"`
+	Downloads       []PackageDownload `yaml:"downloads,omitempty"`
+	SplitPackages   []*PackageInfo    `yaml:"split_packages,omitempty"`
+}
+
+type PackageDownload struct {
+	Url                    string `yaml:"url"`
+	Type                   string `yaml:"type"`
+	NoExtract              bool   `yaml:"no_extract"`
+	ExtractToBPMSource     bool   `yaml:"extract_to_bpm_source"`
+	ExtractStripComponents int    `yaml:"extract_strip_components"`
+	GitBranch              string `yaml:"git_branch"`
+	Filepath               string `yaml:"filepath,omitempty"`
+	Checksum               string `yaml:"checksum"`
 }
 
 type PackageFileEntry struct {
@@ -422,6 +434,7 @@ func ReadPackageInfo(contents string) (*PackageInfo, error) {
 		Conflicts:       make([]string, 0),
 		Replaces:        make([]string, 0),
 		Provides:        make([]string, 0),
+		Downloads:       make([]PackageDownload, 0),
 		SplitPackages:   make([]*PackageInfo, 0),
 	}
 	err := yaml.Unmarshal([]byte(contents), &pkgInfo)