From bc89db8cef9102699b34021404937872dbd879ea Mon Sep 17 00:00:00 2001 From: EnumDev Date: Fri, 26 Sep 2025 16:10:07 +0300 Subject: [PATCH] Add 'downloads' field in pkg.info --- src/bpmlib/compilation.go | 153 +++++++++++++++++++++++++++++++++++++- src/bpmlib/databases.go | 29 ++------ src/bpmlib/net.go | 61 +++++++++++++++ src/bpmlib/packages.go | 47 +++++++----- 4 files changed, 251 insertions(+), 39 deletions(-) create mode 100644 src/bpmlib/net.go diff --git a/src/bpmlib/compilation.go b/src/bpmlib/compilation.go index c218332..ecce113 100644 --- a/src/bpmlib/compilation.go +++ b/src/bpmlib/compilation.go @@ -1,11 +1,15 @@ package bpmlib import ( + "crypto/sha256" + "encoding/hex" "errors" "fmt" + "io" "os" "os/exec" "path" + "slices" "strconv" "strings" "syscall" @@ -116,6 +120,12 @@ func CompileSourcePackage(archiveFilename, outputDirectory string, skipChecks bo return nil, err } + // Download files + err = downloadPackageFiles(bpmpkg.PkgInfo, tempDirectory) + if err != nil { + return nil, err + } + // Setup environment for commands env := os.Environ() env = append(env, "HOME="+tempDirectory) @@ -251,8 +261,9 @@ func CompileSourcePackage(archiveFilename, outputDirectory string, skipChecks bo pkgInfo.Arch = pkg.OutputArch pkgInfo.OutputArch = "" - // Remove split package field + // Remove split package and downloads fields pkgInfo.SplitPackages = nil + pkgInfo.Downloads = nil // Marshal package info pkgInfoBytes, err := yaml.Marshal(pkgInfo) @@ -323,3 +334,143 @@ func CompileSourcePackage(archiveFilename, outputDirectory string, skipChecks bo return outputBpmPackages, nil } + +func downloadPackageFiles(pkgInfo *PackageInfo, tempDirectory string) error { + for _, download := range pkgInfo.Downloads { + // Replace variables in download url + downloadUrl := download.Url + downloadUrl = os.Expand(downloadUrl, func(s string) string { + switch s { + case "BPM_PKG_VERSION": + return pkgInfo.Version + case "BPM_PKG_NAME": + return pkgInfo.Name + default: + return "" + } + }) + + switch download.Type { + case "", "file": + filepath := path.Join(tempDirectory, path.Base(downloadUrl)) + if download.Filepath != "" { + filepath = download.Filepath + } + + err := downloadFile(downloadUrl, filepath, 0644) + if err != nil { + return err + } + + if download.Checksum != "skip" { + f, err := os.Open(filepath) + if err != nil { + return err + } + defer f.Close() + + h := sha256.New() + _, err = io.Copy(h, f) + if err != nil { + return err + } + + if hex.EncodeToString(h.Sum(nil)) != download.Checksum { + fmt.Printf("Downloaded file checksum: %x\n", h.Sum(nil)) + return fmt.Errorf("downloaded file checksums did not match") + } + } else { + fmt.Println("Skipping checksum checking...") + } + + if !download.NoExtract && (strings.Contains(filepath, ".tar") || strings.HasSuffix(filepath, ".tgz")) { + cmd := exec.Command("tar", "xvf", filepath, "--strip-components="+strconv.Itoa(download.ExtractStripComponents)) + if download.ExtractToBPMSource { + cmd.Args = append(cmd.Args, "-C", path.Join(tempDirectory, "source")) + } + + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err := cmd.Run() + if err != nil { + return err + } + } else if !download.NoExtract && strings.HasSuffix(filepath, ".zip") { + cmd := exec.Command("unzip", filepath) + if download.ExtractToBPMSource { + cmd.Args = append(cmd.Args, "-d", path.Join(tempDirectory, "source")) + } else { + err := os.Mkdir(path.Join(tempDirectory, strings.TrimSuffix(path.Base(filepath), ".zip")), 0755) + if err != nil { + return err + } + + cmd.Args = append(cmd.Args, "-d", path.Join(tempDirectory, strings.TrimSuffix(path.Base(filepath), ".zip"))) + } + + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err := cmd.Run() + if err != nil { + return err + } + } + case "git": + // Replace variables in git branch + gitBranch := download.GitBranch + gitBranch = os.Expand(gitBranch, func(s string) string { + switch s { + case "BPM_PKG_VERSION": + return pkgInfo.Version + case "BPM_PKG_NAME": + return pkgInfo.Name + default: + return "" + } + }) + + cmd := exec.Command("git", "clone", "--depth=1", downloadUrl) + if gitBranch != "" { + cmd.Args = slices.Insert(cmd.Args, len(cmd.Args)-1, "--branch="+gitBranch) + } + if download.ExtractToBPMSource { + cmd.Args = append(cmd.Args, path.Join(tempDirectory, "source")) + } + + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err := cmd.Run() + if err != nil { + return err + } + + if download.Checksum != "skip" { + cmd := exec.Command("git", "rev-parse", "HEAD") + if download.ExtractToBPMSource { + cmd.Dir = path.Join(tempDirectory, "source") + } else { + cmd.Dir = path.Join(tempDirectory, strings.TrimSuffix(path.Base(downloadUrl), ".git")) + } + + branchChecksum, err := cmd.Output() + if err != nil { + return err + } + + if strings.TrimSpace(string(branchChecksum)) != download.Checksum { + fmt.Printf("Git branch checksum: %s\n", strings.TrimSpace(string(branchChecksum))) + return fmt.Errorf("Cloned git repository checksum did not match") + } + } else { + fmt.Println("Skipping checksum checking...") + } + default: + return fmt.Errorf("unknown download type (%s)", download.Type) + } + } + + return nil +} diff --git a/src/bpmlib/databases.go b/src/bpmlib/databases.go index 7331e7d..497cfea 100644 --- a/src/bpmlib/databases.go +++ b/src/bpmlib/databases.go @@ -3,8 +3,6 @@ package bpmlib import ( "errors" "fmt" - "io" - "net/http" "net/url" "os" "path" @@ -132,14 +130,10 @@ func (db *configDatabase) SyncLocalDatabaseFile() error { } // Retrieve data from URL - resp, err := http.Get(u) + buffer, err := retrieveUrlData(u) if err != nil { return err } - defer resp.Body.Close() - - // Load data into byte buffer - buffer, err := io.ReadAll(resp.Body) // Unmarshal data to ensure it is a valid BPM database err = yaml.Unmarshal(buffer, &BPMDatabase{}) @@ -233,30 +227,23 @@ func ResolveVirtualPackage(vpkg string) *BPMDatabaseEntry { } func (db *BPMDatabase) FetchPackage(pkg string) (string, error) { + // Check if package exists in database if !db.ContainsPackage(pkg) { return "", errors.New("could not fetch package '" + pkg + "'") } + + // Get package url from database entry := db.Entries[pkg] - URL, err := url.JoinPath(db.Source, entry.Filepath) + u, err := url.JoinPath(db.Source, entry.Filepath) if err != nil { return "", err } - resp, err := http.Get(URL) - if err != nil { - return "", err - } - defer resp.Body.Close() - err = os.MkdirAll("/var/cache/bpm/fetched/", 0755) + // Download package from url + err = downloadFile(u, path.Join("/var/cache/bpm/fetched/", path.Base(entry.Filepath)), 0644) if err != nil { return "", err } - out, err := os.Create("/var/cache/bpm/fetched/" + path.Base(entry.Filepath)) - if err != nil { - return "", err - } - defer out.Close() - _, err = io.Copy(out, resp.Body) - return "/var/cache/bpm/fetched/" + path.Base(entry.Filepath), nil + return path.Join("/var/cache/bpm/fetched/", path.Base(entry.Filepath)), nil } diff --git a/src/bpmlib/net.go b/src/bpmlib/net.go new file mode 100644 index 0000000..60c01e5 --- /dev/null +++ b/src/bpmlib/net.go @@ -0,0 +1,61 @@ +package bpmlib + +import ( + "fmt" + "io" + "net/http" + "os" + "path" + "strings" +) + +func retrieveUrlData(u string) ([]byte, error) { + resp, err := http.Get(u) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + // Load data into byte buffer + buffer, err := io.ReadAll(resp.Body) + + return buffer, nil +} + +func downloadFile(u, filepath string, perm os.FileMode) error { + if strings.HasSuffix(filepath, "/") { + return fmt.Errorf("Filepath must not end in '/'") + } + + data, err := retrieveUrlData(u) + if err != nil { + return err + } + + // Create parent directories + err = os.MkdirAll(path.Dir(filepath), 0755) + if err != nil { + return err + } + + // Create file + file, err := os.Create(filepath) + if err != nil { + return err + } + defer file.Close() + + // Copy data + _, err = file.Write(data) + if err != nil { + return err + } + + // Set file permissions + err = file.Chmod(perm) + if err != nil { + return err + } + + return nil +} diff --git a/src/bpmlib/packages.go b/src/bpmlib/packages.go index 9e4c5cb..b0c2a27 100644 --- a/src/bpmlib/packages.go +++ b/src/bpmlib/packages.go @@ -26,23 +26,35 @@ type BPMPackage struct { } type PackageInfo struct { - Name string `yaml:"name,omitempty"` - Description string `yaml:"description,omitempty"` - Version string `yaml:"version,omitempty"` - Revision int `yaml:"revision,omitempty"` - Url string `yaml:"url,omitempty"` - License string `yaml:"license,omitempty"` - Arch string `yaml:"architecture,omitempty"` - OutputArch string `yaml:"output_architecture,omitempty"` - Type string `yaml:"type,omitempty"` - Keep []string `yaml:"keep,omitempty"` - Depends []string `yaml:"depends,omitempty"` - MakeDepends []string `yaml:"make_depends,omitempty"` - OptionalDepends []string `yaml:"optional_depends,omitempty"` - Conflicts []string `yaml:"conflicts,omitempty"` - Replaces []string `yaml:"replaces,omitempty"` - Provides []string `yaml:"provides,omitempty"` - SplitPackages []*PackageInfo `yaml:"split_packages,omitempty"` + Name string `yaml:"name"` + Description string `yaml:"description"` + Version string `yaml:"version"` + Revision int `yaml:"revision,omitempty"` + Url string `yaml:"url,omitempty"` + License string `yaml:"license,omitempty"` + Arch string `yaml:"architecture,omitempty"` + OutputArch string `yaml:"output_architecture,omitempty"` + Type string `yaml:"type,omitempty"` + Keep []string `yaml:"keep,omitempty"` + Depends []string `yaml:"depends,omitempty"` + MakeDepends []string `yaml:"make_depends,omitempty"` + OptionalDepends []string `yaml:"optional_depends,omitempty"` + Conflicts []string `yaml:"conflicts,omitempty"` + Replaces []string `yaml:"replaces,omitempty"` + Provides []string `yaml:"provides,omitempty"` + Downloads []PackageDownload `yaml:"downloads,omitempty"` + SplitPackages []*PackageInfo `yaml:"split_packages,omitempty"` +} + +type PackageDownload struct { + Url string `yaml:"url"` + Type string `yaml:"type"` + NoExtract bool `yaml:"no_extract"` + ExtractToBPMSource bool `yaml:"extract_to_bpm_source"` + ExtractStripComponents int `yaml:"extract_strip_components"` + GitBranch string `yaml:"git_branch"` + Filepath string `yaml:"filepath,omitempty"` + Checksum string `yaml:"checksum"` } type PackageFileEntry struct { @@ -422,6 +434,7 @@ func ReadPackageInfo(contents string) (*PackageInfo, error) { Conflicts: make([]string, 0), Replaces: make([]string, 0), Provides: make([]string, 0), + Downloads: make([]PackageDownload, 0), SplitPackages: make([]*PackageInfo, 0), } err := yaml.Unmarshal([]byte(contents), &pkgInfo)