Switch to new BPM file structure #8

Merged
EnumDev merged 11 commits from improve_bpm_structure into master 2024-10-23 06:22:40 +00:00
4 changed files with 261 additions and 110 deletions
Showing only changes of commit 7816d0072c - Show all commits

171
main.go
View File

@ -174,12 +174,7 @@ func resolveCommand() {
return
}
pkgsToInstall := orderedmap.NewOrderedMap[string, *struct {
bpmFile string
isDependency bool
shouldFetch bool
bpmpkg *utils.BPMPackage
}]()
operation := utils.BPMOperation{Actions: make([]utils.OperationAction, 0)}
unresolvedDepends := make([]string, 0)
// Search for packages
@ -192,12 +187,11 @@ func resolveCommand() {
if !reinstall && utils.IsPackageInstalled(bpmpkg.PkgInfo.Name, rootDir) && utils.GetPackageInfo(bpmpkg.PkgInfo.Name, rootDir).GetFullVersion() == bpmpkg.PkgInfo.GetFullVersion() {
continue
}
pkgsToInstall.Set(bpmpkg.PkgInfo.Name, &struct {
bpmFile string
isDependency bool
shouldFetch bool
bpmpkg *utils.BPMPackage
}{bpmFile: pkg, isDependency: false, shouldFetch: false, bpmpkg: bpmpkg})
operation.Actions = append(operation.Actions, &utils.InstallPackageAction{
File: pkg,
IsDependency: false,
BpmPackage: bpmpkg,
})
} else {
entry, _, err := utils.GetRepositoryEntry(pkg)
if err != nil {
@ -206,50 +200,44 @@ func resolveCommand() {
if !reinstall && utils.IsPackageInstalled(entry.Info.Name, rootDir) && utils.GetPackageInfo(entry.Info.Name, rootDir).GetFullVersion() == entry.Info.GetFullVersion() {
continue
}
pkgsToInstall.Set(entry.Info.Name, &struct {
bpmFile string
isDependency bool
shouldFetch bool
bpmpkg *utils.BPMPackage
}{bpmFile: "", isDependency: false, shouldFetch: true, bpmpkg: &utils.BPMPackage{
PkgInfo: entry.Info,
PkgFiles: nil,
}})
operation.Actions = append(operation.Actions, &utils.FetchPackageAction{
IsDependency: false,
RepositoryEntry: entry,
})
}
}
pos := 0
for _, value := range slices.Clone(operation.Actions) {
var resolved, unresolved []string
var pkgInfo *utils.PackageInfo
if value.GetActionType() == "install" {
action := value.(*utils.InstallPackageAction)
pkgInfo = action.BpmPackage.PkgInfo
} else if value.GetActionType() == "fetch" {
action := value.(*utils.FetchPackageAction)
pkgInfo = action.RepositoryEntry.Info
}
resolved, unresolved = pkgInfo.ResolveAll(&[]string{}, &[]string{}, pkgInfo.Type == "source", !noOptional, !reinstallAll, verbose, rootDir)
clone := pkgsToInstall.Copy()
pkgsToInstall = orderedmap.NewOrderedMap[string, *struct {
bpmFile string
isDependency bool
shouldFetch bool
bpmpkg *utils.BPMPackage
}]()
for _, pkg := range clone.Keys() {
value, _ := clone.Get(pkg)
resolved, unresolved := value.bpmpkg.PkgInfo.ResolveAll(&[]string{}, &[]string{}, value.bpmpkg.PkgInfo.Type == "source", !noOptional, !reinstall, verbose, rootDir)
unresolvedDepends = append(unresolvedDepends, unresolved...)
for _, depend := range resolved {
if _, ok := pkgsToInstall.Get(depend); !ok && depend != value.bpmpkg.PkgInfo.Name {
if !operation.ActionsContainPackage(depend) && depend != pkgInfo.Name {
if !reinstallAll && utils.IsPackageInstalled(depend, rootDir) {
continue
}
entry, _, err := utils.GetRepositoryEntry(depend)
if err != nil {
log.Fatalf("Error: could not find package (%s) in any repository\n", pkg)
log.Fatalf("Error: could not find package (%s) in any repository\n", pkgInfo.Name)
}
pkgsToInstall.Set(depend, &struct {
bpmFile string
isDependency bool
shouldFetch bool
bpmpkg *utils.BPMPackage
}{bpmFile: "", isDependency: true, shouldFetch: true, bpmpkg: &utils.BPMPackage{
PkgInfo: entry.Info,
PkgFiles: nil,
}})
operation.InsertActionAt(pos, &utils.FetchPackageAction{
IsDependency: true,
RepositoryEntry: entry,
})
pos++
}
}
pkgsToInstall.Set(pkg, value)
pos++
}
// Show summary
@ -260,18 +248,25 @@ func resolveCommand() {
log.Println("Warning: The following dependencies could not be found in any repositories: " + strings.Join(unresolvedDepends, ", "))
}
}
if pkgsToInstall.Len() == 0 {
if len(operation.Actions) == 0 {
fmt.Println("All packages are up to date!")
os.Exit(0)
}
var totalOperationSize uint64 = 0
for _, pkg := range pkgsToInstall.Keys() {
value, _ := pkgsToInstall.Get(pkg)
bpmpkg := value.bpmpkg
installedInfo := utils.GetPackageInfo(bpmpkg.PkgInfo.Name, rootDir)
for _, value := range operation.Actions {
var pkgInfo *utils.PackageInfo
if value.GetActionType() == "install" {
pkgInfo = value.(*utils.InstallPackageAction).BpmPackage.PkgInfo
} else if value.GetActionType() == "fetch" {
pkgInfo = value.(*utils.FetchPackageAction).RepositoryEntry.Info
} else {
pkgInfo = value.(*utils.RemovePackageAction).BpmPackage.PkgInfo
fmt.Printf("%s: %s (Remove)\n", pkgInfo.Name, pkgInfo.GetFullVersion())
}
installedInfo := utils.GetPackageInfo(pkgInfo.Name, rootDir)
sourceInfo := ""
if bpmpkg.PkgInfo.Type == "source" {
if pkgInfo.Type == "source" {
if rootDir != "/" && !force {
log.Fatalf("Error: cannot compile and install source packages to a different root directory")
}
@ -279,35 +274,39 @@ func resolveCommand() {
}
if installedInfo == nil {
fmt.Printf("%s: %s (Install) %s\n", bpmpkg.PkgInfo.Name, bpmpkg.PkgInfo.GetFullVersion(), sourceInfo)
totalOperationSize += bpmpkg.GetInstalledSize()
fmt.Printf("%s: %s (Install) %s\n", pkgInfo.Name, pkgInfo.GetFullVersion(), sourceInfo)
} else {
comparison := utils.ComparePackageVersions(*bpmpkg.PkgInfo, *installedInfo)
comparison := utils.ComparePackageVersions(*pkgInfo, *installedInfo)
if comparison < 0 {
fmt.Printf("%s: %s -> %s (Downgrade) %s\n", bpmpkg.PkgInfo.Name, installedInfo.GetFullVersion(), bpmpkg.PkgInfo.GetFullVersion(), sourceInfo)
fmt.Printf("%s: %s -> %s (Downgrade) %s\n", pkgInfo.Name, installedInfo.GetFullVersion(), pkgInfo.GetFullVersion(), sourceInfo)
} else if comparison > 0 {
fmt.Printf("%s: %s -> %s (Upgrade) %s\n", bpmpkg.PkgInfo.Name, installedInfo.GetFullVersion(), bpmpkg.PkgInfo.GetFullVersion(), sourceInfo)
fmt.Printf("%s: %s -> %s (Upgrade) %s\n", pkgInfo.Name, installedInfo.GetFullVersion(), pkgInfo.GetFullVersion(), sourceInfo)
} else {
fmt.Printf("%s: %s (Reinstall) %s\n", bpmpkg.PkgInfo.Name, bpmpkg.PkgInfo.GetFullVersion(), sourceInfo)
}
totalOperationSize += bpmpkg.GetInstalledSize()
fmt.Printf("%s: %s (Reinstall) %s\n", pkgInfo.Name, pkgInfo.GetFullVersion(), sourceInfo)
}
}
}
if rootDir != "/" {
fmt.Println("Warning: Operating in " + rootDir)
}
if totalOperationSize >= 0 {
fmt.Printf("A total of %s will be used to complete this operation\n", utils.BytesToHumanReadable(totalOperationSize))
if operation.GetTotalDownloadSize() > 0 {
fmt.Printf("%s will be downloaded to complete this operation\n", utils.BytesToHumanReadable(operation.GetTotalDownloadSize()))
}
if operation.GetFinalActionSize(rootDir) > 0 {
fmt.Printf("A total of %s will be installed after the operation finishes\n", utils.BytesToHumanReadable(operation.GetFinalActionSize(rootDir)))
} else if operation.GetFinalActionSize(rootDir) < 0 {
fmt.Printf("A total of %s will be freed after the operation finishes\n", utils.BytesToHumanReadable(operation.GetFinalActionSize(rootDir)))
} else {
fmt.Printf("A total of %s will be freed by this operation\n", utils.BytesToHumanReadable(totalOperationSize))
fmt.Println()
}
if !yesAll {
reader := bufio.NewReader(os.Stdin)
if pkgsToInstall.Len() == 1 {
if len(operation.Actions) == 1 {
fmt.Printf("Do you wish to install this package? [y\\N] ")
} else {
fmt.Printf("Do you wish to install these %d packages? [y\\N] ", pkgsToInstall.Len())
fmt.Printf("Do you wish to install these %d packages? [y\\N] ", len(operation.Actions))
}
text, _ := reader.ReadString('\n')
@ -319,43 +318,49 @@ func resolveCommand() {
// Fetch packages from repositories
fmt.Println("Fetching packages from available repositories...")
for _, pkg := range pkgsToInstall.Keys() {
value, _ := pkgsToInstall.Get(pkg)
if !value.shouldFetch {
for i, action := range operation.Actions {
if action.GetActionType() != "fetch" {
continue
}
entry, repo, err := utils.GetRepositoryEntry(pkg)
entry := action.(*utils.FetchPackageAction).RepositoryEntry
fetchedPackage, err := entry.Repository.FetchPackage(entry.Info.Name)
if err != nil {
log.Fatalf("Error: could not find package (%s) in any repository\n", pkg)
log.Fatalf("Error: could not fetch package (%s): %s\n", entry.Info.Name, err)
}
fetchedPackage, err := repo.FetchPackage(entry.Info.Name)
bpmpkg, err := utils.ReadPackage(fetchedPackage)
if err != nil {
log.Fatalf("Error: could not fetch package (%s): %s\n", pkg, err)
log.Fatalf("Error: could not fetch package (%s): %s\n", entry.Info.Name, err)
}
fmt.Printf("Package (%s) was successfully fetched!\n", bpmpkg.PkgInfo.Name)
operation.Actions[i] = &utils.InstallPackageAction{
File: fetchedPackage,
IsDependency: action.(*utils.FetchPackageAction).IsDependency,
BpmPackage: bpmpkg,
}
fmt.Printf("Package (%s) was successfully fetched!\n", value.bpmpkg.PkgInfo.Name)
value.bpmFile = fetchedPackage
pkgsToInstall.Set(pkg, value)
}
// Install fetched packages
for _, pkg := range pkgsToInstall.Keys() {
value, _ := pkgsToInstall.Get(pkg)
bpmpkg := value.bpmpkg
for _, action := range operation.Actions {
if action.GetActionType() != "install" {
continue
}
value := action.(*utils.InstallPackageAction)
bpmpkg := value.BpmPackage
var err error
if value.isDependency {
err = utils.InstallPackage(value.bpmFile, rootDir, verbose, true, buildSource, skipCheck, keepTempDir)
if value.IsDependency {
err = utils.InstallPackage(value.File, rootDir, verbose, true, buildSource, skipCheck, keepTempDir)
} else {
err = utils.InstallPackage(value.bpmFile, rootDir, verbose, force, buildSource, skipCheck, keepTempDir)
err = utils.InstallPackage(value.File, rootDir, verbose, force, buildSource, skipCheck, keepTempDir)
}
if err != nil {
if bpmpkg.PkgInfo.Type == "source" && keepTempDir {
fmt.Println("BPM temp directory was created at /var/tmp/bpm_source-" + bpmpkg.PkgInfo.Name)
}
log.Fatalf("Error: could not install package (%s): %s\n", pkg, err)
log.Fatalf("Error: could not install package (%s): %s\n", bpmpkg.PkgInfo.Name, err)
}
fmt.Printf("Package (%s) was successfully installed\n", bpmpkg.PkgInfo.Name)
if value.isDependency {
if value.IsDependency {
err := utils.SetInstallationReason(bpmpkg.PkgInfo.Name, utils.Dependency, rootDir)
if err != nil {
log.Fatalf("Error: could not set installation reason for package: %s\n", err)
@ -620,7 +625,9 @@ func resolveCommand() {
var pkgList []string
for _, pkg := range pkgs {
if slices.Contains(utils.GetPackageFiles(pkg, rootDir), absFile) {
if slices.ContainsFunc(utils.GetPackageFiles(pkg, rootDir), func(entry *utils.PackageFileEntry) bool {
return entry.Path == absFile
}) {
pkgList = append(pkgList, pkg)
}
}

96
utils/operations.go Normal file
View File

@ -0,0 +1,96 @@
package utils
type BPMOperation struct {
Actions []OperationAction
}
func (operation *BPMOperation) ActionsContainPackage(pkg string) bool {
for _, action := range operation.Actions {
if action.GetActionType() == "install" {
return action.(*InstallPackageAction).BpmPackage.PkgInfo.Name == pkg
} else if action.GetActionType() == "fetch" {
return action.(*FetchPackageAction).RepositoryEntry.Info.Name == pkg
} else if action.GetActionType() == "remove" {
return action.(*RemovePackageAction).BpmPackage.PkgInfo.Name == pkg
}
}
return false
}
func (operation *BPMOperation) InsertActionAt(index int, action OperationAction) {
if len(operation.Actions) == index { // nil or empty slice or after last element
operation.Actions = append(operation.Actions, action)
}
operation.Actions = append(operation.Actions[:index+1], operation.Actions[index:]...) // index < len(a)
operation.Actions[index] = action
}
func (operation *BPMOperation) GetTotalDownloadSize() uint64 {
var ret uint64 = 0
for _, action := range operation.Actions {
if action.GetActionType() == "fetch" {
ret += action.(*FetchPackageAction).RepositoryEntry.DownloadSize
}
}
return ret
}
func (operation *BPMOperation) GetTotalInstalledSize() uint64 {
var ret uint64 = 0
for _, action := range operation.Actions {
if action.GetActionType() == "install" {
ret += action.(*InstallPackageAction).BpmPackage.GetInstalledSize()
} else if action.GetActionType() == "fetch" {
ret += action.(*FetchPackageAction).RepositoryEntry.InstalledSize
}
}
return ret
}
func (operation *BPMOperation) GetFinalActionSize(rootDir string) uint64 {
var ret uint64 = 0
for _, action := range operation.Actions {
if action.GetActionType() == "install" {
ret += action.(*InstallPackageAction).BpmPackage.GetInstalledSize()
if IsPackageInstalled(action.(*InstallPackageAction).BpmPackage.PkgInfo.Name, rootDir) {
ret -= GetPackage(action.(*InstallPackageAction).BpmPackage.PkgInfo.Name, rootDir).GetInstalledSize()
}
} else if action.GetActionType() == "fetch" {
ret += action.(*FetchPackageAction).RepositoryEntry.InstalledSize
} else if action.GetActionType() == "remove" {
ret -= action.(*RemovePackageAction).BpmPackage.GetInstalledSize()
}
}
return ret
}
type OperationAction interface {
GetActionType() string
}
type InstallPackageAction struct {
File string
IsDependency bool
BpmPackage *BPMPackage
}
func (action *InstallPackageAction) GetActionType() string {
return "install"
}
type FetchPackageAction struct {
IsDependency bool
RepositoryEntry *RepositoryEntry
}
func (action *FetchPackageAction) GetActionType() string {
return "fetch"
}
type RemovePackageAction struct {
BpmPackage *BPMPackage
}
func (action *RemovePackageAction) GetActionType() string {
return "remove"
}

View File

@ -2,7 +2,6 @@ package utils
import (
"archive/tar"
"bufio"
"bytes"
"compress/gzip"
"errors"
@ -180,23 +179,23 @@ func ReadPackage(filename string) (*BPMPackage, error) {
continue
}
stringEntry := strings.Split(strings.TrimSpace(line), " ")
if len(stringEntry) != 4 {
if len(stringEntry) < 4 {
return nil, errors.New("pkg.files is not formatted correctly")
}
uid, err := strconv.ParseInt(stringEntry[1], 0, 32)
uid, err := strconv.ParseInt(stringEntry[len(stringEntry)-3], 0, 32)
if err != nil {
return nil, err
}
gid, err := strconv.ParseInt(stringEntry[2], 0, 32)
gid, err := strconv.ParseInt(stringEntry[len(stringEntry)-2], 0, 32)
if err != nil {
return nil, err
}
size, err := strconv.ParseUint(stringEntry[3], 0, 64)
size, err := strconv.ParseUint(stringEntry[len(stringEntry)-1], 0, 64)
if err != nil {
return nil, err
}
pkgFiles = append(pkgFiles, &PackageFileEntry{
Path: stringEntry[0],
Path: strings.Join(stringEntry[:len(stringEntry)-3], " "),
UserID: int(uid),
GroupID: int(gid),
SizeInBytes: size,
@ -501,8 +500,7 @@ func extractPackage(bpmpkg *BPMPackage, verbose bool, filename, rootDir string)
if err != nil {
return err, nil
}
trimmedName := strings.TrimPrefix(header.Name, "./")
extractFilename := path.Join(rootDir, trimmedName)
extractFilename := path.Join(rootDir, header.Name)
switch header.Typeflag {
case tar.TypeDir:
files = append(files, strings.TrimPrefix(header.Name, "./"))
@ -520,7 +518,7 @@ func extractPackage(bpmpkg *BPMPackage, verbose bool, filename, rootDir string)
if _, err := os.Stat(extractFilename); err == nil {
for _, k := range bpmpkg.PkgInfo.Keep {
if strings.HasSuffix(k, "/") {
if strings.HasPrefix(trimmedName, k) {
if strings.HasPrefix(header.Name, k) {
if verbose {
fmt.Println("Skipping File: " + extractFilename + " (Containing directory is set to be kept during installs/updates)")
}
@ -529,7 +527,7 @@ func extractPackage(bpmpkg *BPMPackage, verbose bool, filename, rootDir string)
continue
}
} else {
if trimmedName == k {
if header.Name == k {
if verbose {
fmt.Println("Skipping File: " + extractFilename + " (File is configured to be kept during installs/updates)")
}
@ -1061,15 +1059,15 @@ func InstallPackage(filename, rootDir string, verbose, force, binaryPkgFromSrc,
}
packageInstalled := IsPackageInstalled(bpmpkg.PkgInfo.Name, rootDir)
if packageInstalled {
oldFiles = GetPackageFiles(bpmpkg.PkgInfo.Name, rootDir)
fileEntries := GetPackageFiles(bpmpkg.PkgInfo.Name, rootDir)
for _, entry := range fileEntries {
files = append(files, entry.Path)
}
}
if !force {
if bpmpkg.PkgInfo.Arch != "any" && bpmpkg.PkgInfo.Arch != GetArch() {
return errors.New("cannot install a package with a different architecture")
}
if unresolved := bpmpkg.PkgInfo.CheckDependencies(bpmpkg.PkgInfo.Type == "source", true, rootDir); len(unresolved) != 0 {
return errors.New("the following dependencies are not installed: " + strings.Join(unresolved, ", "))
}
}
if bpmpkg.PkgInfo.Type == "binary" {
err, i := extractPackage(bpmpkg, verbose, filename, rootDir)
@ -1115,12 +1113,11 @@ func InstallPackage(filename, rootDir string, verbose, force, binaryPkgFromSrc,
if err != nil {
return err
}
for _, line := range files {
_, err := f.WriteString(line + "\n")
bs, err := ReadTarballContent(filename, "pkg.files")
if err != nil {
return err
}
}
_, err = f.Write(bs)
err = f.Close()
if err != nil {
return err
@ -1375,8 +1372,8 @@ func GetInstalledPackages(rootDir string) ([]string, error) {
return ret, nil
}
func GetPackageFiles(pkg, rootDir string) []string {
var ret []string
func GetPackageFiles(pkg, rootDir string) []*PackageFileEntry {
var pkgFiles []*PackageFileEntry
installedDir := path.Join(rootDir, "var/lib/bpm/installed/")
pkgDir := path.Join(installedDir, pkg)
files := path.Join(pkgDir, "files")
@ -1386,15 +1383,46 @@ func GetPackageFiles(pkg, rootDir string) []string {
if _, err := os.Stat(pkgDir); os.IsNotExist(err) {
return nil
}
file, err := os.Open(files)
bs, err := os.ReadFile(files)
if err != nil {
return nil
}
scanner := bufio.NewScanner(file)
for scanner.Scan() {
ret = append(ret, scanner.Text())
for _, line := range strings.Split(string(bs), "\n") {
if strings.TrimSpace(line) == "" {
continue
}
return ret
stringEntry := strings.Split(strings.TrimSpace(line), " ")
if len(stringEntry) < 4 {
pkgFiles = append(pkgFiles, &PackageFileEntry{
Path: line,
UserID: 0,
GroupID: 0,
SizeInBytes: 0,
})
continue
}
uid, err := strconv.ParseInt(stringEntry[len(stringEntry)-3], 0, 32)
if err != nil {
return nil
}
gid, err := strconv.ParseInt(stringEntry[len(stringEntry)-2], 0, 32)
if err != nil {
return nil
}
size, err := strconv.ParseUint(stringEntry[len(stringEntry)-1], 0, 64)
if err != nil {
return nil
}
pkgFiles = append(pkgFiles, &PackageFileEntry{
Path: strings.Join(stringEntry[:len(stringEntry)-3], " "),
UserID: int(uid),
GroupID: int(gid),
SizeInBytes: size,
})
}
return pkgFiles
}
func GetPackageInfo(pkg, rootDir string) *PackageInfo {
@ -1422,6 +1450,17 @@ func GetPackageInfo(pkg, rootDir string) *PackageInfo {
return info
}
func GetPackage(pkg, rootDir string) *BPMPackage {
if !IsPackageInstalled(pkg, rootDir) {
return nil
}
return &BPMPackage{
PkgInfo: GetPackageInfo(pkg, rootDir),
PkgFiles: GetPackageFiles(pkg, rootDir),
}
}
func RemovePackage(pkg string, verbose bool, rootDir string) error {
installedDir := path.Join(rootDir, "var/lib/bpm/installed/")
pkgDir := path.Join(installedDir, pkg)
@ -1429,10 +1468,10 @@ func RemovePackage(pkg string, verbose bool, rootDir string) error {
if pkgInfo == nil {
return errors.New("could not get package info")
}
files := GetPackageFiles(pkg, rootDir)
fileEntries := GetPackageFiles(pkg, rootDir)
var symlinks []string
for _, file := range files {
file = path.Join(rootDir, file)
for _, entry := range fileEntries {
file := path.Join(rootDir, entry.Path)
lstat, err := os.Lstat(file)
if os.IsNotExist(err) {
continue

View File

@ -22,7 +22,10 @@ type Repository struct {
type RepositoryEntry struct {
Info *PackageInfo `yaml:"info"`
Download string `yaml:"download"`
DownloadSize uint64 `yaml:"download_size"`
InstalledSize uint64 `yaml:"installed_size"`
IsVirtualPackage bool `yaml:"-"`
Repository *Repository
}
func (repo *Repository) ContainsPackage(pkg string) bool {
@ -63,7 +66,10 @@ func (repo *Repository) ReadLocalDatabase() error {
Provides: make([]string, 0),
},
Download: "",
DownloadSize: 0,
InstalledSize: 0,
IsVirtualPackage: false,
Repository: repo,
}
err := yaml.Unmarshal([]byte(b), &entry)
if err != nil {
@ -84,7 +90,10 @@ func (repo *Repository) ReadLocalDatabase() error {
entry := RepositoryEntry{
Info: repo.Entries[value[0]].Info,
Download: repo.Entries[value[0]].Download,
DownloadSize: repo.Entries[value[0]].DownloadSize,
InstalledSize: repo.Entries[value[0]].InstalledSize,
IsVirtualPackage: true,
Repository: repo,
}
repo.Entries[key] = &entry
}