diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 646dc337e..39fb2a227 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -51,7 +51,6 @@ code [1], alternatively you can use the script `tools/prepare-source.sh`: $ go test ./... $ go build ./... $ go generate ./... -$ go run ./cmd/osbuild-pipeline/ ``` ### Testing diff --git a/Makefile b/Makefile index ae9eb18ed..1ef65e183 100644 --- a/Makefile +++ b/Makefile @@ -112,7 +112,6 @@ build: - mkdir -p bin go build -o bin/osbuild-composer ./cmd/osbuild-composer/ go build -o bin/osbuild-worker ./cmd/osbuild-worker/ - go build -o bin/osbuild-pipeline ./cmd/osbuild-pipeline/ go build -o bin/osbuild-upload-azure ./cmd/osbuild-upload-azure/ go build -o bin/osbuild-upload-aws ./cmd/osbuild-upload-aws/ go build -o bin/osbuild-upload-gcp ./cmd/osbuild-upload-gcp/ diff --git a/cmd/osbuild-pipeline/main.go b/cmd/osbuild-pipeline/main.go deleted file mode 100644 index 601191e80..000000000 --- a/cmd/osbuild-pipeline/main.go +++ /dev/null @@ -1,261 +0,0 @@ -package main - -import ( - "encoding/json" - "flag" - "fmt" - "io" - "os" - "path" - - "github.com/osbuild/images/pkg/container" - "github.com/osbuild/images/pkg/distro" - "github.com/osbuild/images/pkg/distroregistry" - "github.com/osbuild/images/pkg/ostree" - "github.com/osbuild/osbuild-composer/internal/common" - "github.com/osbuild/osbuild-composer/internal/dnfjson" - - "github.com/osbuild/images/pkg/rpmmd" - "github.com/osbuild/osbuild-composer/internal/blueprint" -) - -type repository struct { - Id string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - BaseURL string `json:"baseurl,omitempty"` - Metalink string `json:"metalink,omitempty"` - MirrorList string `json:"mirrorlist,omitempty"` - GPGKey string `json:"gpgkey,omitempty"` - CheckGPG bool `json:"check_gpg,omitempty"` - CheckRepoGPG bool `json:"repo_check_gpg,omitempty"` - IgnoreSSL bool `json:"ignore_ssl,omitempty"` - ModuleHotfixes *bool `json:"module_hotfixes,omitempty"` - PackageSets []string `json:"package_sets,omitempty"` - RHSM bool `json:"rhsm,omitempty"` -} - -type ostreeOptions struct { - Ref string `json:"ref"` - Parent string `json:"parent"` - URL string `json:"url"` -} - -type composeRequest struct { - Distro string `json:"distro"` - Arch string `json:"arch"` - ImageType string `json:"image-type"` - Blueprint blueprint.Blueprint `json:"blueprint"` - Repositories []repository `json:"repositories"` - OSTree ostreeOptions `json:"ostree"` -} - -// osbuild-pipeline is a utility command and is often run from within the -// source tree. Find the dnf-json binary in case the osbuild-composer package -// isn't installed. This prioritises the local source version over the system -// version if run from within the source tree. -func findDnfJsonBin() string { - locations := []string{"./dnf-json", "/usr/libexec/osbuild-composer/dnf-json", "/usr/lib/osbuild-composer/dnf-json"} - for _, djPath := range locations { - _, err := os.Stat(djPath) - if !os.IsNotExist(err) { - return djPath - } - } - - // can't run: panic - panic(fmt.Sprintf("could not find 'dnf-json' in any of the known paths: %+v", locations)) -} - -func resolveContainers(sourceSpecs []container.SourceSpec, archName string) ([]container.Spec, error) { - if len(sourceSpecs) == 0 { - return nil, nil - } - - resolver := container.NewResolver(archName) - - for _, c := range sourceSpecs { - resolver.Add(c) - } - - return resolver.Finish() -} - -func main() { - var rpmmdArg bool - flag.BoolVar(&rpmmdArg, "rpmmd", false, "output rpmmd struct instead of pipeline manifest") - var seedArg int64 - flag.Int64Var(&seedArg, "seed", 0, "seed for generating manifests (default: 0)") - flag.Parse() - - // Path to composeRequet or '-' for stdin - composeRequestArg := flag.Arg(0) - - composeRequest := &composeRequest{} - if composeRequestArg != "" { - var reader io.Reader - if composeRequestArg == "-" { - reader = os.Stdin - } else { - var err error - reader, err = os.Open(composeRequestArg) - if err != nil { - panic("Could not open compose request: " + err.Error()) - } - } - file, err := io.ReadAll(reader) - if err != nil { - panic("Could not read compose request: " + err.Error()) - } - err = json.Unmarshal(file, &composeRequest) - if err != nil { - panic("Could not parse blueprint: " + err.Error()) - } - } - - distros := distroregistry.NewDefault() - d := distros.GetDistro(composeRequest.Distro) - if d == nil { - _, _ = fmt.Fprintf(os.Stderr, "The provided distribution '%s' is not supported. Use one of these:\n", composeRequest.Distro) - for _, d := range distros.List() { - _, _ = fmt.Fprintln(os.Stderr, " *", d) - } - return - } - - arch, err := d.GetArch(composeRequest.Arch) - if err != nil { - fmt.Fprintf(os.Stderr, "The provided architecture '%s' is not supported by %s. Use one of these:\n", composeRequest.Arch, d.Name()) - for _, a := range d.ListArches() { - _, _ = fmt.Fprintln(os.Stderr, " *", a) - } - return - } - - imageType, err := arch.GetImageType(composeRequest.ImageType) - if err != nil { - fmt.Fprintf(os.Stderr, "The provided image type '%s' is not supported by %s for %s. Use one of these:\n", composeRequest.ImageType, d.Name(), arch.Name()) - for _, t := range arch.ListImageTypes() { - _, _ = fmt.Fprintln(os.Stderr, " *", t) - } - return - } - - repos := make([]rpmmd.RepoConfig, len(composeRequest.Repositories)) - for i, repo := range composeRequest.Repositories { - repoName := repo.Name - if repoName == "" { - repoName = fmt.Sprintf("repo-%d", i) - } - repoId := repo.Id - if repoId == "" { - repoId = fmt.Sprintf("repo-%d", i) - } - var urls []string - if repo.BaseURL != "" { - urls = []string{repo.BaseURL} - } - var keys []string - if repo.GPGKey != "" { - keys = []string{repo.GPGKey} - } - checkGPG := repo.CheckGPG - repos[i] = rpmmd.RepoConfig{ - Id: repoId, - Name: repoName, - BaseURLs: urls, - Metalink: repo.Metalink, - MirrorList: repo.MirrorList, - GPGKeys: keys, - CheckGPG: &checkGPG, - CheckRepoGPG: common.ToPtr(false), - IgnoreSSL: common.ToPtr(false), - ModuleHotfixes: repo.ModuleHotfixes, - PackageSets: repo.PackageSets, - RHSM: repo.RHSM, - } - } - - options := distro.ImageOptions{ - Size: imageType.Size(0), - OSTree: &ostree.ImageOptions{ - ImageRef: composeRequest.OSTree.Ref, - ParentRef: composeRequest.OSTree.Parent, - URL: composeRequest.OSTree.URL, - }, - } - - home, err := os.UserHomeDir() - if err != nil { - panic("os.UserHomeDir(): " + err.Error()) - } - - solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch.Name(), d.Name(), path.Join(home, ".cache/osbuild-composer/rpmmd")) - solver.SetDNFJSONPath(findDnfJsonBin()) - - // Set cache size to 3 GiB - // osbuild-pipeline is often used to generate a lot of manifests in a row - // let the cache grow to fit much more repository metadata than we usually allow - solver.SetMaxCacheSize(3 * 1024 * 1024 * 1024) - - ibp := blueprint.Convert(composeRequest.Blueprint) - manifest, _, err := imageType.Manifest(&ibp, options, repos, seedArg) - if err != nil { - panic(err.Error()) - } - - depsolvedSets := make(map[string][]rpmmd.PackageSpec) - for name, pkgSet := range manifest.GetPackageSetChains() { - res, err := solver.Depsolve(pkgSet) - if err != nil { - panic("Could not depsolve: " + err.Error()) - } - depsolvedSets[name] = res - } - - containerSources := manifest.GetContainerSourceSpecs() - containers := make(map[string][]container.Spec, len(containerSources)) - for name, sourceSpecs := range containerSources { - containerSpecs, err := resolveContainers(sourceSpecs, arch.Name()) - if err != nil { - panic("Could not resolve containers: " + err.Error()) - } - containers[name] = containerSpecs - } - - commitSources := manifest.GetOSTreeSourceSpecs() - commits := make(map[string][]ostree.CommitSpec, len(commitSources)) - for name, commitSources := range commitSources { - commitSpecs := make([]ostree.CommitSpec, len(commitSources)) - for idx, commitSource := range commitSources { - commitSpec, err := ostree.Resolve(commitSource) - if err != nil { - panic("Could not resolve ostree commit: " + err.Error()) - } - commitSpecs[idx] = commitSpec - } - commits[name] = commitSpecs - } - - var bytes []byte - if rpmmdArg { - bytes, err = json.Marshal(depsolvedSets) - if err != nil { - panic(err) - } - } else { - ms, err := manifest.Serialize(depsolvedSets, containers, commits) - if err != nil { - panic(err.Error()) - } - - bytes, err = json.Marshal(ms) - if err != nil { - panic(err) - } - } - os.Stdout.Write(bytes) - if err := solver.CleanCache(); err != nil { - // print to stderr but don't exit with error - fmt.Fprintf(os.Stderr, "Error during rpm repo cache cleanup: %s", err.Error()) - } -}