genall.go: create all test manifests
- Standalone executable for generating all test manifests in parallel.
- Command line flags:
- Output directory (-output)
- Number of concurrent workers (-workers)
- Collects list of image types from the distro list and reads:
- tools/test-case-generators/repos.json for repositories
- tools/test-case-generators/format-request-map.json for
customizations
- Prints progress (finished/total)
- Collects errors and failures and prints them after all jobs are
finished
This commit is contained in:
Achilleas Koutsou
Tom Gundersen
parent
41ab3e0f44
commit
0ef4e7ee84
1 changed files with 515 additions and 0 deletions
515
tools/test-case-generators/genall.go
Normal file
515
tools/test-case-generators/genall.go
Normal file
|
|
@ -0,0 +1,515 @@
|
|||
// Standalone executable for generating all test manifests in parallel.
|
||||
// Collects list of image types from the distro list. Must be run from the
|
||||
// root of the repository and reads tools/test-case-generators/repos.json for
|
||||
// repositories tools/test-case-generators/format-request-map.json for
|
||||
// customizations Collects errors and failures and prints them after all jobs
|
||||
// are finished.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/osbuild/osbuild-composer/internal/blueprint"
|
||||
"github.com/osbuild/osbuild-composer/internal/distro"
|
||||
"github.com/osbuild/osbuild-composer/internal/distroregistry"
|
||||
"github.com/osbuild/osbuild-composer/internal/dnfjson"
|
||||
"github.com/osbuild/osbuild-composer/internal/ostree"
|
||||
"github.com/osbuild/osbuild-composer/internal/rpmmd"
|
||||
)
|
||||
|
||||
type repository struct {
|
||||
Name string `json:"name"`
|
||||
BaseURL string `json:"baseurl,omitempty"`
|
||||
Metalink string `json:"metalink,omitempty"`
|
||||
MirrorList string `json:"mirrorlist,omitempty"`
|
||||
GPGKey string `json:"gpgkey,omitempty"`
|
||||
CheckGPG bool `json:"check_gpg,omitempty"`
|
||||
RHSM bool `json:"rhsm,omitempty"`
|
||||
MetadataExpire string `json:"metadata_expire,omitempty"`
|
||||
ImageTypeTags []string `json:"image_type_tags,omitempty"`
|
||||
}
|
||||
|
||||
type ostreeOptions struct {
|
||||
Ref string `json:"ref"`
|
||||
URL string `json:"url"`
|
||||
Parent string `json:"parent"`
|
||||
}
|
||||
|
||||
type crBlueprint struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Version string `json:"version,omitempty"`
|
||||
Packages []blueprint.Package `json:"packages,omitempty"`
|
||||
Modules []blueprint.Package `json:"modules,omitempty"`
|
||||
Groups []blueprint.Group `json:"groups,omitempty"`
|
||||
Customizations *blueprint.Customizations `json:"customizations,omitempty"`
|
||||
Distro string `json:"distro,omitempty"`
|
||||
}
|
||||
|
||||
type composeRequest struct {
|
||||
Distro string `json:"distro,omitempty"`
|
||||
Arch string `json:"arch,omitempty"`
|
||||
ImageType string `json:"image-type,omitempty"`
|
||||
Repositories []repository `json:"repositories,omitempty"`
|
||||
Filename string `json:"filename,omitempty"`
|
||||
OSTree *ostreeOptions `json:"ostree,omitempty"`
|
||||
Blueprint *crBlueprint `json:"blueprint,omitempty"`
|
||||
}
|
||||
|
||||
type manifestRequest struct {
|
||||
ComposeRequest composeRequest `json:"compose-request"`
|
||||
Overrides map[string]composeRequest `json:"overrides"`
|
||||
SupportedArches []string `json:"supported_arches"`
|
||||
}
|
||||
|
||||
type formatRequestMap map[string]manifestRequest
|
||||
|
||||
func loadFormatRequestMap() formatRequestMap {
|
||||
requestMapPath := "./tools/test-case-generators/format-request-map.json"
|
||||
fp, err := os.Open(requestMapPath)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("failed to open format request map %q: %s", requestMapPath, err.Error()))
|
||||
}
|
||||
defer fp.Close()
|
||||
data, err := io.ReadAll(fp)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("failed to read format request map %q: %s", requestMapPath, err.Error()))
|
||||
}
|
||||
var frm formatRequestMap
|
||||
if err := json.Unmarshal(data, &frm); err != nil {
|
||||
panic(fmt.Sprintf("failed to unmarshal format request map %q: %s", requestMapPath, err.Error()))
|
||||
}
|
||||
|
||||
return frm
|
||||
}
|
||||
|
||||
type manifestJob func(chan string) error
|
||||
|
||||
func makeManifestJob(name string, imgType distro.ImageType, cr composeRequest, distribution distro.Distro, archName string, seedArg int64, path string) (manifestJob, string) {
|
||||
distroName := distribution.Name()
|
||||
u := func(s string) string {
|
||||
return strings.Replace(s, "-", "_", -1)
|
||||
}
|
||||
filename := fmt.Sprintf("%s-%s-%s-boot.json", u(distroName), u(archName), u(name))
|
||||
workerName := archName + distribution.Name()
|
||||
cacheDir := filepath.Join("/tmp", "rpmmd", workerName)
|
||||
|
||||
options := distro.ImageOptions{Size: 0}
|
||||
if cr.OSTree != nil {
|
||||
options.OSTree = ostree.RequestParams{
|
||||
URL: cr.OSTree.URL,
|
||||
Ref: cr.OSTree.Ref,
|
||||
Parent: cr.OSTree.Parent,
|
||||
}
|
||||
}
|
||||
job := func(msgq chan string) error {
|
||||
defer func() { msgq <- fmt.Sprintf("Finished job %s", filename) }()
|
||||
msgq <- fmt.Sprintf("Starting job %s", filename)
|
||||
repos := convertRepos(cr.Repositories)
|
||||
var bp blueprint.Blueprint
|
||||
if cr.Blueprint != nil {
|
||||
bp = blueprint.Blueprint(*cr.Blueprint)
|
||||
}
|
||||
|
||||
packageSpecs, err := depsolve(cacheDir, imgType, bp, repos, distribution, archName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("[%s] depsolve failed: %s", filename, err.Error())
|
||||
}
|
||||
if packageSpecs == nil {
|
||||
return fmt.Errorf("[%s] nil package specs", filename)
|
||||
}
|
||||
if options.OSTree.Ref == "" {
|
||||
// use default OSTreeRef for image type
|
||||
options.OSTree.Ref = imgType.OSTreeRef()
|
||||
}
|
||||
manifest, err := imgType.Manifest(cr.Blueprint.Customizations, options, repos, packageSpecs, seedArg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("[%s] failed: %s", filename, err)
|
||||
}
|
||||
request := composeRequest{
|
||||
Distro: distribution.Name(),
|
||||
Arch: archName,
|
||||
ImageType: cr.ImageType,
|
||||
Repositories: cr.Repositories,
|
||||
Filename: cr.Filename,
|
||||
Blueprint: cr.Blueprint,
|
||||
OSTree: cr.OSTree,
|
||||
}
|
||||
return save(manifest, packageSpecs, request, path, filename)
|
||||
}
|
||||
return job, workerName
|
||||
}
|
||||
|
||||
type DistroArchRepoMap map[string]map[string][]repository
|
||||
|
||||
func convertRepo(r repository) rpmmd.RepoConfig {
|
||||
return rpmmd.RepoConfig{
|
||||
Name: r.Name,
|
||||
BaseURL: r.BaseURL,
|
||||
Metalink: r.Metalink,
|
||||
MirrorList: r.MirrorList,
|
||||
GPGKey: r.GPGKey,
|
||||
CheckGPG: r.CheckGPG,
|
||||
MetadataExpire: r.MetadataExpire,
|
||||
ImageTypeTags: r.ImageTypeTags,
|
||||
}
|
||||
}
|
||||
|
||||
func convertRepos(rr []repository) []rpmmd.RepoConfig {
|
||||
cr := make([]rpmmd.RepoConfig, len(rr))
|
||||
for idx, r := range rr {
|
||||
cr[idx] = convertRepo(r)
|
||||
}
|
||||
return cr
|
||||
}
|
||||
|
||||
func readRepos() DistroArchRepoMap {
|
||||
file := "./tools/test-case-generators/repos.json"
|
||||
var darm DistroArchRepoMap
|
||||
fp, err := os.Open(file)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer fp.Close()
|
||||
data, err := io.ReadAll(fp)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := json.Unmarshal(data, &darm); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return darm
|
||||
}
|
||||
|
||||
func depsolve(cacheDir string, imageType distro.ImageType, bp blueprint.Blueprint, repos []rpmmd.RepoConfig, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, error) {
|
||||
solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, cacheDir)
|
||||
solver.SetDNFJSONPath("./dnf-json")
|
||||
packageSets := imageType.PackageSets(bp, repos)
|
||||
depsolvedSets := make(map[string][]rpmmd.PackageSpec)
|
||||
for name, pkgSet := range packageSets {
|
||||
res, err := solver.Depsolve(pkgSet)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
depsolvedSets[name] = res.Dependencies
|
||||
}
|
||||
return depsolvedSets, nil
|
||||
}
|
||||
|
||||
func save(manifest distro.Manifest, pkgs map[string][]rpmmd.PackageSpec, cr composeRequest, path, filename string) error {
|
||||
data := struct {
|
||||
ComposeRequest composeRequest `json:"compose-request"`
|
||||
Manifest distro.Manifest `json:"manifest"`
|
||||
RPMMD map[string][]rpmmd.PackageSpec `json:"rpmmd"`
|
||||
NoImageInfo bool `json:"no-image-info"`
|
||||
}{
|
||||
cr, manifest, pkgs, true,
|
||||
}
|
||||
b, err := json.MarshalIndent(data, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal data for %q: %s\n", filename, err.Error())
|
||||
}
|
||||
b = append(b, '\n') // add new line at end of file
|
||||
fpath := filepath.Join(path, filename)
|
||||
fp, err := os.Create(fpath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create output file %q: %s\n", fpath, err.Error())
|
||||
}
|
||||
defer fp.Close()
|
||||
if _, err := fp.Write(b); err != nil {
|
||||
return fmt.Errorf("failed to write output file %q: %s\n", fpath, err.Error())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func filterRepos(repos []repository, typeName string) []repository {
|
||||
filtered := make([]repository, 0)
|
||||
for _, repo := range repos {
|
||||
if len(repo.ImageTypeTags) == 0 {
|
||||
filtered = append(filtered, repo)
|
||||
} else {
|
||||
for _, tt := range repo.ImageTypeTags {
|
||||
if tt == typeName {
|
||||
filtered = append(filtered, repo)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
// collects requests from a formatRequestMap based on image type
|
||||
func requestsByImageType(requestMap formatRequestMap) map[string]map[string]manifestRequest {
|
||||
imgTypeRequestMap := make(map[string]map[string]manifestRequest)
|
||||
|
||||
for name, req := range requestMap {
|
||||
it := req.ComposeRequest.ImageType
|
||||
reqs := imgTypeRequestMap[it]
|
||||
if reqs == nil {
|
||||
reqs = make(map[string]manifestRequest)
|
||||
}
|
||||
reqs[name] = req
|
||||
imgTypeRequestMap[it] = reqs
|
||||
}
|
||||
return imgTypeRequestMap
|
||||
}
|
||||
|
||||
type workerQueue struct {
|
||||
// manifest job channel
|
||||
jobQueue chan manifestJob
|
||||
|
||||
// channel for sending messages from jobs to the printer
|
||||
msgQueue chan string
|
||||
|
||||
// channel for sending errors from jobs to the collector
|
||||
errQueue chan error
|
||||
|
||||
// global error list
|
||||
errors []error
|
||||
|
||||
// total job count defined on workerQueue creation
|
||||
// sets the length of the job queue so that pushing to the queue doesn't block
|
||||
njobs uint32
|
||||
|
||||
// total workers defined on workerQueue creation
|
||||
nworkers uint32
|
||||
|
||||
// active worker count
|
||||
activeWorkers int32
|
||||
|
||||
// wait group for all workers
|
||||
workerWG sync.WaitGroup
|
||||
|
||||
// wait group for internal routines (printer and error collector)
|
||||
utilWG sync.WaitGroup
|
||||
}
|
||||
|
||||
func newWorkerQueue(nworkers uint32, njobs uint32) *workerQueue {
|
||||
wq := workerQueue{
|
||||
jobQueue: make(chan manifestJob, njobs),
|
||||
msgQueue: make(chan string, nworkers),
|
||||
errQueue: make(chan error, nworkers),
|
||||
errors: make([]error, 0, nworkers),
|
||||
nworkers: nworkers,
|
||||
activeWorkers: 0,
|
||||
njobs: njobs,
|
||||
}
|
||||
|
||||
return &wq
|
||||
}
|
||||
|
||||
func (wq *workerQueue) start() {
|
||||
wq.startMessagePrinter()
|
||||
wq.startErrorCollector()
|
||||
for idx := uint32(0); idx < wq.nworkers; idx++ {
|
||||
wq.startWorker(idx)
|
||||
}
|
||||
}
|
||||
|
||||
func (wq *workerQueue) wait() []error {
|
||||
wq.finish()
|
||||
return wq.errors
|
||||
}
|
||||
|
||||
// close all queues and wait for waitgroups
|
||||
func (wq *workerQueue) finish() {
|
||||
// close job channel and wait for workers to finish
|
||||
close(wq.jobQueue)
|
||||
wq.workerWG.Wait()
|
||||
|
||||
// close message channels and wait for them to finish their work so we don't miss any messages or errors
|
||||
close(wq.msgQueue)
|
||||
close(wq.errQueue)
|
||||
wq.utilWG.Wait()
|
||||
}
|
||||
|
||||
func (wq *workerQueue) startWorker(idx uint32) {
|
||||
wq.workerWG.Add(1)
|
||||
go func() {
|
||||
atomic.AddInt32(&(wq.activeWorkers), 1)
|
||||
defer atomic.AddInt32(&(wq.activeWorkers), -1)
|
||||
defer wq.workerWG.Done()
|
||||
for job := range wq.jobQueue {
|
||||
err := job(wq.msgQueue)
|
||||
if err != nil {
|
||||
wq.errQueue <- err
|
||||
}
|
||||
}
|
||||
}()
|
||||
wq.msgQueue <- fmt.Sprintf("Worker %d started", idx)
|
||||
}
|
||||
|
||||
func (wq *workerQueue) startMessagePrinter() {
|
||||
wq.utilWG.Add(1)
|
||||
go func() {
|
||||
defer wq.utilWG.Done()
|
||||
var msglen int
|
||||
for {
|
||||
select {
|
||||
case msg, open := <-wq.msgQueue:
|
||||
// clear previous line (avoids leftover trailing characters from progress)
|
||||
fmt.Printf(strings.Repeat(" ", msglen) + "\r")
|
||||
if !open {
|
||||
fmt.Println()
|
||||
return
|
||||
}
|
||||
fmt.Println(msg)
|
||||
default:
|
||||
msglen, _ = fmt.Printf(" == Jobs == Queue: %4d Active: %4d Total: %4d\r", len(wq.jobQueue), wq.activeWorkers, wq.njobs)
|
||||
// sleep a bit when printing progress to avoid constantly pushing out the same progress message
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (wq *workerQueue) startErrorCollector() {
|
||||
wq.utilWG.Add(1)
|
||||
go func() {
|
||||
defer wq.utilWG.Done()
|
||||
for err := range wq.errQueue {
|
||||
wq.errors = append(wq.errors, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (wq *workerQueue) submitJob(j manifestJob) {
|
||||
wq.jobQueue <- j
|
||||
}
|
||||
|
||||
func archIsSupported(req manifestRequest, arch string) bool {
|
||||
if len(req.SupportedArches) == 0 {
|
||||
// none specified: all arches supported implicitly
|
||||
return true
|
||||
}
|
||||
for _, supportedArch := range req.SupportedArches {
|
||||
if supportedArch == arch {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func mergeOverrides(base, overrides composeRequest) composeRequest {
|
||||
// NOTE: in most cases overrides are only used for blueprints and probably
|
||||
// doesn't make sense to use them for most fields, but let's merge all
|
||||
// regardless
|
||||
merged := composeRequest(base)
|
||||
if overrides.Blueprint != nil {
|
||||
merged.Blueprint = overrides.Blueprint
|
||||
}
|
||||
|
||||
if overrides.Filename != "" {
|
||||
merged.Filename = overrides.Filename
|
||||
}
|
||||
if overrides.ImageType != "" {
|
||||
merged.ImageType = overrides.ImageType
|
||||
}
|
||||
if overrides.OSTree != nil {
|
||||
merged.OSTree = overrides.OSTree
|
||||
}
|
||||
if overrides.Distro != "" {
|
||||
merged.Distro = overrides.Distro
|
||||
}
|
||||
if overrides.Arch != "" {
|
||||
merged.Arch = overrides.Arch
|
||||
}
|
||||
if len(overrides.Repositories) > 0 {
|
||||
merged.Repositories = overrides.Repositories
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
func main() {
|
||||
outputDirFlag := flag.String("output", "test/data/manifests.plain/", "")
|
||||
nWorkersFlag := flag.Int("workers", 16, "")
|
||||
flag.Parse()
|
||||
|
||||
outputDir := *outputDirFlag
|
||||
nWorkers := *nWorkersFlag
|
||||
|
||||
seedArg := int64(0)
|
||||
darm := readRepos()
|
||||
distros := distroregistry.NewDefault()
|
||||
jobs := make([]manifestJob, 0)
|
||||
|
||||
requestMap := loadFormatRequestMap()
|
||||
itRequestMap := requestsByImageType(requestMap)
|
||||
|
||||
if err := os.MkdirAll(outputDir, 0770); err != nil {
|
||||
panic(fmt.Sprintf("failed to create target directory: %s", err.Error()))
|
||||
}
|
||||
|
||||
fmt.Println("Collecting jobs")
|
||||
for _, distroName := range distros.List() {
|
||||
distribution := distros.GetDistro(distroName)
|
||||
for _, archName := range distribution.ListArches() {
|
||||
arch, err := distribution.GetArch(archName)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("distro %q lists arch %q but GetArch() failed: %s", distroName, archName, err.Error()))
|
||||
}
|
||||
for _, imgTypeName := range arch.ListImageTypes() {
|
||||
imgType, err := arch.GetImageType(imgTypeName)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("distro %q (%q) lists image type %q but GetImageType() failed: %s\n", distroName, archName, imgTypeName, err.Error()))
|
||||
}
|
||||
|
||||
// get repositories
|
||||
repos := darm[distroName][archName]
|
||||
if len(repos) == 0 {
|
||||
fmt.Printf("no repositories defined for %s/%s\n", distroName, archName)
|
||||
fmt.Println("Skipping")
|
||||
continue
|
||||
}
|
||||
|
||||
// run through jobs from request map that match the image type
|
||||
for jobName, req := range itRequestMap[imgTypeName] {
|
||||
// skip if architecture is not supported
|
||||
if !archIsSupported(req, archName) {
|
||||
continue
|
||||
}
|
||||
|
||||
// check for distro-specific overrides
|
||||
if or, exist := req.Overrides[distroName]; exist {
|
||||
req.ComposeRequest = mergeOverrides(req.ComposeRequest, or)
|
||||
}
|
||||
|
||||
composeReq := req.ComposeRequest
|
||||
composeReq.Repositories = filterRepos(repos, imgTypeName)
|
||||
|
||||
job, _ := makeManifestJob(jobName, imgType, composeReq, distribution, archName, seedArg, outputDir)
|
||||
jobs = append(jobs, job)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
nJobs := len(jobs)
|
||||
fmt.Printf("Collected %d jobs\n", nJobs)
|
||||
wq := newWorkerQueue(uint32(nWorkers), uint32(nJobs))
|
||||
wq.start()
|
||||
fmt.Printf("Initialised %d workers\n", nWorkers)
|
||||
fmt.Printf("Submitting %d jobs... ", nJobs)
|
||||
for _, j := range jobs {
|
||||
wq.submitJob(j)
|
||||
}
|
||||
errs := wq.wait()
|
||||
if len(errs) > 0 {
|
||||
fmt.Printf("Encountered %d errors:\n", len(errs))
|
||||
for idx, err := range errs {
|
||||
fmt.Printf("%3d: %s\n", idx, err.Error())
|
||||
}
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue