diff --git a/cmd/osbuild-composer/composer.go b/cmd/osbuild-composer/composer.go
index 13523db08..3070e644e 100644
--- a/cmd/osbuild-composer/composer.go
+++ b/cmd/osbuild-composer/composer.go
@@ -65,7 +65,7 @@ func NewComposer(config *ComposerConfigFile, stateDir, cacheDir string, logger *
 		return nil, fmt.Errorf("cannot create jobqueue: %v", err)
 	}
 
-	c.workers = worker.NewServer(c.logger, jobs, artifactsDir, c.config.WorkerAPI.IdentityFilter)
+	c.workers = worker.NewServer(c.logger, jobs, artifactsDir, c.config.Worker.IdentityFilter)
 
 	return &c, nil
 }
@@ -108,7 +108,7 @@ func (c *Composer) InitLocalWorker(l net.Listener) {
 }
 
 func (c *Composer) InitRemoteWorkers(cert, key string, l net.Listener) error {
-	if len(c.config.WorkerAPI.IdentityFilter) > 0 {
+	if len(c.config.Worker.IdentityFilter) > 0 {
 		c.workerListener = l
 	} else {
 		tlsConfig, err := createTLSConfig(&connectionConfig{
diff --git a/cmd/osbuild-composer/config.go b/cmd/osbuild-composer/config.go
index 6a2f0c9f5..4f25f1538 100644
--- a/cmd/osbuild-composer/config.go
+++ b/cmd/osbuild-composer/config.go
@@ -14,13 +14,11 @@ type ComposerConfigFile struct {
 	Worker struct {
 		AllowedDomains []string `toml:"allowed_domains"`
 		CA             string   `toml:"ca"`
+		IdentityFilter []string `toml:"identity_filter"`
 	} `toml:"worker"`
 	ComposerAPI struct {
 		IdentityFilter []string `toml:"identity_filter"`
 	} `toml:"composer_api"`
-	WorkerAPI struct {
-		IdentityFilter []string `toml:"identity_filter"`
-	} `toml:"worker_api"`
 }
 
 func LoadConfig(name string) (*ComposerConfigFile, error) {