diff --git a/cmd/osbuild-worker/main.go b/cmd/osbuild-worker/main.go
index fa14dbff5..b124c5618 100644
--- a/cmd/osbuild-worker/main.go
+++ b/cmd/osbuild-worker/main.go
@@ -67,9 +67,7 @@ func newConnection(remoteAddress string) (net.Conn, error) {
 			return nil, err
 		}
 
-		address := fmt.Sprintf("%s:%d", remoteAddress, RemoteWorkerPort)
-
-		return tls.Dial("tcp", address, conf)
+		return tls.Dial("tcp", remoteAddress, conf)
 	}
 
 	// plain non-encrypted connection
@@ -191,6 +189,10 @@ func main() {
 	flag.StringVar(&remoteAddress, "remote", "", "Connect to a remote composer using the specified address")
 	flag.Parse()
 
+	if remoteAddress != "" {
+		remoteAddress = fmt.Sprintf("%s:%d", remoteAddress, RemoteWorkerPort)
+	}
+
 	client := NewClient(remoteAddress)
 	for {
 		if err := handleJob(client); err != nil {