diff --git a/include/meson.build b/include/meson.build
index 6c1c1dcd4a..1ca0e76cf1 100644
--- a/include/meson.build
+++ b/include/meson.build
@@ -199,7 +199,9 @@ conf_data.set('UNIXCONN', host_machine.system() != 'windows' ? '1' : false)
 conf_data.set('IPv6', build_ipv6 ? '1' : false)
 
 conf_data.set('BIGREQS', '1')
-conf_data.set('COMPOSITE', '1')
+if build_composite
+    conf_data.set('COMPOSITE', '1')
+endif
 conf_data.set('DAMAGE', '1')
 conf_data.set('DBE', '1')
 conf_data.set('DGA', build_dga ? '1' : false)
diff --git a/meson.build b/meson.build
index ff5d271967..546bb4e4b7 100644
--- a/meson.build
+++ b/meson.build
@@ -269,8 +269,10 @@ else
     build_xquartz = get_option('xquartz') == 'true'
 endif
 
+build_composite = true
 build_rootless = false
 if build_xquartz
+    build_composite = false
     build_rootless = true
 endif
 
@@ -761,7 +763,9 @@ subdir('fb')
 subdir('mi')
 subdir('os')
 # X extensions
-subdir('composite')
+if build_composite
+    subdir('composite')
+endif
 subdir('damageext')
 subdir('dbe')
 subdir('miext/damage')
@@ -795,7 +799,6 @@ libxserver = [
     libxserver_mi,
     libxserver_dix,
 
-    libxserver_composite,
     libxserver_damageext,
     libxserver_dbe,
     libxserver_randr,
@@ -812,6 +815,10 @@ libxserver = [
     libxserver_os,
 ]
 
+if build_composite
+    libxserver += libxserver_composite
+endif
+
 libxserver += libxserver_dri3
 
 subdir('hw')