After a common install of vmware-server...

vm-win ~ # emerge vmware-server -pv

These are the packages that would be merged, in order:

Calculating dependencies... done!
[ebuild   Rf  ] app-emulation/vmware-server-2.0.2.203138-r1  0 kB

Total: 1 package (1 reinstall), Size of downloads: 0 kB
Fetch Restriction: 1 package

 * IMPORTANT: 3 news items need reading for repository 'gentoo'.
 * Use eselect news to read news items.

... our vmware-server cannot start, to solve this problem we need to work with some PaX utils and applications, such as paxctl and chpax, in order to change the rules of security but only with some applications.

vm-win ~ # emerge pax-utils pax paxctl paxtest -pv
These are the packages that would be merged, in order:

Calculating dependencies... done!
[ebuild   R   ] sys-apps/paxctl-0.5  0 kB
[ebuild   R   ] app-misc/pax-utils-0.2.1  USE="caps" 0 kB
[ebuild   R   ] app-arch/pax-3.3.4  0 kB
[ebuild   R   ] app-admin/paxtest-0.9.9-r1  0 kB

Total: 4 packages (4 reinstalls), Size of downloads: 0 kB

 * IMPORTANT: 4 news items need reading for repository 'gentoo'.
 * Use eselect news to read news items.

Here are some options.

vm-win ~ # chpax --help
chpax 0.7 .::. Manage PaX flags for binaries
Usage: chpax OPTIONS FILE1 FILE2 FILEN ...
  -P	enforce paging based non-executable pages
  -p	do not enforce paging based non-executable pages
  -E	emulate trampolines
  -e	do not emulate trampolines
  -M	restrict mprotect()
  -m	do not restrict mprotect()
  -R	randomize mmap() base [ELF only]
  -r	do not randomize mmap() base [ELF only]
  -X	randomize ET_EXEC base [ELF only]
  -x	do not randomize ET_EXEC base [ELF only]
  -S	enforce segmentation based non-executable pages
  -s	do not enforce segmentation based non-executable pages
  -v	view current flag mask 
  -z	zero flag mask (next flags still apply)

The flags only have effect when running the patched Linux kernel.
vm-win ~ # paxctl -h

PaX control v0.5
Copyright 2004,2005,2006,2007 PaX Team 

usage: paxctl 

options:
	-p: disable PAGEEXEC		-P: enable PAGEEXEC
	-e: disable EMUTRMAP		-E: enable EMUTRMAP
	-m: disable MPROTECT		-M: enable MPROTECT
	-r: disable RANDMMAP		-R: enable RANDMMAP
	-x: disable RANDEXEC		-X: enable RANDEXEC
	-s: disable SEGMEXEC		-S: enable SEGMEXEC

	-v: view flags			-z: restore default flags
	-q: suppress error messages	-Q: report flags in short format
	-c: convert PT_GNU_STACK into PT_PAX_FLAGS (see manpage!)
	-C: create PT_PAX_FLAGS (see manpage!)

The next script will change security policy rules of the vmware-server-2.0.2.
First the config and opts file.

vm-win ~ # cat /etc/conf.d/grsecurity

# /etc/conf.d/grsecurity

CHPAX_CMD="/sbin/chpax"
CHPAX_OPTIONS="-rsm"

PAXCTL_CMD="/sbin/paxctl"
PAXCTL_OPTIONS="-pemrxs"
PAXCTL_OPTIONS_="-z"
PAXCTL_CONVERT="-c"

Then the init script.

vm-win ~ # cat /etc/init.d/grsecurity

#!/sbin/runscript
# Copyright 1999-2010 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
# $Header: $

depend() {
	before vmware
}

start() {
	ebegin "Setting grsecurity rules and options"
	${PAXCTL_CMD} ${PAXCTL_CONVERT} /opt/vmware/server/lib/bin/vmware-hostd
	${PAXCTL_CMD} ${PAXCTL_OPTIONS} /opt/vmware/server/lib/bin/vmware-hostd

	${PAXCTL_CMD} ${PAXCTL_CONVERT} /opt/vmware/server/bin/vmnet-bridge
	${PAXCTL_CMD} ${PAXCTL_OPTIONS} /opt/vmware/server/bin/vmnet-bridge

	${PAXCTL_CMD} ${PAXCTL_CONVERT} /opt/vmware/server/lib/bin/vmware-vmx
	${PAXCTL_CMD} ${PAXCTL_OPTIONS} /opt/vmware/server/lib/bin/vmware-vmx
}

stop(){
	ebegin "Restoring default grsecurity rules and options"
	${PAXCTL_CMD} ${PAXCTL_CONVERT} /opt/vmware/server/lib/bin/vmware-hostd
        ${PAXCTL_CMD} ${PAXCTL_OPTIONS_} /opt/vmware/server/lib/bin/vmware-hostd

	${PAXCTL_CMD} ${PAXCTL_CONVERT} /opt/vmware/server/bin/vmnet-bridge
        ${PAXCTL_CMD} ${PAXCTL_OPTIONS_} /opt/vmware/server/bin/vmnet-bridge

	${PAXCTL_CMD} ${PAXCTL_CONVERT} /opt/vmware/server/lib/bin/vmware-vmx
        ${PAXCTL_CMD} ${PAXCTL_OPTIONS_} /opt/vmware/server/lib/bin/vmware-vmx
}

restart() {
	stop
	start
}

I know!!!!.... This is a really ugly script, but it worked in a "Hardened Gentoo [server] " security level.

┌────────────────────────── Security Level ──────────────────────────┐                                                                  
│  Use the arrow keys to navigate this window or press the hotkey of │                                                                  
│  the item you wish to select followed by the . Press    │                                                                  
│  > for additional information about this option.                 │                                                                  
│ ┌────────────────────────────────────────────────────────────────┐ │                                                                  
│ │           ( ) Low                                              │ │                                                                  
│ │           ( ) Medium                                           │ │                                                                  
│ │           ( ) High                                             │ │                                                                  
│ │           (X) Hardened Gentoo [server]                         │ │                                                                  
│ │           ( ) Hardened Gentoo [server no rbac]                 │ │                                                                  
│ │           ( ) Hardened Gentoo [workstation]                    │ │                                                                  
│ └───────────────v(+)─────────────────────────────────────────────┘ │                                                                  
├────────────────────────────────────────────────────────────────────┤                                                                  
│                                                    │                                                                  
└────────────────────────────────────────────────────────────────────┘                                                                  
vm-win ~ # uname -a

Linux vm-win 2.6.35-hardened #7 SMP Thu Sep 30 22:13:03 GMT 2010 x86_64 Intel(R) Xeon(R) CPU L5410 @ 2.33GHz GenuineIntel GNU/Linux

And now... all is running correctly.

vm-win ~ # ps ax

  PID TTY      STAT   TIME COMMAND
    1 ?        Ss     0:00 init [3]  

   ..........

 2010 ?        Ss     0:00 /usr/bin/svscan /service
 2038 ?        Ss     0:00 /usr/sbin/cron
 2125 ?        Ss     0:00 /opt/vmware/server/bin/vmnet-bridge -d /var/run/vmnet-bridge-0.pid -n 0 -i eth0
 2404 ?        Ss     0:00 /opt/vmware/server/sbin/vmware-authdlauncher
 2418 ?        S      0:00 /bin/sh /opt/vmware/server/bin/vmware-watchdog -s webAccess -u 30 -q 5 /opt/vmware/server/lib/webAccess/java/jre1.5.0_15/bin/webAccess -client -Xmx64m -XX:MinHeapFreeRatio=30 
 2441 ?        Ssl    0:29 /opt/vmware/server/lib/webAccess/java/jre1.5.0_15/bin/webAccess -client -Xmx64m -XX:MinHeapFreeRatio=30 -XX:MaxHeapFreeRatio=30 -Djava.util.logging.manager=org.apache.juli.Cla
 2523 ?        Ssl    0:21 /opt/vmware/server/lib/bin/vmware-hostd -a -d -u /etc/vmware/hostd/config.xml
 2576 ?        Ssl    9:31 /opt/vmware/server/lib/bin/vmware-vmx -# product=2;name=VMware Server;version=2.0.2;buildnumber=203138;licensename=VMware GSX Server for Linux;licenseversion=3.0 build-203138;
 2642 ?        Ssl    3:34 /opt/vmware/server/lib/bin/vmware-vmx -# product=2;name=VMware Server;version=2.0.2;buildnumber=203138;licensename=VMware GSX Server for Linux;licenseversion=3.0 build-203138;
 2676 ?        Ssl    3:48 /opt/vmware/server/lib/bin/vmware-vmx -# product=2;name=VMware Server;version=2.0.2;buildnumber=203138;licensename=VMware GSX Server for Linux;licenseversion=3.0 build-203138;