nice workaround from JMP support: TS-00198153
use row states, they don't interfere with the possibility to control the appearance for different overlay groups.

dt = Open( "$SAMPLE_DATA/Big Class Families.jmp" );
// Color or mark by column
Data Table( "Big Class Families" ) << Color or Mark by Column(
:height,
Color Theme( "Spectral" ),
Marker( 0 ),
Continuous Scale( 1 )
);
dt << Graph Builder(
Size( 437, 453 ),
Show Control Panel( 0 ),
Variables( X( :age ), Y( :weight ), Overlay( :sex ) ),
Elements( Points( X, Y, Legend( 6 ) ) ),
SendToReport(
Dispatch( {}, "400", ScaleBox,
{Legend Model(
6,
Properties(
0,
{Line Color( 0 ), Marker( "FilledCircle" ), Transparency( 0.2 )},
Item ID( "F", 1 )
),
Properties(
1,
{Line Color( 0 ), Marker( "FilledSquare" )},
Item ID( "M", 1 )
)
)}
),
Dispatch( {}, "Graph Builder", FrameBox,
{Marker Size( 5 ), Marker Drawing Mode( "Normal" ),
Marker Selection Mode( "Selected Outlined" )}
)
)
);