<script type="text/javascript" language="javascript">
function CustomValidator1_ClientValidate(source,args)
{ 
var listbox = document.getElementById("<%=ListBox1.ClientID %>");
var total = 0;
var i=0;
for( i = 0; i < listbox.options.length; i++ )
{
 if (listbox.options[i].selected) 
  {
   total ++; 
    if (total >10) {
      args.IsValid = false;
      return;
  }
  } 
 } 
 args.IsValid = true; return;
}
</script>
 
<asp:ListBox ID="ListBox1" runat="server" SelectionMode="Multiple" Height="100%"
Width="100%"></asp:ListBox>
<asp:Button ID="Button3" runat="server" OnClick="Button1_Click" Text="Button" />
<asp:CustomValidator ID="CustomValidator1" runat="server" Display="Dynamic" ErrorMessage="Please select 10 items only."
ClientValidationFunction="CustomValidator1_ClientValidate" OnServerValidate="CustomValidator1_ServerValidate">
</asp:CustomValidator>

Code Behind:

protected void CustomValidator1_ServerValidate(object source, ServerValidateEventArgs args) 
{ 
  int count = 0; 
  for (int i = 0; i < ListBox1.Items.Count; i++) 
  { 
        if (ListBox1.Items[i].Selected) 
            count++; 
  } 
 args.IsValid = (count > 10) ? false : true; 
}