• In API manager we have provided many extension points to extend functionalities based on custom requirements.
• Workflow
• JWT generation(http://wso2.com/library/articles/2014/12/customize-json-web-token-generation-with-wso2-api-manager-1.8.0/)
• Claim retrieve
• Adding new handlers
• Extension handlers
• Sequence support.
• Mediation extension(https://docs.wso2.com/display/AM160/Adding+a+Mediation+Extension)\
• How to pass 2 auth headers in same request(one for API Manager and one for backend).(http://sanjeewamalalgoda.blogspot.com/2014/11/how-to-use-custom-authentication-header.html)
• Enable mutual SSL for server to server communication. We can use this for API Manager gateway to key manager validation call.(http://sanjeewamalalgoda.blogspot.com/2014/11/how-to-enable-mutual-ssl-connection.html)

 vi test.sh 

#!/bin/bash
# 1: ['command\ name' or PID number(,s)] 2: MAX_CPU_PERCENT
[[ $# -ne 2 ]] && exit 1
PID_NAMES=$1
# get all PIDS as nn,nn,nn
if [[ ! "$PID_NAMES" =~ ^[0-9,]+$ ]] ; then
    PIDS=$(pgrep -d ',' -x $PID_NAMES)
else
    PIDS=$PID_NAMES
fi
#  echo "$PIDS $MAX_CPU"
MAX_CPU="$2"
MAX_CPU="$(echo "($MAX_CPU+0.5)/1" | bc)"
LOOP=1
while [[ $LOOP -eq 1 ]] ; do
    sleep 0.3s
    # Depending on your 'top' version and OS you might have
    #   to change head and tail line-numbers
    LINE="$(top -b -d 0 -n 1 -p $PIDS | head -n 8 \
        | tail -n 1 | sed -r 's/[ ]+/,/g' | \
        sed -r 's/^\,|\,$//')"
    # If multiple processes in $PIDS, $LINE will only match\
    #   the most active process
    CURR_PID=$(echo "$LINE" | cut -d ',' -f 1)
    # calculate cpu limits
    CURR_CPU_FLOAT=$(echo "$LINE"| cut -d ',' -f 9)
    CURR_CPU=$(echo "($CURR_CPU_FLOAT+0.5)/1" | bc)
    echo "PID $CURR_PID: $CURR_CPU""%"
    if [[ $CURR_CPU -ge $MAX_CPU ]] ; then
        now="$(date)"
        echo "PID $CURR_PID ($PID_NAMES) went over $MAX_CPU""%" on $now
        jstack $CURR_PID > ./$now+jlog.txt
        echo "[[ $CURR_CPU""% -ge $MAX_CPU""% ]]"
        LOOP=0
        break
    fi
done
echo "Stopped"

sanjeewa@sanjeewa-ThinkPad-T530:~/work$ jps
30755 Bootstrap
8543 Jps
4892 Main

 sh test.sh <processId><CPU Limit>

sanjeewa@sanjeewa-ThinkPad-T530:~/work$ sh test.sh 30755 10
PID 30755: 0%
PID 30755: 0%
PID 30755: 0%
PID 30755: 0%
PID 30755 (30755) went over 10% on 2015 පෙබරවාරි 19 වැනි බ්‍රහස්පතින්දා 14:44:55 +0530
[[ 13% -ge 10% ]]
Stopped

• Back up the existing IDN_OAUTH2_ACCESS_TOKEN table.
• Turn off SQL_SAFE_UPDATES.
• Delete the non-active tokens other than a single record for each state for each combination of CONSUMER_KEY, AUTHZ_USER and TOKEN_SCOPE.
• Restore the original SQL_SAFE_UPDATES value.

USE `WSO2AM_DB`;
DROP PROCEDURE IF EXISTS `cleanup_tokens`;

DELIMITER $$
CREATE PROCEDURE `cleanup_tokens` ()
BEGIN

-- Backup IDN_OAUTH2_ACCESS_TOKEN table
DROP TABLE IF EXISTS `IDN_OAUTH2_ACCESS_TOKEN_BAK`;
CREATE TABLE `IDN_OAUTH2_ACCESS_TOKEN_BAK` AS SELECT * FROM `IDN_OAUTH2_ACCESS_TOKEN`;

-- 'Turn off SQL_SAFE_UPDATES'
SET @OLD_SQL_SAFE_UPDATES = @@SQL_SAFE_UPDATES;
SET SQL_SAFE_UPDATES = 0;

-- 'Keep the most recent INACTIVE key for each CONSUMER_KEY, AUTHZ_USER, TOKEN_SCOPE combination'
DELETE FROM IDN_OAUTH2_ACCESS_TOKEN WHERE TOKEN_STATE = 'INACTIVE' AND ACCESS_TOKEN NOT IN (SELECT ACCESS_TOKEN
FROM (SELECT * FROM IDN_OAUTH2_ACCESS_TOKEN WHERE TOKEN_STATE = 'INACTIVE' ORDER BY CONSUMER_KEY, AUTHZ_USER, TOKEN_SCOPE, TIME_CREATED DESC) x
GROUP BY CONSUMER_KEY, AUTHZ_USER, TOKEN_SCOPE);

-- 'Keep the most recent REVOKED key for each CONSUMER_KEY, AUTHZ_USER, TOKEN_SCOPE combination'
DELETE FROM IDN_OAUTH2_ACCESS_TOKEN WHERE TOKEN_STATE = 'REVOKED' AND ACCESS_TOKEN NOT IN (SELECT ACCESS_TOKEN
FROM (SELECT * FROM IDN_OAUTH2_ACCESS_TOKEN WHERE TOKEN_STATE = 'REVOKED' ORDER BY CONSUMER_KEY, AUTHZ_USER, TOKEN_SCOPE, TIME_CREATED DESC) x
GROUP BY CONSUMER_KEY, AUTHZ_USER, TOKEN_SCOPE);

-- 'Keep the most recent EXPIRED key for each CONSUMER_KEY, AUTHZ_USER, TOKEN_SCOPE combination'
DELETE FROM IDN_OAUTH2_ACCESS_TOKEN WHERE TOKEN_STATE = 'EXPIRED' AND ACCESS_TOKEN NOT IN (SELECT ACCESS_TOKEN
FROM (SELECT * FROM IDN_OAUTH2_ACCESS_TOKEN WHERE TOKEN_STATE = 'EXPIRED' ORDER BY CONSUMER_KEY, AUTHZ_USER, TOKEN_SCOPE, TIME_CREATED DESC) x
GROUP BY CONSUMER_KEY, AUTHZ_USER, TOKEN_SCOPE);

-- 'Restore the original SQL_SAFE_UPDATES value'
SET SQL_SAFE_UPDATES = @OLD_SQL_SAFE_UPDATES;

END$$
DELIMITER ;

USE `WSO2AM_DB`;
DROP EVENT IF EXISTS `cleanup_tokens_event`;
CREATE EVENT `cleanup_tokens_event`
    ON SCHEDULE
      EVERY 1 WEEK STARTS '2015-01-01 00:00.00'
    DO
      CALL `WSO2AM_DB`.`cleanup_tokens`();

-- 'Turn on the event_scheduler'
SET GLOBAL event_scheduler = ON;

Configuration changes.

<KeyValidatorClientType>WSClient</KeyValidatorClientType> 

<EnableThriftServer>false</EnableThriftServer> 

<EnableThriftServer>false</EnableThriftServer> 

Gateway nodes.

Key manager nodes.

<maxWait>60000</maxWait>
<initialSize>20</initialSize>
<maxActive>150</maxActive>
<maxIdle>60</maxIdle>
<minIdle>40</minIdle>
<testOnBorrow>false</testOnBorrow>

<testOnBorrow>true</testOnBorrow>
<validationQuery>SELECT 1</validationQuery>
<validationInterval>120000</validationInterval>

<Connector  protocol="org.apache.coyote.http11.Http11NioProtocol"
               port="9443"
               bindOnInit="false"
               sslProtocol="TLS"
               maxHttpHeaderSize="8192"
               acceptorThreadCount="2"
               maxThreads="750"
               minSpareThreads="250"
               disableUploadTimeout="false"
               enableLookups="false"
               connectionUploadTimeout="120000"
               maxKeepAliveRequests="400"
               acceptCount="400"
               server="WSO2 Carbon Server"
               clientAuth="false"
               compression="off"
               scheme="https"
               secure="true"
               SSLEnabled="true"
               compressionMinSize="2048"
               noCompressionUserAgents="gozilla, traviata"                
compressableMimeType="text/html,text/javascript,application/x-javascript,application/javascript,application/xml,text
/css,application/xslt+xml,text/xsl,image/gif,image/jpg,image/jpeg"
               URIEncoding="UTF-8"/>

Testing

Test 02 - Single gateway/key manager test

Results table.

Response Time Graph

For 300 concurrency 1 node
Average TPS -4956
Average response time - 55ms
Average delay added by gateway - 49ms

package org.wso2.carbon.test.gateway;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.synapse.Mediator;
import org.apache.synapse.MessageContext;
import org.apache.synapse.core.axis2.Axis2MessageContext;
import org.apache.synapse.rest.AbstractHandler;
import org.apache.synapse.rest.RESTConstants;
import org.wso2.carbon.apimgt.impl.APIConstants;
import org.wso2.carbon.context.PrivilegedCarbonContext;
import org.wso2.carbon.utils.multitenancy.MultitenantConstants;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;

public class TestHandler extends AbstractHandler {

    private static final String EXT_SEQUENCE_PREFIX = "WSO2AM--Ext--";
    private static final String DIRECTION_OUT = "Out";
    private static final Log log = LogFactory.getLog(TestHandler.class);

    public boolean mediate(MessageContext messageContext, String direction) {
        log.info("===============================================================================");
        return true;
    }


    public boolean handleRequest(MessageContext messageContext) {
        log.info("===============================================================================");
        return true;
    }

    public boolean handleResponse(MessageContext messageContext) {
        return mediate(messageContext, DIRECTION_OUT);
    }
}

package test.com.apim.workflow;

import java.util.List;
import java.util.Map;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.wso2.carbon.apimgt.api.APIManagementException;
import org.wso2.carbon.apimgt.impl.APIConstants;
import org.wso2.carbon.apimgt.impl.dao.ApiMgtDAO;
import org.wso2.carbon.apimgt.impl.dto.ApplicationWorkflowDTO;
import org.wso2.carbon.apimgt.impl.dto.WorkflowDTO;
import org.wso2.carbon.apimgt.impl.workflow.WorkflowException;
import org.wso2.carbon.apimgt.impl.workflow.WorkflowExecutor;
import org.wso2.carbon.apimgt.impl.workflow.WorkflowConstants;
import org.wso2.carbon.apimgt.impl.workflow.WorkflowStatus;

import org.wso2.carbon.apimgt.impl.dto.ApplicationRegistrationWorkflowDTO;
import org.wso2.carbon.apimgt.impl.APIManagerFactory;
import org.wso2.carbon.apimgt.api.APIConsumer;

public class ApplicationCreationExecutor extends WorkflowExecutor {

    private static final Log log =
            LogFactory.getLog(ApplicationCreationExecutor.class);

    private String userName;
    private String appName;

    @Override
    public String getWorkflowType() {
        return WorkflowConstants.WF_TYPE_AM_APPLICATION_CREATION;
    }

    /**
     * Execute the workflow executor
     *
     * @param workFlowDTO
     *            - {@link ApplicationWorkflowDTO}
     * @throws WorkflowException
     */

    public void execute(WorkflowDTO workFlowDTO) throws WorkflowException {
        if (log.isDebugEnabled()) {
            log.info("Executing Application creation Workflow..");
        }
        workFlowDTO.setStatus(WorkflowStatus.APPROVED);
        complete(workFlowDTO);

    }

    /**
     * Complete the external process status
     * Based on the workflow status we will update the status column of the
     * Application table
     *
     * @param workFlowDTO - WorkflowDTO
     */
    public void complete(WorkflowDTO workFlowDTO) throws WorkflowException {
        if (log.isDebugEnabled()) {
            log.info("Complete  Application creation Workflow..");
        }

        String status = null;
        if ("CREATED".equals(workFlowDTO.getStatus().toString())) {
            status = APIConstants.ApplicationStatus.APPLICATION_CREATED;
        } else if ("REJECTED".equals(workFlowDTO.getStatus().toString())) {
            status = APIConstants.ApplicationStatus.APPLICATION_REJECTED;
        } else if ("APPROVED".equals(workFlowDTO.getStatus().toString())) {
            status = APIConstants.ApplicationStatus.APPLICATION_APPROVED;
        }

        ApiMgtDAO dao = new ApiMgtDAO();

        try {
            dao.updateApplicationStatus(Integer.parseInt(workFlowDTO.getWorkflowReference()),status);
        } catch (APIManagementException e) {
            String msg = "Error occured when updating the status of the Application creation process";
            log.error(msg, e);
            throw new WorkflowException(msg, e);
        }

        ApplicationWorkflowDTO appDTO = (ApplicationWorkflowDTO) workFlowDTO;
        userName = appDTO.getUserName();

        appName = appDTO.getApplication().getName();

        System.out.println("UseName : " + userName + "   --- appName = " + appName) ;

        Map mapConsumerKeySecret = null ;

        try {
            APIConsumer apiConsumer = APIManagerFactory.getInstance().getAPIConsumer(userName);
            String[] appliedDomains = {""};
//Key generation
            mapConsumerKeySecret = apiConsumer.requestApprovalForApplicationRegistration(userName, appName, "PRODUCTION", "", appliedDomains, "3600");
        } catch(APIManagementException e) {
            throw new WorkflowException(
                    "An error occurred while generating token.", e);
        }

        for (Map.Entry entry : mapConsumerKeySecret.entrySet()) {
            String key = entry.getKey();
            String value = entry.getValue();

            System.out.println("Key : " + key + "   ---  value = " + value);
        }
    }

    @Override
    public List getWorkflowDetails(String workflowStatus) throws WorkflowException {
        return null;
    }

}

(function($) {
       function rating(rate, message) {
        return {
            rate: rate,
            messageKey: message
        };
    }
    function uncapitalize(str) {
        return str.substring(0, 1).toLowerCase() + str.substring(1);
    }
    $.validator.passwordRating = function(password, username) {
            var minLength = 6;
            var passwordStrength   = 0;
            if ((password.length >0) && (password.length <=5)) passwordStrength=0;
            if (password.length >= minLength) passwordStrength++;
            if ((password.match(/[a-z]/)) && (password.match(/[A-Z]/))) passwordStrength++;
            if (password.match(/\d+/)) passwordStrength++;
            if (password.match(/.[!,@,#,$,%,^,&,*]/)) passwordStrength++;
            if (password.length > 12) passwordStrength++;
            if (username && password.toLowerCase()== username.toLowerCase()){
                passwordStrength = 0;
            }
            $('#pwdMeter').removeClass();
            $('#pwdMeter').addClass('neutral');
            switch(passwordStrength){
            case 1:
                return rating(1, "very-weak");
              break;
            case 2:
                return rating(2, "weak");
              break;
            case 3:
                return rating(3, "weak");
              break;
            case 4:
                return rating(4, "medium");
              break;
            case 5:
                 return rating(5, "strong");
              break;
            case 6:
                 return rating(5, "vstrong");
              break;
            default:
                return rating(1, "very-weak");
            }
    }
    $.validator.passwordRating.messages = {
        "similar-to-username": "Too similar to username",
        "very-weak": "Very weak",
        "weak": "Weak",
        "medium": "Medium",
        "strong": "Strong",
        "vstrong": "Very Strong"
    }
    $.validator.addMethod("password", function(value, element, usernameField) {
        // use untrimmed value
        var password = element.value,
        // get username for comparison, if specified
            username = $(typeof usernameField != "boolean" ? usernameField : []);        
        var rating = $.validator.passwordRating(password, username.val());
        // update message for this field
        var meter = $(".password-meter", element.form);
        meter.find(".password-meter-bar").removeClass().addClass("password-meter-bar").addClass("password-meter-" + rating.messageKey);
        meter.find(".password-meter-message")
        .removeClass()
        .addClass("password-meter-message")
        .addClass("password-meter-message-" + rating.messageKey)
        .text($.validator.passwordRating.messages[rating.messageKey]);
        // display process bar instead of error message
        return rating.rate > 3;
    }, "Minimum system requirements not met");
    // manually add class rule, to make username param optional
    $.validator.classRuleSettings.password = { password: true };
   
})(jQuery);

<div class="help-block" id="password-help" style="display:none">
 <%=i18n.localize("pwHelpMsgLine1")%>
 <ul>
 <li>This is new changed TIP :-) </li>                
<li><%=i18n.localize("pwHelpMsgLine3")%></li> 
<li><%=i18n.localize("pwHelpMsgLine4")%></li>
<li><%=i18n.localize("pwHelpMsgLine5")%></li>
</ul>
</div>

    "theme" : {
        "base" : "fancy",
        "subtheme" : "custom-signup"
    },

<?xmlversion="1.0"encoding="UTF-8"?>
<sequencexmlns="http://ws.apache.org/ns/synapse"name="_throttle_out_handler_">
<propertyname="X-JWT-Assertion"scope="transport"action="remove"/>
   <sequencekey="convert"/>
   <drop/>
</sequence>

  <sequencename="convert">
      <sequencekey="_build_"/>
            <payloadFactorymedia-type="xml">
               <format>
                  <am:faultxmlns:am="http://wso2.org/apimanager">
                     <am:code>$1</am:code>
                     <am:type>Status report</am:type>
                     <am:message>Runtime Error</am:message>
                     <am:description>$2</am:description>
                  </am:fault>
               </format>
               <args>
                  <argevaluator="xml"expression="$ctx:ERROR_CODE"/>
                  <argevaluator="xml"expression="$ctx:ERROR_MESSAGE"/>
               </args>
            </payloadFactory>
      <propertyname="RESPONSE"value="true"/>
      <headername="To"action="remove"/>
      <propertyname="HTTP_SC"value="555"scope="axis2"/>
      <propertyname="NO_ENTITY_BODY"scope="axis2"action="remove"/>
      <propertyname="ContentType"scope="axis2"action="remove"/>
      <propertyname="Authorization"scope="transport"action="remove"/>
      <propertyname="Host"scope="transport"action="remove"/>
      <propertyname="Accept"scope="transport"action="remove"/>
      <propertyname="X-JWT-Assertion"scope="transport"action="remove"/>
      <propertyname="messageType"value="application/json"scope="axis2"/>
      <send/>
   </sequence>

package demo.jaxrs.server;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import org.apache.cxf.message.Message;
import org.apache.cxf.phase.AbstractPhaseInterceptor;
import org.apache.cxf.phase.Phase;

public class CustomOutInterceptor extends AbstractPhaseInterceptor {

    public CustomOutInterceptor() {
        //We will use PRE_INVOKE phase as we need to process message before hit actual service
        super(Phase.PRE_INVOKE );
    }

    public void handleMessage(Message outMessage) {
        System.out.println("Token: "+ ((TreeMap) outMessage.get(Message.PROTOCOL_HEADERS)).get("Authorization"));
  // Do your processing with Authorization transport header.
    }
}

<beansxmlns="http://www.springframework.org/schema/beans"
       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
       xmlns:jaxrs="http://cxf.apache.org/jaxrs"
       xsi:schemaLocation="
         http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
         http://cxf.apache.org/jaxrs http://cxf.apache.org/schemas/jaxrs.xsd">
    <jaxrs:serverid="APIService"address="/">
        <jaxrs:serviceBeans>
            <refbean="serviceBean"/>
        </jaxrs:serviceBeans>

        <jaxrs:inInterceptors>
            <refbean="testInterceptor"/>
        </jaxrs:inInterceptors>
    </jaxrs:server>
    <beanid="testInterceptor"class="demo.jaxrs.server.CustomOutInterceptor" />
    <beanid="serviceBean"class="demo.jaxrs.server.APIService"/>
</beans>

curl -k -v -H "Authorization: Bearer d5701a8ed6f677f215fa4d65c05e361" http://127.0.0.1:9763/APIManager/qqqq-1.0.0-admin/

Token: [Bearer d5701a8ed6f677f215fa4d65c05e361]
API Service -- invoking getAPI, API id is: qqqq-1.0.0-admin

        $('#visibility').change(function(){
            var visibility = $('#visibility').find(":selected").val();
            if (visibility == "public" || visibility == "private" || visibility == "controlled"){
                $('#rolesDiv').hide();
            } else{
                $('#rolesDiv').show();
            }
        });

        $('#visibility').change(function(){
            var visibility = $('#visibility').find(":selected").val();

            if (visibility != "public"){
            jagg.message({content:"You have changed visibility of API, so based on visibility subscription availability may change",type:"warn"});
            }

            if (visibility == "public" || visibility == "private" || visibility == "controlled"){
                $('#rolesDiv').hide();
            } else{
                $('#rolesDiv').show();
            }
        });

        "theme" : {
               "base" : "fancy",
                "subtheme" : "new-theme"
        }