In model define auth and janrain

# normal auth
auth = Auth(globals(),db)
auth.define_tables()
...
...

#janrain
from gluon.contrib.login_methods.rpx_account import RPXAccount
from gluon.contrib.login_methods.extended_login_form import ExtendedLoginForm
# The line below should be removed or commented
# auth.settings.actions_disabled=['register','change_password','request_reset_password']

# define where to go after RPX login
if request.vars._next:
    url = "http://yourappdomain/init/default/user/login?_next=%s" % request.vars._next
else:
    url = "http://yourappdomain/init/default/user/login"

rpxform = RPXAccount(request, 
    api_key='<your_api_key>',
    domain='yourdomain',
    url = url,
    language="yourlanguage",
    embed=True
    )

Now in controller, note you have two objects 'rpx' and 'auth'

def user():
    rpx = ''
    registerurl=URL('default','user',args='register',vars=dict(_next='/init/default/somepage'))
    if request.vars.token:
        auth.settings.login_form = rpxform
        return dict(form=auth())
    if 'login' in request.args:
        rpx = rpxform.login_form()
        html = DIV(H1('Login'),
               rpx,BR(),BR(),
               H1(A('Click here to register',
                       _href=registerurl),BR(),
                       'Or sign-in using your email and password'),
               auth(),
              )
    else:
        html = auth()

    return dict(form=html)

Then in view

{{extend 'layout.html'}}
{{title=dict(
         login='',
         register=H1('Register'),
         logout=H1('Logged out'),
         profile=H1('Change your profile'),
         change_password=H1('Change your password'),
         request_reset_password=H1('Reset your password'),
         reset_password=H1('Define a new password'),
         verify_email=H1('Verification mail'),
         retrieve_username=H1('Request username'),
         not_authorized=H1('Denied!')
         )}}
{{=title[request.args(0)]}}
{{=form}}
{{if request.args(0)=='login':}}
{{if not 'request_reset_password' in auth.settings.actions_disabled:}}
<br/><a href="{{=URL(r=request,args='request_reset_password')}}">Remember Password</a>
{{pass}}
{{pass}}