The widget

Place the following code into a model file (models/A_widget.py)

# coding: utf8
from gluon.sqlhtml import *

class SELECT_OR_ADD_OPTION(object):  #and even EDIT

    def __init__(self, referenced_table, controller="default", function="referenced_data", dialog_width=450):
        self.referenced_table = referenced_table
        self.controller = controller
        self.function = function
        self.dialog_width = dialog_width

    def widget(self, field, value):
        #generate the standard widget for this field
        select_widget = OptionsWidget.widget(field, value)

        #get the widget's id (need to know later on so can tell receiving controller what to update)
        my_select_id = select_widget.attributes.get('_id', None) 
        wrapper = DIV(_id=my_select_id+"__reference-actions__wrapper") 
        wrapper.components.extend([select_widget, ])
        style_icons = {'new':"icon plus icon-plus", 'edit': "icon pen icon-pencil" }
        actions = ['new']
        if value: actions.append('edit')  # if we already have selected value
        for action in actions:
            extra_args = [my_select_id, action, self.referenced_table]
            if action == 'edit':
                extra_args.append(value)
            #create a div that will load the specified controller via ajax
            form_loader_div = DIV(LOAD(c=self.controller, f=self.function, args=extra_args, ajax=True), _id=my_select_id+"_"+action+"_dialog-form", _title=action+": "+self.referenced_table)
            #generate the "add/edit" button that will appear next the options widget and open our dialog
            action_button = A([SPAN(_class=style_icons[action]), SPAN( _class="buttontext button") ], 
                              _title=T(action), _id=my_select_id+"_option_%s_trigger"%action, _class="button btn", _style="vertical-align:top" )
    
            #create javascript for creating and opening the dialog
            js = '$( "#%s_%s_dialog-form" ).dialog({autoOpen: false, not__modal:true, show: "blind", hide: "explode", width: %s});' % (my_select_id, action,  self.dialog_width)
            js += '$( "#%s_option_%s_trigger" ).click(function() { $( "#%s_%s_dialog-form" ).dialog( "open" );return false;}); ' % (my_select_id, action, my_select_id, action, )
            js += '$(function() { $( "#%s_option_%s_trigger" ).button({text: true, icons: { primary: "ui-icon-circle-plus"} }); });' % (my_select_id, action, )
            if action=='edit':
                # hide if reference changed - as load is constructed for initial value only (or would need some lazy loading mechanizm)
                js += '$(function() {$("#%s").change(function() {    $( "#%s_option_%s_trigger" ).hide(); } ) });' % (my_select_id,  my_select_id, 'edit', )
            jq_script=SCRIPT(js, _type="text/javascript")
    
            wrapper.components.extend([form_loader_div, action_button, jq_script])
        return wrapper

DB Model (assign widgets)

db.define_table('category',
    Field('name', 'string', notnull=True, unique=True),
    Field('description', 'text'),
    format="%(name)s"
)

db.define_table('product',
    Field('category_id', db.category, widget=SELECT_OR_ADD_OPTION("category").widget), # can override defaults:  controller="default", function="referenced_data",
    Field('name', 'string', notnull=True),
    Field('description', 'text'),
    Field('price', 'decimal(10,2)', notnull=True)
    )

Controller functions

def create_product(): # can be any function which displays SQLFORM
    form = crud.create(db.product)
    return response.render('generic.html', context=form)

def update_product():  # for edit functionality (again can be any edti form generating function)
    form = crud.update(db.product, int(request.args[0]))
    return response.render('generic.html', context=form)

    #you need jqueryUI for the widget to work, include in calling controllers or comment out in  layout.html
    response.files.append("//ajax.googleapis.com/ajax/libs/jqueryui/1.8.9/jquery-ui.js")
    response.files.append("//ajax.googleapis.com/ajax/libs/jqueryui/1.8.9/themes/smoothness/jquery-ui.css")

############ important function - it should be indicated in widget constructor #######

def referenced_data(): 
    """ shows dialog with reference add/edit form
    the idea is taken from "data" function, just first argument is the id of calling select box 
    """

    try:    references_options_list_id = request.args[0]
    except: return T("ERR: references_options_list_id lacking")
    
    try:    action = request.args[1]
    except: return  T("ERR: action lacking")
    
    try:    referenced_table= request.args[2]
    except: return T("ERR: referenced_table lacking")

    if action=="edit":
        try: referenced_record_id = int( request.args[3] )
        except: response.flash = T("ERR: referenced_record_id lacking"); return (response.flash)
        form = SQLFORM(db[referenced_table], referenced_record_id) # edit/update/change
    else:
        form = SQLFORM(db[referenced_table]) # new/create/add

    if form.accepts(request.vars):
        #Then let the user know adding via our widget worked
        response.flash = "done: %s %s" %( T(action),  referenced_table) # added / edited
        #close the widget's dialog box
        response.js = '$( "#%s_%s_dialog-form" ).dialog( "close" ); ' %(references_options_list_id, action)

        def format_referenced(id): 
            #return format(db[referenced_table], id)  #should get from table
            table = db[referenced_table]
            if isinstance(table._format, str):     return table._format % table[id]
            elif callable(table._format):          return table._format(table[id])        
            else: return "???"
            
        if action=='new':
            #update the options they can select their new category in the main form
            response.js += """$("#%s").append("<option value='%s'>%s</option>");""" % (references_options_list_id, form.vars.id, format_referenced(form.vars.id))
            #and select the one they just added
            response.js += """$("#%s").val("%s");""" % (references_options_list_id, form.vars.id)
        if action=='edit':
            #response.js += """alert( $('#%s option[value="%s"]').html());""" % (references_options_list_id, form.vars.id) # format_referenced(form.vars.id) )
            response.js += """$('#%s option[value="%s"]').html('%s')""" % (references_options_list_id, form.vars.id, format_referenced(form.vars.id) )
        
    return BEAUTIFY(form)

In Action