For this to work you will need the Jquery form plugin http://jquery.malsup.com/form/ At the top of web2py_ajax.html add

...
response.files.insert(3,URL(r=request,c='static/js',f='jquery.form.js'))
...

at the bottom of web2py_ajax.html replace with the following code

...
function web2py_trap_form(action,target) {
   jQuery('#'+target+' form').each(function(i){
      var form=jQuery(this);
      if(!form.hasClass('no_trap')){
         if(form.find('.upload').length>0){
            //using ajaxForm has the disadvantage that the header is not returned in xhr
            //can this be fixed in the ajaxForm plugin???
             form.ajaxForm({ 
                url: action,
                success: function(data, statusText, xhr) {
                    complete_web2py_ajax_page(xhr, data, action, target)
                }
             });
         }else{
            form.submit(function(obj){
             jQuery('.flash').hide().html('');
             web2py_ajax_page('post',action,form.serialize(),target);
             return false;
            });
         }
      }
   });
}
function complete_web2py_ajax_page (xhr, text, action, target){
      var html=xhr.responseText;
      var content=xhr.getResponseHeader('web2py-component-content'); 
      var command=xhr.getResponseHeader('web2py-component-command');
      var flash=xhr.getResponseHeader('web2py-component-flash');
      var t = jQuery('#'+target);
      if(content=='prepend') t.prepend(html); 
      else if(content=='append') t.append(html);
      else if(content!='hide') t.html(html);  
      web2py_trap_form(action,target);
      web2py_ajax_init();      
      if(command) eval(command);
      if(flash) jQuery('.flash').html(flash).slideDown();
}
function web2py_ajax_page(method,action,data,target) {
  jQuery.ajax({'type':method,'url':action,'data':data,
    'beforeSend':function(xhr) {
      xhr.setRequestHeader('web2py-component-location',document.location);
      xhr.setRequestHeader('web2py-component-element',target);},
    'complete':function(xhr,text){
        complete_web2py_ajax_page(xhr, text, action, target);
      }
    });
}
function web2py_component(action,target) {
    //jQuery(document).ready(function(){ //this caused some trouble for me
        $('#'+target).prepend('{{=IMG(_alt="loading ...", _src=URL(request.application, 'static/images', 'loading.gif'))}}');// just some eyecandy, you can remove it if you want
        web2py_ajax_page('get',action,null,target); 
      //  });
}

This will use the jquery form plugin when you LOAD a component that contains a form with an upload field. Only if this condition if fulfilled something changes, in all other cases everything will stay as it was before.

Here is an example that will not work with a default web2py installation

models/db.py

...
db.define_table('filetable',
        Field('somefile', 'upload', requires = IS_NOT_EMPTY()),
        )
...

controllers/default.py

def index():
    return dict(message = TAG[''](H1('upload from'), LOAD('default', 'theform', ajax=True)))

def theform():
   def on_accecpt(form):
          response.flash = 'Success :), this message will not be send :('
   return crud.create('filetable', onaccept=on_accept)

The drawback is that if the form plugin is used you will loose header information like "web2py-component-command" or "web2py-component-flash". You will have to attach these to your returned page instead. However uploads will be submitted properly.

This workaround might some day be unnecessary: http://hublog.hubmed.org/archives/001941.html