from __future__ import with_statement
__license__   = 'GPL v3'
__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'

# Imports {{{
import os, traceback, Queue, time, cStringIO, re, sys
from threading import Thread

from PyQt4.Qt import QMenu, QAction, QActionGroup, QIcon, SIGNAL, \
                     Qt, pyqtSignal, QDialog

from calibre.customize.ui import available_input_formats, available_output_formats, \
    device_plugins
from calibre.devices.interface import DevicePlugin
from calibre.devices.errors import UserFeedback, OpenFeedback
from calibre.gui2.dialogs.choose_format_device import ChooseFormatDeviceDialog
from calibre.utils.ipc.job import BaseJob
from calibre.devices.scanner import DeviceScanner
from calibre.gui2 import config, error_dialog, Dispatcher, dynamic, \
        warning_dialog, info_dialog, choose_dir
from calibre.ebooks.metadata import authors_to_string
from calibre import preferred_encoding, prints, force_unicode, as_unicode
from calibre.utils.filenames import ascii_filename
from calibre.devices.errors import FreeSpaceError
from calibre.devices.apple.driver import ITUNES_ASYNC
from calibre.devices.folder_device.driver import FOLDER_DEVICE
from calibre.devices.bambook.driver import BAMBOOK, BAMBOOKWifi
from calibre.ebooks.metadata.meta import set_metadata
from calibre.constants import DEBUG
from calibre.utils.config import prefs, tweaks
from calibre.utils.magick.draw import thumbnail
from calibre.library.save_to_disk import plugboard_any_device_value, \
                                         plugboard_any_format_value
# }}}

class DeviceJob(BaseJob): # {{{

    def __init__(self, func, done, job_manager, args=[], kwargs={},
            description=''):
        BaseJob.__init__(self, description, done=done)
        self.func = func
        self.args, self.kwargs = args, kwargs
        self.exception = None
        self.job_manager = job_manager
        self._details = _('No details available.')
        self._aborted = False

    def start_work(self):
        self.start_time = time.time()
        self.job_manager.changed_queue.put(self)

    def job_done(self):
        self.duration = time.time() - self.start_time
        self.percent = 1
        self.job_manager.changed_queue.put(self)

    def report_progress(self, percent, msg=''):
        self.notifications.put((percent, msg))
        self.job_manager.changed_queue.put(self)

    def run(self):
        self.start_work()
        try:
            self.result = self.func(*self.args, **self.kwargs)
            if self._aborted:
                return
        except (Exception, SystemExit), err:
            if self._aborted:
                return
            self.failed = True
            ex = as_unicode(err)
            self._details = ex + '\n\n' + \
                traceback.format_exc()
            self.exception = err
        finally:
            self.job_done()

    def abort(self, err):
        call_job_done = False
        if self.run_state == self.WAITING:
            self.start_work()
            call_job_done = True
        self._aborted = True
        self.failed = True
        self._details = unicode(err)
        self.exception = err
        if call_job_done:
            self.job_done()

    @property
    def log_file(self):
        return cStringIO.StringIO(self._details.encode('utf-8'))

    # }}}

def find_plugboard(device_name, format, plugboards):
    cpb = None
    if format in plugboards:
        cpb = plugboards[format]
    elif plugboard_any_format_value in plugboards:
        cpb = plugboards[plugboard_any_format_value]
    if cpb is not None:
        if device_name in cpb:
            cpb = cpb[device_name]
        elif plugboard_any_device_value in cpb:
            cpb = cpb[plugboard_any_device_value]
        else:
            cpb = None
    if DEBUG:
        prints('Device using plugboard', format, device_name, cpb)
    return cpb

def device_name_for_plugboards(device_class):
    if hasattr(device_class, 'DEVICE_PLUGBOARD_NAME'):
        return device_class.DEVICE_PLUGBOARD_NAME
    return device_class.__class__.__name__

class DeviceManager(Thread): # {{{

    def __init__(self, connected_slot, job_manager, open_feedback_slot,
            open_feedback_msg, sleep_time=2):
        '''
        :sleep_time: Time to sleep between device probes in secs
        '''
        Thread.__init__(self)
        self.setDaemon(True)
        # [Device driver, Showing in GUI, Ejected]
        self.devices        = list(device_plugins())
        self.sleep_time     = sleep_time
        self.connected_slot = connected_slot
        self.jobs           = Queue.Queue(0)
        self.keep_going     = True
        self.job_manager    = job_manager
        self.reported_errors = set([])
        self.current_job    = None
        self.scanner        = DeviceScanner()
        self.connected_device = None
        self.connected_device_kind = None
        self.ejected_devices  = set([])
        self.mount_connection_requests = Queue.Queue(0)
        self.open_feedback_slot = open_feedback_slot
        self.open_feedback_msg = open_feedback_msg

    def report_progress(self, *args):
        pass

    @property
    def is_device_connected(self):
        return self.connected_device is not None

    @property
    def device(self):
        return self.connected_device

    def do_connect(self, connected_devices, device_kind):
        for dev, detected_device in connected_devices:
            if dev.OPEN_FEEDBACK_MESSAGE is not None:
                self.open_feedback_slot(dev.OPEN_FEEDBACK_MESSAGE)
            try:
                dev.reset(detected_device=detected_device,
                    report_progress=self.report_progress)
                dev.open()
            except OpenFeedback, e:
                if dev not in self.ejected_devices:
                    self.open_feedback_msg(dev.get_gui_name(), e.feedback_msg)
                    self.ejected_devices.add(dev)
                continue
            except:
                tb = traceback.format_exc()
                if DEBUG or tb not in self.reported_errors:
                    self.reported_errors.add(tb)
                    prints('Unable to open device', str(dev))
                    prints(tb)
                continue
            self.connected_device = dev
            self.connected_device_kind = device_kind
            self.connected_slot(True, device_kind)
            return True
        return False

    def connected_device_removed(self):
        while True:
            try:
                job = self.jobs.get_nowait()
                job.abort(Exception(_('Device no longer connected.')))
            except Queue.Empty:
                break
        try:
            self.connected_device.post_yank_cleanup()
        except:
            pass
        if self.connected_device in self.ejected_devices:
            self.ejected_devices.remove(self.connected_device)
        else:
            self.connected_slot(False, self.connected_device_kind)
        self.connected_device = None

    def detect_device(self):
        self.scanner.scan()
        if self.is_device_connected:
            connected, detected_device = \
                self.scanner.is_device_connected(self.connected_device,
                        only_presence=True)
            if not connected:
                self.connected_device_removed()
        else:
            possibly_connected_devices = []
            for device in self.devices:
                if device in self.ejected_devices:
                    continue
                possibly_connected, detected_device = \
                        self.scanner.is_device_connected(device)
                if possibly_connected:
                    possibly_connected_devices.append((device, detected_device))
            if possibly_connected_devices:
                if not self.do_connect(possibly_connected_devices,
                                       device_kind='device'):
                    if DEBUG:
                        prints('Connect to device failed, retrying in 5 seconds...')
                    time.sleep(5)
                    if not self.do_connect(possibly_connected_devices,
                                       device_kind='usb'):
                        if DEBUG:
                            prints('Device connect failed again, giving up')

    # Mount devices that don't use USB, such as the folder device and iTunes
    # This will be called on the GUI thread. Because of this, we must store
    # information that the scanner thread will use to do the real work.
    def mount_device(self, kls, kind, path):
        self.mount_connection_requests.put((kls, kind, path))

    # disconnect a device
    def umount_device(self, *args):
        if self.is_device_connected and not self.job_manager.has_device_jobs():
            if self.connected_device_kind == 'device':
                self.connected_device.eject()
                self.ejected_devices.add(self.connected_device)
                self.connected_slot(False, self.connected_device_kind)
            elif hasattr(self.connected_device, 'unmount_device'):
                # As we are on the wrong thread, this call must *not* do
                # anything besides set a flag that the right thread will see.
                self.connected_device.unmount_device()

    def next(self):
        if not self.jobs.empty():
            try:
                return self.jobs.get_nowait()
            except Queue.Empty:
                pass

    def run(self):
        while self.keep_going:
            kls = None
            while True:
                try:
                    (kls,device_kind, folder_path) = \
                                self.mount_connection_requests.get_nowait()
                except Queue.Empty:
                    break
            if kls is not None:
                try:
                    dev = kls(folder_path)
                    self.do_connect([[dev, None],], device_kind=device_kind)
                except:
                    prints('Unable to open %s as device (%s)'%(device_kind, folder_path))
                    traceback.print_exc()
            else:
                self.detect_device()
            while True:
                job = self.next()
                if job is not None:
                    self.current_job = job
                    self.device.set_progress_reporter(job.report_progress)
                    self.current_job.run()
                    self.current_job = None
                else:
                    break
            time.sleep(self.sleep_time)

    def create_job(self, func, done, description, args=[], kwargs={}):
        job = DeviceJob(func, done, self.job_manager,
                        args=args, kwargs=kwargs, description=description)
        self.job_manager.add_job(job)
        self.jobs.put(job)
        return job

    def has_card(self):
        try:
            return bool(self.device.card_prefix())
        except:
            return False

    def _get_device_information(self):
        info = self.device.get_device_information(end_session=False)
        info = [i.replace('\x00', '').replace('\x01', '') for i in info]
        cp = self.device.card_prefix(end_session=False)
        fs = self.device.free_space()
        return info, cp, fs

    def get_device_information(self, done):
        '''Get device information and free space on device'''
        return self.create_job(self._get_device_information, done,
                    description=_('Get device information'))

    def _books(self):
        '''Get metadata from device'''
        mainlist = self.device.books(oncard=None, end_session=False)
        cardalist = self.device.books(oncard='carda')
        cardblist = self.device.books(oncard='cardb')
        return (mainlist, cardalist, cardblist)

    def books(self, done):
        '''Return callable that returns the list of books on device as two booklists'''
        return self.create_job(self._books, done, description=_('Get list of books on device'))

    def _annotations(self, path_map):
        return self.device.get_annotations(path_map)

    def annotations(self, done, path_map):
        '''Return mapping of ids to annotations. Each annotation is of the
        form (type, location_info, content). path_map is a mapping of
        ids to paths on the device.'''
        return self.create_job(self._annotations, done, args=[path_map],
                description=_('Get annotations from device'))

    def _sync_booklists(self, booklists):
        '''Sync metadata to device'''
        self.device.sync_booklists(booklists, end_session=False)
        return self.device.card_prefix(end_session=False), self.device.free_space()

    def sync_booklists(self, done, booklists, plugboards):
        if hasattr(self.connected_device, 'set_plugboards') and \
                callable(self.connected_device.set_plugboards):
            self.connected_device.set_plugboards(plugboards, find_plugboard)
        return self.create_job(self._sync_booklists, done, args=[booklists],
                        description=_('Send metadata to device'))

    def upload_collections(self, done, booklist, on_card):
        return self.create_job(booklist.rebuild_collections, done,
                               args=[booklist, on_card],
                        description=_('Send collections to device'))

    def _upload_books(self, files, names, on_card=None, metadata=None, plugboards=None):
        '''Upload books to device: '''
        if hasattr(self.connected_device, 'set_plugboards') and \
                callable(self.connected_device.set_plugboards):
            self.connected_device.set_plugboards(plugboards, find_plugboard)
        if metadata and files and len(metadata) == len(files):
            for f, mi in zip(files, metadata):
                if isinstance(f, unicode):
                    ext = f.rpartition('.')[-1].lower()
                    cpb = find_plugboard(
                            device_name_for_plugboards(self.connected_device),
                            ext, plugboards)
                    if ext:
                        try:
                            if DEBUG:
                                prints('Setting metadata in:', mi.title, 'at:',
                                        f, file=sys.__stdout__)
                            with open(f, 'r+b') as stream:
                                if cpb:
                                    newmi = mi.deepcopy_metadata()
                                    newmi.template_to_attribute(mi, cpb)
                                else:
                                    newmi = mi
                                set_metadata(stream, newmi, stream_type=ext)
                        except:
                            if DEBUG:
                                prints(traceback.format_exc(), file=sys.__stdout__)

        return self.device.upload_books(files, names, on_card,
                                        metadata=metadata, end_session=False)

    def upload_books(self, done, files, names, on_card=None, titles=None,
                     metadata=None, plugboards=None):
        desc = _('Upload %d books to device')%len(names)
        if titles:
            desc += u':' + u', '.join(titles)
        return self.create_job(self._upload_books, done, args=[files, names],
                kwargs={'on_card':on_card,'metadata':metadata,'plugboards':plugboards}, description=desc)

    def add_books_to_metadata(self, locations, metadata, booklists):
        self.device.add_books_to_metadata(locations, metadata, booklists)

    def _delete_books(self, paths):
        '''Remove books from device'''
        self.device.delete_books(paths, end_session=True)

    def delete_books(self, done, paths):
        return self.create_job(self._delete_books, done, args=[paths],
                        description=_('Delete books from device'))

    def remove_books_from_metadata(self, paths, booklists):
        self.device.remove_books_from_metadata(paths, booklists)

    def _save_books(self, paths, target):
        '''Copy books from device to disk'''
        for path in paths:
            name = path.rpartition(os.sep)[2]
            dest = os.path.join(target, name)
            if os.path.abspath(dest) != os.path.abspath(path):
                f = open(dest, 'wb')
                self.device.get_file(path, f)
                f.close()

    def save_books(self, done, paths, target):
        return self.create_job(self._save_books, done, args=[paths, target],
                        description=_('Download books from device'))

    def _view_book(self, path, target):
        f = open(target, 'wb')
        self.device.get_file(path, f)
        f.close()
        return target

    def view_book(self, done, path, target):
        return self.create_job(self._view_book, done, args=[path, target],
                        description=_('View book on device'))

    # }}}

class DeviceAction(QAction): # {{{

    a_s = pyqtSignal(object)

    def __init__(self, dest, delete, specific, icon_path, text, parent=None):
        QAction.__init__(self, QIcon(icon_path), text, parent)
        self.dest = dest
        self.delete = delete
        self.specific = specific
        self.triggered.connect(self.emit_triggered)

    def emit_triggered(self, *args):
        self.a_s.emit(self)

    def __repr__(self):
        return self.__class__.__name__ + ':%s:%s:%s'%(self.dest, self.delete,
                self.specific)
    # }}}

class DeviceMenu(QMenu): # {{{

    fetch_annotations = pyqtSignal()
    disconnect_mounted_device = pyqtSignal()

    def __init__(self, parent=None):
        QMenu.__init__(self, parent)
        self.group = QActionGroup(self)
        self.actions = []
        self._memory = []

        self.set_default_menu = QMenu(_('Set default send to device action'))
        self.set_default_menu.setIcon(QIcon(I('config.png')))


        basic_actions = [
                ('main:', False, False,  I('reader.png'),
                    _('Send to main memory')),
                ('carda:0', False, False, I('sd.png'),
                    _('Send to storage card A')),
                ('cardb:0', False, False, I('sd.png'),
                    _('Send to storage card B')),
        ]

        delete_actions = [
                ('main:', True, False,   I('reader.png'),
                    _('Main Memory')),
                ('carda:0', True, False,  I('sd.png'),
                    _('Storage Card A')),
                ('cardb:0', True, False,  I('sd.png'),
                    _('Storage Card B')),
        ]

        specific_actions = [
                ('main:', False, True,  I('reader.png'),
                    _('Main Memory')),
                ('carda:0', False, True, I('sd.png'),
                    _('Storage Card A')),
                ('cardb:0', False, True, I('sd.png'),
                    _('Storage Card B')),
        ]

        later_menus = []

        for menu in (self, self.set_default_menu):
            for actions, desc in (
                    (basic_actions, ''),
                    (specific_actions, _('Send specific format to')),
                    (delete_actions, _('Send and delete from library')),
                    ):
                mdest = menu
                if actions is not basic_actions:
                    mdest = QMenu(desc)
                    self._memory.append(mdest)
                    later_menus.append(mdest)
                    if menu is self.set_default_menu:
                        menu.addMenu(mdest)
                        menu.addSeparator()

                for dest, delete, specific, icon, text in actions:
                    action = DeviceAction(dest, delete, specific, icon, text, self)
                    self._memory.append(action)
                    if menu is self.set_default_menu:
                        action.setCheckable(True)
                        action.setText(action.text())
                        self.group.addAction(action)
                    else:
                        action.a_s.connect(self.action_triggered)
                        self.actions.append(action)
                    mdest.addAction(action)
                if actions is basic_actions:
                    menu.addSeparator()

        da = config['default_send_to_device_action']
        done = False
        for action in self.group.actions():
            if repr(action) == da:
                action.setChecked(True)
                done = True
                break
        if not done:
            action = list(self.group.actions())[0]
            action.setChecked(True)
            config['default_send_to_device_action'] = repr(action)

        self.group.triggered.connect(self.change_default_action)
        self.addSeparator()

        self.addMenu(later_menus[0])
        self.addSeparator()

        mitem = self.addAction(QIcon(I('eject.png')), _('Eject device'))
        mitem.setEnabled(False)
        mitem.triggered.connect(lambda x : self.disconnect_mounted_device.emit())
        self.disconnect_mounted_device_action = mitem
        self.addSeparator()

        self.addMenu(self.set_default_menu)
        self.addSeparator()

        self.addMenu(later_menus[1])
        self.addSeparator()

        annot = self.addAction(_('Fetch annotations (experimental)'))
        annot.setEnabled(False)
        annot.triggered.connect(lambda x :
                self.fetch_annotations.emit())
        self.annotation_action = annot
        self.enable_device_actions(False)

    def change_default_action(self, action):
        config['default_send_to_device_action'] = repr(action)
        action.setChecked(True)

    def action_triggered(self, action):
        self.emit(SIGNAL('sync(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'),
                action.dest, action.delete, action.specific)

    def trigger_default(self, *args):
        r = config['default_send_to_device_action']
        for action in self.actions:
            if repr(action) == r:
                self.action_triggered(action)
                break

    def enable_device_actions(self, enable, card_prefix=(None, None),
            device=None):
        for action in self.actions:
            if action.dest in ('main:', 'carda:0', 'cardb:0'):
                if not enable:
                    action.setEnabled(False)
                else:
                    if action.dest == 'main:':
                        action.setEnabled(True)
                    elif action.dest == 'carda:0':
                        if card_prefix and card_prefix[0] != None:
                            action.setEnabled(True)
                        else:
                            action.setEnabled(False)
                    elif action.dest == 'cardb:0':
                        if card_prefix and card_prefix[1] != None:
                            action.setEnabled(True)
                        else:
                            action.setEnabled(False)

        annot_enable = enable and getattr(device, 'SUPPORTS_ANNOTATIONS', False)
        self.annotation_action.setEnabled(annot_enable)

    # }}}

class DeviceMixin(object): # {{{

    def __init__(self):
        self.device_error_dialog = error_dialog(self, _('Error'),
                _('Error communicating with device'), ' ')
        self.device_error_dialog.setModal(Qt.NonModal)
        self.device_manager = DeviceManager(Dispatcher(self.device_detected),
                self.job_manager, Dispatcher(self.status_bar.show_message),
                Dispatcher(self.show_open_feedback))
        self.device_manager.start()
        if tweaks['auto_connect_to_folder']:
            self.connect_to_folder_named(tweaks['auto_connect_to_folder'])

    def show_open_feedback(self, devname, msg):
        self.__of_dev_mem__ = d = info_dialog(self, devname, msg)
        d.show()

    def auto_convert_question(self, msg, autos):
        autos = u'\n'.join(map(unicode, map(force_unicode, autos)))
        return self.ask_a_yes_no_question(
                _('No suitable formats'), msg,
                ans_when_user_unavailable=True,
                det_msg=autos
        )

    def set_default_thumbnail(self, height):
        img = I('book.png', data=True)
        self.default_thumbnail = thumbnail(img, height, height)

    def connect_to_folder_named(self, folder):
        if os.path.exists(folder) and os.path.isdir(folder):
            self.device_manager.mount_device(kls=FOLDER_DEVICE, kind='folder',
                    path=folder)

    def connect_to_folder(self):
        dir = choose_dir(self, 'Select Device Folder',
                             _('Select folder to open as device'))
        if dir is not None:
            self.device_manager.mount_device(kls=FOLDER_DEVICE, kind='folder', path=dir)

    def connect_to_bambook(self):
        self.device_manager.mount_device(kls=BAMBOOKWifi, kind='bambook',
                                         path=BAMBOOK.settings().extra_customization)

    def connect_to_itunes(self):
        self.device_manager.mount_device(kls=ITUNES_ASYNC, kind='itunes', path=None)

    # disconnect from both folder and itunes devices
    def disconnect_mounted_device(self):
        self.device_manager.umount_device()

    def _sync_action_triggered(self, *args):
        m = getattr(self, '_sync_menu', None)
        if m is not None:
            m.trigger_default()

    def create_device_menu(self):
        self._sync_menu = DeviceMenu(self)
        self.iactions['Send To Device'].qaction.setMenu(self._sync_menu)
        self.iactions['Connect Share'].build_email_entries()
        self.connect(self._sync_menu,
                SIGNAL('sync(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'),
                self.dispatch_sync_event)
        self._sync_menu.fetch_annotations.connect(
                self.iactions['Fetch Annotations'].fetch_annotations)
        self._sync_menu.disconnect_mounted_device.connect(self.disconnect_mounted_device)
        self.iactions['Connect Share'].set_state(self.device_connected)
        if self.device_connected:
            self._sync_menu.disconnect_mounted_device_action.setEnabled(True)
        else:
            self._sync_menu.disconnect_mounted_device_action.setEnabled(False)

    def device_job_exception(self, job):
        '''
        Handle exceptions in threaded device jobs.
        '''
        if isinstance(getattr(job, 'exception', None), UserFeedback):
            ex = job.exception
            func = {UserFeedback.ERROR:error_dialog,
                    UserFeedback.WARNING:warning_dialog,
                    UserFeedback.INFO:info_dialog}[ex.level]
            return func(self, _('Failed'), ex.msg, det_msg=ex.details if
                    ex.details else '', show=True)

        try:
            if 'Could not read 32 bytes on the control bus.' in \
                    unicode(job.details):
                error_dialog(self, _('Error talking to device'),
                             _('There was a temporary error talking to the '
                             'device. Please unplug and reconnect the device '
                             'and or reboot.')).show()
                return
        except:
            pass
        try:
            prints(job.details, file=sys.stderr)
        except:
            pass
        if not self.device_error_dialog.isVisible():
            self.device_error_dialog.set_details(job.details)
            self.device_error_dialog.show()

    # Device connected {{{

    def set_device_menu_items_state(self, connected):
        self.iactions['Connect Share'].set_state(connected)
        if connected:
            self._sync_menu.disconnect_mounted_device_action.setEnabled(True)
            self._sync_menu.enable_device_actions(True,
                    self.device_manager.device.card_prefix(),
                    self.device_manager.device)
            self.eject_action.setEnabled(True)
        else:
            self._sync_menu.disconnect_mounted_device_action.setEnabled(False)
            self._sync_menu.enable_device_actions(False)
            self.eject_action.setEnabled(False)

    def device_detected(self, connected, device_kind):
        '''
        Called when a device is connected to the computer.
        '''
        # This can happen as this function is called in a queued connection and
        # the user could have yanked the device in the meantime
        if connected and not self.device_manager.is_device_connected:
            connected = False
        self.set_device_menu_items_state(connected)
        if connected:
            self.device_manager.get_device_information(\
                    Dispatcher(self.info_read))
            self.set_default_thumbnail(\
                    self.device_manager.device.THUMBNAIL_HEIGHT)
            self.status_bar.show_message(_('Device: ')+\
                self.device_manager.device.__class__.get_gui_name()+\
                        _(' detected.'), 3000)
            self.device_connected = device_kind
            self.library_view.set_device_connected(self.device_connected)
            self.refresh_ondevice (reset_only = True)
        else:
            self.device_connected = None
            self.status_bar.device_disconnected()
            if self.current_view() != self.library_view:
                self.book_details.reset_info()
            self.location_manager.update_devices()
            self.library_view.set_device_connected(self.device_connected)
            self.refresh_ondevice()

    def info_read(self, job):
        '''
        Called once device information has been read.
        '''
        if job.failed:
            return self.device_job_exception(job)
        info, cp, fs = job.result
        self.location_manager.update_devices(cp, fs,
                self.device_manager.device.icon)
        self.status_bar.device_connected(info[0])
        self.device_manager.books(Dispatcher(self.metadata_downloaded))

    def metadata_downloaded(self, job):
        '''
        Called once metadata has been read for all books on the device.
        '''
        if job.failed:
            self.device_job_exception(job)
            return
        # set_books_in_library might schedule a sync_booklists job
        self.set_books_in_library(job.result, reset=True)
        mainlist, cardalist, cardblist = job.result
        self.memory_view.set_database(mainlist)
        self.memory_view.set_editable(self.device_manager.device.CAN_SET_METADATA,
                                      self.device_manager.device.BACKLOADING_ERROR_MESSAGE
                                      is None)
        self.card_a_view.set_database(cardalist)
        self.card_a_view.set_editable(self.device_manager.device.CAN_SET_METADATA,
                                      self.device_manager.device.BACKLOADING_ERROR_MESSAGE
                                      is None)
        self.card_b_view.set_database(cardblist)
        self.card_b_view.set_editable(self.device_manager.device.CAN_SET_METADATA,
                                      self.device_manager.device.BACKLOADING_ERROR_MESSAGE
                                      is None)
        self.sync_news()
        self.sync_catalogs()
        self.refresh_ondevice()

    def refresh_ondevice(self, reset_only = False):
        '''
        Force the library view to refresh, taking into consideration new
        device books information
        '''
        self.book_on_device(None, reset=True)
        if reset_only:
            return
        self.library_view.model().refresh_ondevice()

    # }}}

    def remove_paths(self, paths):
        return self.device_manager.delete_books(
                Dispatcher(self.books_deleted), paths)

    def books_deleted(self, job):
        '''
        Called once deletion is done on the device
        '''
        for view in (self.memory_view, self.card_a_view, self.card_b_view):
            view.model().deletion_done(job, job.failed)
        if job.failed:
            self.device_job_exception(job)
            return

        dm = self.iactions['Remove Books'].delete_memory
        if dm.has_key(job):
            paths, model = dm.pop(job)
            self.device_manager.remove_books_from_metadata(paths,
                    self.booklists())
            model.paths_deleted(paths)
        # Force recomputation the library's ondevice info. We need to call
        # set_books_in_library even though books were not added because
        # the deleted book might have been an exact match. Upload the booklists
        # if set_books_in_library did not.
        if not self.set_books_in_library(self.booklists(), reset=True):
            self.upload_booklists()
        self.book_on_device(None, reset=True)
        # We need to reset the ondevice flags in the library. Use a big hammer,
        # so we don't need to worry about whether some succeeded or not.
        self.refresh_ondevice(reset_only=False)

    def dispatch_sync_event(self, dest, delete, specific):
        rows = self.library_view.selectionModel().selectedRows()
        if not rows or len(rows) == 0:
            error_dialog(self, _('No books'), _('No books')+' '+\
                    _('selected to send')).exec_()
            return

        fmt = None
        if specific:
            formats = []
            aval_out_formats = available_output_formats()
            format_count = {}
            for row in rows:
                fmts = self.library_view.model().db.formats(row.row())
                if fmts:
                    for f in fmts.split(','):
                        f = f.lower()
                        if format_count.has_key(f):
                            format_count[f] += 1
                        else:
                            format_count[f] = 1
            for f in self.device_manager.device.settings().format_map:
                if f in format_count.keys():
                    formats.append((f, _('%i of %i Books') % (format_count[f],
                        len(rows)), True if f in aval_out_formats else False))
                elif f in aval_out_formats:
                    formats.append((f, _('0 of %i Books') % len(rows), True))
            d = ChooseFormatDeviceDialog(self, _('Choose format to send to device'), formats)
            if d.exec_() != QDialog.Accepted:
                return
            if d.format():
                fmt = d.format().lower()
        dest, sub_dest = dest.split(':')
        if dest in ('main', 'carda', 'cardb'):
            if not self.device_connected or not self.device_manager:
                error_dialog(self, _('No device'),
                        _('Cannot send: No device is connected')).exec_()
                return
            if dest == 'carda' and not self.device_manager.has_card():
                error_dialog(self, _('No card'),
                        _('Cannot send: Device has no storage card')).exec_()
                return
            if dest == 'cardb' and not self.device_manager.has_card():
                error_dialog(self, _('No card'),
                        _('Cannot send: Device has no storage card')).exec_()
                return
            if dest == 'main':
                on_card = None
            else:
                on_card = dest
            self.sync_to_device(on_card, delete, fmt)
        elif dest == 'mail':
            to, fmts = sub_dest.split(';')
            fmts = [x.strip().lower() for x in fmts.split(',')]
            self.send_by_mail(to, fmts, delete)

    def cover_to_thumbnail(self, data):
        if self.device_manager.device and \
                hasattr(self.device_manager.device, 'THUMBNAIL_WIDTH'):
            try:
                return thumbnail(data,
                                 self.device_manager.device.THUMBNAIL_WIDTH,
                                 self.device_manager.device.THUMBNAIL_HEIGHT,
                                 preserve_aspect_ratio=False)
            except:
                pass
            return
        ht = self.device_manager.device.THUMBNAIL_HEIGHT \
                if self.device_manager else DevicePlugin.THUMBNAIL_HEIGHT
        try:
            return thumbnail(data, ht, ht)
        except:
            pass

    def sync_catalogs(self, send_ids=None, do_auto_convert=True):
        if self.device_connected:
            settings = self.device_manager.device.settings()
            ids = list(dynamic.get('catalogs_to_be_synced', set([]))) if send_ids is None else send_ids
            ids = [id for id in ids if self.library_view.model().db.has_id(id)]
            files, _auto_ids = self.library_view.model().get_preferred_formats_from_ids(
                                ids, settings.format_map,
                                exclude_auto=do_auto_convert)
            auto = []
            if do_auto_convert and _auto_ids:
                for id in _auto_ids:
                    dbfmts = self.library_view.model().db.formats(id, index_is_id=True)
                    formats = [] if dbfmts is None else \
                        [f.lower() for f in dbfmts.split(',')]
                    if set(formats).intersection(available_input_formats()) \
                            and set(settings.format_map).intersection(available_output_formats()):
                        auto.append(id)
            if auto:
                format = None
                for fmt in settings.format_map:
                    if fmt in list(set(settings.format_map).intersection(set(available_output_formats()))):
                        format = fmt
                        break
                if format is not None:
                    autos = [self.library_view.model().db.title(id, index_is_id=True) for id in auto]
                    if self.auto_convert_question(
                        _('Auto convert the following books before uploading to '
                            'the device?'), autos):
                        self.iactions['Convert Books'].auto_convert_catalogs(auto, format)
            files = [f for f in files if f is not None]
            if not files:
                dynamic.set('catalogs_to_be_synced', set([]))
                return
            metadata = self.library_view.model().metadata_for(ids)
            names = []
            for mi in metadata:
                prefix = ascii_filename(mi.title)
                if not isinstance(prefix, unicode):
                    prefix = prefix.decode(preferred_encoding, 'replace')
                prefix = ascii_filename(prefix)
                names.append('%s_%d%s'%(prefix, id,
                    os.path.splitext(f)[1]))
                if mi.cover and os.access(mi.cover, os.R_OK):
                    mi.thumbnail = self.cover_to_thumbnail(open(mi.cover,
                        'rb').read())
            dynamic.set('catalogs_to_be_synced', set([]))
            if files:
                remove = []
                space = { self.location_manager.free[0] : None,
                    self.location_manager.free[1] : 'carda',
                    self.location_manager.free[2] : 'cardb' }
                on_card = space.get(sorted(space.keys(), reverse=True)[0], None)
                self.upload_books(files, names, metadata,
                        on_card=on_card,
                        memory=[files, remove])
                self.status_bar.show_message(_('Sending catalogs to device.'), 5000)


    @dynamic_property
    def news_to_be_synced(self):
        doc = 'Set of ids to be sent to device'
        def fget(self):
            ans = []
            try:
                ans = self.library_view.model().db.prefs.get('news_to_be_synced',
                        [])
            except:
                import traceback
                traceback.print_exc()
            return set(ans)

        def fset(self, ids):
            try:
                self.library_view.model().db.prefs.set('news_to_be_synced',
                        list(ids))
            except:
                import traceback
                traceback.print_exc()

        return property(fget=fget, fset=fset, doc=doc)


    def sync_news(self, send_ids=None, do_auto_convert=True):
        if self.device_connected:
            del_on_upload = config['delete_news_from_library_on_upload']
            settings = self.device_manager.device.settings()
            ids = list(self.news_to_be_synced) if send_ids is None else send_ids
            ids = [id for id in ids if self.library_view.model().db.has_id(id)]
            files, _auto_ids = self.library_view.model().get_preferred_formats_from_ids(
                                ids, settings.format_map,
                                exclude_auto=do_auto_convert)
            auto = []
            if do_auto_convert and _auto_ids:
                for id in _auto_ids:
                    dbfmts = self.library_view.model().db.formats(id, index_is_id=True)
                    formats = [] if dbfmts is None else \
                        [f.lower() for f in dbfmts.split(',')]
                    if set(formats).intersection(available_input_formats()) \
                            and set(settings.format_map).intersection(available_output_formats()):
                        auto.append(id)
            if auto:
                format = None
                for fmt in settings.format_map:
                    if fmt in list(set(settings.format_map).intersection(set(available_output_formats()))):
                        format = fmt
                        break
                if format is not None:
                    autos = [self.library_view.model().db.title(id, index_is_id=True) for id in auto]
                    if self.auto_convert_question(
                        _('Auto convert the following books before uploading to '
                            'the device?'), autos):
                        self.iactions['Convert Books'].auto_convert_news(auto, format)
            files = [f for f in files if f is not None]
            for f in files:
                f.deleted_after_upload = del_on_upload
            if not files:
                self.news_to_be_synced = set([])
                return
            metadata = self.library_view.model().metadata_for(ids)
            names = []
            for mi in metadata:
                prefix = ascii_filename(mi.title)
                if not isinstance(prefix, unicode):
                    prefix = prefix.decode(preferred_encoding, 'replace')
                prefix = ascii_filename(prefix)
                names.append('%s_%d%s'%(prefix, id,
                    os.path.splitext(f)[1]))
                if mi.cover and os.access(mi.cover, os.R_OK):
                    mi.thumbnail = self.cover_to_thumbnail(open(mi.cover,
                        'rb').read())
            self.news_to_be_synced = set([])
            if config['upload_news_to_device'] and files:
                remove = ids if del_on_upload else []
                space = { self.location_manager.free[0] : None,
                    self.location_manager.free[1] : 'carda',
                    self.location_manager.free[2] : 'cardb' }
                on_card = space.get(sorted(space.keys(), reverse=True)[0], None)
                try:
                    total_size = sum([os.stat(f).st_size for f in files])
                except:
                    try:
                        import traceback
                        traceback.print_exc()
                    except:
                        pass
                    total_size = self.location_manager.free[0]
                if self.location_manager.free[0] > total_size + (1024**2):
                    # Send news to main memory if enough space available
                    # as some devices like the Nook Color cannot handle
                    # periodicals on SD cards properly
                    on_card = None
                self.upload_books(files, names, metadata,
                        on_card=on_card,
                        memory=[files, remove])
                self.status_bar.show_message(_('Sending news to device.'), 5000)


    def sync_to_device(self, on_card, delete_from_library,
            specific_format=None, send_ids=None, do_auto_convert=True):
        ids = [self.library_view.model().id(r) \
               for r in self.library_view.selectionModel().selectedRows()] \
                                if send_ids is None else send_ids
        if not self.device_manager or not ids or len(ids) == 0 or \
                not self.device_manager.is_device_connected:
            return

        settings = self.device_manager.device.settings()

        _files, _auto_ids = self.library_view.model().get_preferred_formats_from_ids(ids,
                                    settings.format_map,
                                    specific_format=specific_format,
                                    exclude_auto=do_auto_convert)
        if do_auto_convert:
            ok_ids = list(set(ids).difference(_auto_ids))
            ids = [i for i in ids if i in ok_ids]
        else:
            _auto_ids = []

        metadata = self.library_view.model().metadata_for(ids)
        ids = iter(ids)
        for mi in metadata:
            if mi.cover and os.access(mi.cover, os.R_OK):
                mi.thumbnail = self.cover_to_thumbnail(open(mi.cover, 'rb').read())
        imetadata = iter(metadata)

        bad, good, gf, names, remove_ids = [], [], [], [], []
        for f in _files:
            mi = imetadata.next()
            id = ids.next()
            if f is None:
                bad.append(mi.title)
            else:
                remove_ids.append(id)
                good.append(mi)
                gf.append(f)
                t = mi.title
                if not t:
                    t = _('Unknown')
                a = mi.format_authors()
                if not a:
                    a = _('Unknown')
                prefix = ascii_filename(t+' - '+a)
                if not isinstance(prefix, unicode):
                    prefix = prefix.decode(preferred_encoding, 'replace')
                prefix = ascii_filename(prefix)
                names.append('%s_%d%s'%(prefix, id, os.path.splitext(f)[1]))
        remove = remove_ids if delete_from_library else []
        self.upload_books(gf, names, good, on_card, memory=(_files, remove))
        self.status_bar.show_message(_('Sending books to device.'), 5000)

        auto = []
        if _auto_ids != []:
            for id in _auto_ids:
                if specific_format == None:
                    formats = self.library_view.model().db.formats(id, index_is_id=True)
                    formats = formats.split(',') if formats is not None else []
                    formats = [f.lower().strip() for f in formats]
                    if list(set(formats).intersection(available_input_formats())) != [] and list(set(settings.format_map).intersection(available_output_formats())) != []:
                        auto.append(id)
                    else:
                        bad.append(self.library_view.model().db.title(id, index_is_id=True))
                else:
                    if specific_format in list(set(settings.format_map).intersection(set(available_output_formats()))):
                        auto.append(id)
                    else:
                        bad.append(self.library_view.model().db.title(id, index_is_id=True))

        if auto != []:
            format = specific_format if specific_format in \
                            list(set(settings.format_map).intersection(set(available_output_formats()))) \
                            else None
            if not format:
                for fmt in settings.format_map:
                    if fmt in list(set(settings.format_map).intersection(set(available_output_formats()))):
                        format = fmt
                        break
            if not format:
                bad += auto
            else:
                autos = [self.library_view.model().db.title(id, index_is_id=True) for id in auto]
                if self.auto_convert_question(
                    _('Auto convert the following books before uploading to '
                        'the device?'), autos):
                    self.iactions['Convert Books'].auto_convert(auto, on_card, format)

        if bad:
            bad = '\n'.join('%s'%(i,) for i in bad)
            d = warning_dialog(self, _('No suitable formats'),
                    _('Could not upload the following books to the device, '
                'as no suitable formats were found. Convert the book(s) to a '
                'format supported by your device first.'
                ), bad)
            d.exec_()

    def upload_booklists(self):
        '''
        Upload metadata to device.
        '''
        plugboards = self.library_view.model().db.prefs.get('plugboards', {})
        self.device_manager.sync_booklists(Dispatcher(self.metadata_synced),
                                           self.booklists(), plugboards)

    def metadata_synced(self, job):
        '''
        Called once metadata has been uploaded.
        '''
        if job.failed:
            self.device_job_exception(job)
            return
        cp, fs = job.result
        self.location_manager.update_devices(cp, fs,
                self.device_manager.device.icon)
        # reset the views so that up-to-date info is shown. These need to be
        # here because some drivers update collections in sync_booklists
        self.memory_view.reset()
        self.card_a_view.reset()
        self.card_b_view.reset()

    def _upload_collections(self, job):
        if job.failed:
            self.device_job_exception(job)

    def upload_collections(self, booklist, view=None, oncard=None):
        return self.device_manager.upload_collections(self._upload_collections,
                                                       booklist, oncard)

    def upload_books(self, files, names, metadata, on_card=None, memory=None):
        '''
        Upload books to device.
        :param files: List of either paths to files or file like objects
        '''
        titles = [i.title for i in metadata]
        plugboards = self.library_view.model().db.prefs.get('plugboards', {})
        job = self.device_manager.upload_books(
                Dispatcher(self.books_uploaded),
                files, names, on_card=on_card,
                metadata=metadata, titles=titles, plugboards=plugboards
              )
        self.upload_memory[job] = (metadata, on_card, memory, files)

    def books_uploaded(self, job):
        '''
        Called once books have been uploaded.
        '''
        metadata, on_card, memory, files = self.upload_memory.pop(job)

        if job.exception is not None:
            if isinstance(job.exception, FreeSpaceError):
                where = 'in main memory.' if 'memory' in str(job.exception) \
                        else 'on the storage card.'
                titles = '\n'.join(['<li>'+mi.title+'</li>' \
                                    for mi in metadata])
                d = error_dialog(self, _('No space on device'),
                                 _('<p>Cannot upload books to device there '
                                 'is no more free space available ')+where+
                                 '</p>\n<ul>%s</ul>'%(titles,))
                d.exec_()
            else:
                self.device_job_exception(job)
            return

        self.device_manager.add_books_to_metadata(job.result,
                metadata, self.booklists())

        books_to_be_deleted = []
        if memory and memory[1]:
            books_to_be_deleted = memory[1]
            self.library_view.model().delete_books_by_id(books_to_be_deleted)

        # There are some cases where sending a book to the device overwrites a
        # book already there with a different book. This happens frequently in
        # news. When this happens, the book match indication will be wrong
        # because the UUID changed. Force both the device and the library view
        # to refresh the flags. Set_books_in_library could upload the booklists.
        # If it does not, then do it here.
        if not self.set_books_in_library(self.booklists(), reset=True):
            self.upload_booklists()
        with self.library_view.preserve_selected_books:
            self.book_on_device(None, reset=True)
            self.refresh_ondevice()

        view = self.card_a_view if on_card == 'carda' else \
            self.card_b_view if on_card == 'cardb' else self.memory_view
        view.model().resort(reset=False)
        view.model().research()
        for f in files:
            getattr(f, 'close', lambda : True)()

    def book_on_device(self, id, reset=False):
        '''
        Return an indication of whether the given book represented by its db id
        is on the currently connected device. It returns a 5 element list. The
        first three elements represent memory locations main, carda, and cardb,
        and are true if the book is identifiably in that memory. The fourth
        is a count of how many instances of the book were found across all
        the memory locations. The fifth is a set of paths to the
        matching books on the device.
        '''
        loc = [None, None, None, 0, set([])]

        if reset:
            self.book_db_id_cache = None
            self.book_db_id_counts = None
            self.book_db_uuid_path_map = None
            return

        if not hasattr(self, 'db_book_uuid_cache'):
            return loc

        if self.book_db_id_cache is None:
            self.book_db_id_cache = []
            self.book_db_id_counts = {}
            self.book_db_uuid_path_map = {}
            for i, l in enumerate(self.booklists()):
                self.book_db_id_cache.append(set())
                for book in l:
                    db_id = getattr(book, 'application_id', None)
                    if db_id is not None:
                        # increment the count of books on the device with this
                        # db_id.
                        self.book_db_id_cache[i].add(db_id)
                        if db_id not in self.book_db_uuid_path_map:
                            self.book_db_uuid_path_map[db_id] = set()
                        if getattr(book, 'lpath', False):
                            self.book_db_uuid_path_map[db_id].add(book.lpath)
                        c = self.book_db_id_counts.get(db_id, 0)
                        self.book_db_id_counts[db_id] = c + 1

        for i, l in enumerate(self.booklists()):
            if id in self.book_db_id_cache[i]:
                loc[i] = True
                loc[3] = self.book_db_id_counts.get(id, 0)
                loc[4] |= self.book_db_uuid_path_map[id]
        return loc

    def set_books_in_library(self, booklists, reset=False):
        '''
        Set the ondevice indications in the device database.
        This method should be called before book_on_device is called, because
        it sets the application_id for matched books. Book_on_device uses that
        to both speed up matching and to count matches.
        '''

        string_pat = re.compile('(?u)\W|[_]')
        def clean_string(x):
            x = x.lower() if x else ''
            return string_pat.sub('', x)

        update_metadata = prefs['manage_device_metadata'] == 'on_connect'

        # Force a reset if the caches are not initialized
        if reset or not hasattr(self, 'db_book_title_cache'):
            # Build a cache (map) of the library, so the search isn't On**2
            db_book_title_cache = {}
            db_book_uuid_cache = {}
            # It might be possible to get here without having initialized the
            # library view. In this case, simply give up
            try:
                db = self.library_view.model().db
            except:
                return False

            get_covers = False
            if update_metadata and self.device_manager.is_device_connected:
                if self.device_manager.device.WANTS_UPDATED_THUMBNAILS:
                    get_covers = True

            for id in db.data.iterallids():
                mi = db.get_metadata(id, index_is_id=True, get_cover=get_covers)
                title = clean_string(mi.title)
                if title not in db_book_title_cache:
                    db_book_title_cache[title] = \
                                {'authors':{}, 'author_sort':{}, 'db_ids':{}}
                # If there are multiple books in the library with the same title
                # and author, then remember the last one. That is OK, because as
                # we can't tell the difference between the books, one is as good
                # as another.
                if mi.authors:
                    authors = clean_string(authors_to_string(mi.authors))
                    db_book_title_cache[title]['authors'][authors] = mi
                if mi.author_sort:
                    aus = clean_string(mi.author_sort)
                    db_book_title_cache[title]['author_sort'][aus] = mi
                db_book_title_cache[title]['db_ids'][mi.application_id] = mi
                db_book_uuid_cache[mi.uuid] = mi
            self.db_book_title_cache = db_book_title_cache
            self.db_book_uuid_cache = db_book_uuid_cache

        # Now iterate through all the books on the device, setting the
        # in_library field. If the UUID matches a book in the library, then
        # do not consider that book for other matching. In all cases set
        # the application_id to the db_id of the matching book. This value
        # will be used by books_on_device to indicate matches.

        for booklist in booklists:
            for book in booklist:
                book.in_library = None
                if getattr(book, 'uuid', None) in self.db_book_uuid_cache:
                    if update_metadata:
                        book.smart_update(self.db_book_uuid_cache[book.uuid],
                                          replace_metadata=True)
                    book.in_library = 'UUID'
                    # ensure that the correct application_id is set
                    book.application_id = \
                        self.db_book_uuid_cache[book.uuid].application_id
                    continue
                # No UUID exact match. Try metadata matching.
                book_title = clean_string(book.title)
                d = self.db_book_title_cache.get(book_title, None)
                if d is not None:
                    # At this point we know that the title matches. The book
                    # will match if any of the db_id, author, or author_sort
                    # also match.
                    if getattr(book, 'application_id', None) in d['db_ids']:
                        # app_id already matches a db_id. No need to set it.
                        if update_metadata:
                            book.smart_update(d['db_ids'][book.application_id],
                                              replace_metadata=True)
                        book.in_library = 'APP_ID'
                        continue
                    # Sonys know their db_id independent of the application_id
                    # in the metadata cache. Check that as well.
                    if getattr(book, 'db_id', None) in d['db_ids']:
                        if update_metadata:
                            book.smart_update(d['db_ids'][book.db_id],
                                              replace_metadata=True)
                        book.in_library = 'DB_ID'
                        book.application_id = \
                                    d['db_ids'][book.db_id].application_id
                        continue
                    # We now know that the application_id is not right. Set it
                    # to None to prevent book_on_device from accidentally
                    # matching on it. It will be set to a correct value below if
                    # the book is matched with one in the library
                    book.application_id = None
                    if book.authors:
                        # Compare against both author and author sort, because
                        # either can appear as the author
                        book_authors = clean_string(authors_to_string(book.authors))
                        if book_authors in d['authors']:
                            if update_metadata:
                                book.smart_update(d['authors'][book_authors],
                                                  replace_metadata=True)
                            book.in_library = 'AUTHOR'
                            book.application_id = \
                                    d['authors'][book_authors].application_id
                        elif book_authors in d['author_sort']:
                            if update_metadata:
                                book.smart_update(d['author_sort'][book_authors],
                                                  replace_metadata=True)
                            book.in_library = 'AUTH_SORT'
                            book.application_id = \
                                d['author_sort'][book_authors].application_id
                else:
                    # Book definitely not matched. Clear its application ID
                    book.application_id = None
                # Set author_sort if it isn't already
                asort = getattr(book, 'author_sort', None)
                if not asort and book.authors:
                    book.author_sort = self.library_view.model().db.\
                                author_sort_from_authors(book.authors)

        if update_metadata:
            if self.device_manager.is_device_connected:
                if self.device_manager.device.WANTS_UPDATED_THUMBNAILS:
                    for blist in booklists:
                        for book in blist:
                            if book.cover and os.access(book.cover, os.R_OK):
                                book.thumbnail = \
                                    self.cover_to_thumbnail(open(book.cover, 'rb').read())
                plugboards = self.library_view.model().db.prefs.get('plugboards', {})
                self.device_manager.sync_booklists(
                                    Dispatcher(self.metadata_synced), booklists,
                                    plugboards)
        return update_metadata
    # }}}

