Ticket #305: scanfiles.2.py

#!/usr/bin/env python
#

import os
import os.path
import datetime
import string
import optparse
import exceptions
import re

import eyeD3
import eyeD3.tag
import eyeD3.frames
import eyeD3.utils

# We are not running inside the django framework so we need to tell it where
# to find our django app framework.
#
os.environ["DJANGO_SETTINGS_MODULE"] = "mediaserv.settings.main"

from django.models.music import *

"""This program is intended to be invoked periodically on the system hosting
the mediaserv app.

It will load up the mediaserv models.

It will then iterate through the defined MusicRoot's. For each MusicRoot it
will scan all the files in it and all of its sub-directories.

For every file that is an mp3 it will attempt to parse the ID3 tags for that
file and based on those tags it will create Artist, ArtistName, Album, and
Track records in our database.

For every track that we encounter we will update the 'last_scanned' field.

After we finish scanning a MusicRoot we ask for all of the tracks whose
last_scanned field is before its MusicRoot's last_scan_started date. We will
then remove those tracks from the system.
"""

quote_pattern = re.compile("'")

############################################################################
#
def q( string ):
    """Simple function to take the given string and append single quotes ( ' )
    around it. it will also go through the string first and double any single
    quotes in it so that they are SQL safe.
    """

    return "'%s'" % quote_pattern.sub("''", string)

############################################################################
#
def is_simple_charset( value ):
    """Given a string return True if it only contains printable ascii
    characters, otherwise return false.

    This is our simple test to see if a string is a fancy UTF-8 one or
    something else, or is a simple plain ASCII string.
    """
    for char in value:
        if char not in string.printable:
            return False
    return True

############################################################################
#
def create_artist(id3_artist):
    """We are called with the name of an artist. We create a new Arist object
    and ArtistName and tie them together.

    If the ArtistName is SJIS or has 8bit characters then we also flag this
    name as not having a simple character set.
    """

    art = artists.Artist(date_added = datetime.datetime.now())
    art.save()
    art.add_artistname("'%s'" % id3_artist, preference = 0,
                       simple_char_set = is_simple_charset(id3_artist))
    return art

############################################################################
#
def create_album(id3_album):
    """Create a new album object with the given name.
    """
    alb = albums.Album(name = "'%s'" % id3_album,
                       date_added = datetime.datetime.now())
    alb.save()
    return alb
    
############################################################################
#
def add_file(filename, af, music_root, verbosity = "quiet"):
    """We have a file. We know it is an audio/mpeg file. We know we do not have
    this file in our track db. Query the file for its info and add the
    requisite objects to our db.
    """

    tag = af.getTag()

    playtime = af.getPlayTime()
    (vbr, bitrate) = af.getBitRate()
    if vbr == 0:
        vbr = False
    else:
        vbr = True
        
    # Let us pull out all the id3 tags we have, filling in default info
    #
    if tag is None:
        # The file has no id3 info. We create a title based on the file name
        # and everything else is empty.
        #
        track = tracks.Track(title = "'%s'" % os.path.basename(filename),
                             filename =  filename,
                             last_scanned = datetime.datetime.now(),
                             play_time = playtime, bit_rate = bitrate,
                             vbr = vbr, musicroot_id = music_root.id)
        track.save()
        return


    # Get the title, if it does not exist, use the file name.
    #
    title = tag.getTitle()
    if not title:
        title = filename

    # At this point we have the information to create the basic track object.
    # The rest of the fields are optional.
    #
    track = tracks.Track(title = q(title), filename = filename,
                         last_scanned = datetime.datetime.now(),
                         play_time = playtime, bit_rate = bitrate, vbr = vbr,
                         musicroot_id = music_root.id)
    
    # Get the artist tag. See if we have an artist with this exact name.
    # If we do not, then create a new artist.
    #
    id3_artist = tag.getArtist()
    if id3_artist:
        try:
            artist = artistnames.get_object(name__exact = \
                                            "'%s'" % id3_artist).get_artist()
        except artistnames.ArtistNameDoesNotExist:
            artist = create_artist(id3_artist)
        track.artist_id = artist.id

    # See if we can get the track number & disc number (note: getTrackNum() &
    # getDiscNum() return a tuple (track num, total tracks) (disc num, total
    # discs) so we only one element 0 of the tuple.
    #
    id3_tracknum = tag.getTrackNum()[0]
    if id3_tracknum:
        track.track_number = id3_tracknum
    id3_discnum = tag.getDiscNum()[0]
    if id3_discnum:
        track.disc_number = id3_discnum

    # The album is like the artist. We see if we can find an album that exists
    # with the exact name. If we can then we use it. If we can not then we
    # create a new album and use that.
    #
    id3_album = tag.getAlbum()
    if id3_album:
        try:
            album = albums.get_object(name__exact = "'%s'" % id3_album)
        except albums.AlbumDoesNotExist:
            album = create_album(id3_album)
        track.album_id = album.id

    # Going to skip genre for now. Just really do not care much about it.
    #
    track.save()

    # Depending on the verbosity print out info on the track we just scanned.
    #
    if verbosity == "terse":
        print "Added track: %s" % track
    elif verbose == "verbose":
        print "Added track: %s .. (and other info)" % track
        
    print "Track %s, last scanned: %s" % (track, track.last_scanned)
    return

############################################################################
#
#
def scan_file(filename, music_root, verbosity = "quiet"):
    """This function is given a file name as an actual absolute file path.
    We will now use the eyeD3 library to scan any id3 tags it may have and
    based on the content of those tags create tracks, artists, artistnames, and
    albums in our database.

    If a track object already exists for this file we will see if any of the
    records in our database need to be updated (and update them.)
    """
    
    # If the file is not an mp3 file we just return.
    #
    if not eyeD3.tag.isMp3File(filename):
        if verbosity == "verbose":
            print "Skipping file %s (not an audio/mpeg file)" % filename
        return

    if verbosity == "verbose":
        print "  Scanning file %s" % filename
        
    try:
        af = eyeD3.tag.Mp3AudioFile(filename)
    except Exception, e:
        print "Unable to parse file: %s" % filename
        return
    tag = af.getTag()
    if tag is None:
        if verbosity == "verbose":
            print "File %s had no id3 tag information. Filling in defaults" % \
                  filename

    # First see if a track already exists that refers to this exact same
    # file. This is because files are the acutal item that identifies a
    # track. If the file already exists then we already have this track in our
    # db. We just need to make sure that all the fields we have in the db match
    # the ones in this file.
    #
    try:
        track = tracks.get_object(filename__exact = filename)
    except tracks.TrackDoesNotExist:
        add_file(filename, af, music_root, verbosity)
        return

    # This track already existed in our db. Check to see if any of its id3 tags
    # differ from what we already have in the db. If they do, update the db.
    #
    print "We would normually update track %s, but we are skipping it for " \
          "now" % os.path.basename(filename)
    track.last_scanned = datetime.datetime.now()
    track.save()
    #compare_update_file(filename, af, music_root, verbosity = verbosity)
    return

############################################################################
#
#
def run(verbosity = "quiet"):
    """This is the function that actually does the work of scanning all of our
    MusicRoots for .mp3 files.

    It expects a single argument: a string that indicates the verbosity
    level. This mean be either 'verbose', 'terse', or 'quiet.' If not specified
    it will default to 'quiet.'
    """

    # Get the list of defined MusicRoots. These had better point to real
    # directories!
    #
    music_roots = musicroots.get_list()
    for music_root in music_roots:

        # We first mark that we actually started to scan this music root.
        #
        music_root.last_scan_started = datetime.datetime.now()
        music_root.save()

#        music_root = musicroots.get_object(pk = music_root.id)

        if verbosity == "verbose":
            print "Started scanning MusicRoot %s at %s" % \
                  (music_root.directory, music_root.last_scan_started)

        # Then the magic walk happens
        #
        for root, dirs, files in os.walk(music_root.directory):
            if verbosity == "verbose":
                print "Scanning directory: %s" % root
                
            for f in files:
                check_file = os.path.join(root, f)
                scan_file(check_file, music_root, verbosity = verbosity)

        # Okay. Our magic walk happeend. Now we need to delete any tracks that
        # had been a part of this music root but were not scanned in this run
        #
        missing_tracks = \
                       music_root.get_track_list(last_scanned__lt = \
                                                 music_root.last_scan_started)

        print "\n\n** Music root last scanned: %s" % music_root.last_scan_started
        for track in missing_tracks:
            # If it is a member of any playlists remove it..
            #
            print "Track %s last scanned: %s" % (track, track.last_scanned)
            if verbosity == "verbose" or verbosity == "terse":
                print "Track %s not found in scan. Deleting from MusicRoot " \
                      "%s" % (track, music_root)
            track.set_playlists([])
            track.delete()

        # Done scanning a music root. Indicate when we finished scanning it.
        #
        music_root.last_scan_finished = datetime.datetime.now()
        music_root.save()

    return

############################################################################
#
#
def setup_option_parser():
    """This function uses the python OptionParser module to define an option
    parser for parsing the command line options for this script. This does not
    actually parse the command line options. It returns the parser object that
    can be used for parsing them.
    """
    parser = optparse.OptionParser(usage = "%prog [options]",
                                   version = "%prog 1.0")
    parser.add_option("-v", "--verbosity", type="choice", dest="verbosity",
                      default="terse", choices = ["verbose", "terse",
                                                  "quiet"],
                      help = """Controls how talkative the script is about what
                      it is doing. In 'verbose' mode it will tell you
                      every track it finds. In 'terse' mode it will only tell
                      you about tracks that are changed, added or removed.
                      In 'quiet' mode it will say nothing. DEFAULT:
                      '%default'""")
    return parser

############################################################################
#
def main():
    """The main routine. This is invoked if this file is run as a program
    instead of being imported as a library.

    If you are running this as a module you should not invoke the 'main()'
    function but should instead invoke the 'run()' function.
    """

    parser = setup_option_parser()
    (opts, args) = parser.parse_args()

    run(opts.verbosity)

###########
#
# The work starts here
#

if __name__ == "__main__":
    main()

#
#
#
###########