X-Git-Url: https://git.rmz.io/dotfiles.git/blobdiff_plain/61d7dd11d4a450a64f2817ee4db0ec7fa5880b42..b879c5b2c5b47d6fe1e9f5fb99efd09bccaaf752:/bin/old/jamu.py diff --git a/bin/old/jamu.py b/bin/old/jamu.py index b738afc..37b5f9c 100755 --- a/bin/old/jamu.py +++ b/bin/old/jamu.py @@ -47,7 +47,7 @@ Users of this script are encouraged to populate both themoviedb.com and thetvdb. fan art and banners and meta data. The richer the source the more valuable the script. ''' -__version__=u"v0.7.3" +__version__=u"v0.7.8" # 0.1.0 Initial development # 0.2.0 Inital beta release # 0.3.0 Add mythvideo metadata updating including movie graphics through @@ -302,6 +302,14 @@ __version__=u"v0.7.3" # 0.7.2 Fixed a bug where an inetref field was not properly initialized and caused an abort. Ticket #8243 # 0.7.3 Fixed a bug where a user selected TMDB# was not being used. # Minor change to fuzzy matching of a file named parsed title with those from TMDB and TVDB. + # 0.7.4 Update for changes in Python bindings + # 0.7.5 Added the TMDB MovieRating as videometadata table "rating" field + # 0.7.6 Modifications to support MythTV python bindings changes + # 0.7.7 Pull hostname from python bindings instead of socket libraries + # Added support of unicode characters within a jamu.conf file + # Replace 'xml' module version check with generic Python version, to correct failure in Python 2.7 + # 0.7.8 Fixed a bug which caused jamu to crash due to an extra unicode conversion introduced in 0.7.7. + # See also #9637. usage_txt=u''' @@ -440,7 +448,7 @@ without_ep_name (%(series)s - S%(seasonnumber)02dE%(episodenumber)02d.%(ext)s) import sys, os, re, locale, subprocess, locale, ConfigParser, urllib, codecs, shutil, datetime, fnmatch, string from datetime import date from optparse import OptionParser -from socket import gethostname, gethostbyname +from socket import gethostbyname import tempfile, struct import logging @@ -472,14 +480,10 @@ class OutStreamEncoder(object): sys.stdout = OutStreamEncoder(sys.stdout, 'utf8') sys.stderr = OutStreamEncoder(sys.stderr, 'utf8') -try: - import xml -except Exception, e: - print '''The python module xml must be installed. error(%s)''' % e +if sys.version_info <= (2,5): + print '''JAMU requires Python 2.5 or newer to run.''' sys.exit(1) -if xml.__version__ < u'41660': - print ''' -\n! Warning - The module xml (v41660 or greater) must be installed. Your version is different (v%s) than what Jamu was tested with. Jamu may not work on your installation.\nIt is recommended that you upgrade.\n''' % xml.__version__ + import xml.etree.cElementTree as ElementTree @@ -488,11 +492,11 @@ try: '''If the MythTV python interface is found, we can insert data directly to MythDB or get the directories to store poster, fanart, banner and episode graphics. ''' - from MythTV import MythDB, DBData, Video, MythVideo, MythBE, FileOps, MythError, MythLog + from MythTV import MythDB, Video, MythVideo, MythBE, MythError, MythLog, RecordedProgram + from MythTV.database import DBData mythdb = None mythvideo = None mythbeconn = None - localhostname = gethostname() try: '''Create an instance of each: MythDB, MythVideo ''' @@ -508,7 +512,8 @@ try: else: print u'\n! Warning - Check that (%s) is correctly configured\n' % filename except Exception, e: - print u"\n! Warning - Creating an instance caused an error for one of: MythDBConn or MythVideo, error(%s)\n" % e + print u"\n! Warning - Creating an instance caused an error for one of: MythDB or MythVideo, error(%s)\n" % e + localhostname = mythdb.gethostname() try: MythLog._setlevel('none') # Some non option -M cannot have any logging on stdout mythbeconn = MythBE(backend=localhostname, db=mythdb) @@ -578,33 +583,23 @@ if imdb_lib: sys.exit(1) class VideoTypes( DBData ): - table = 'videotypes' - where = 'intid=%s' - setwheredat = 'self.intid,' - logmodule = 'Python VideoType' - @staticmethod - def getAll(db=None): - db = MythDB(db) - c = db.cursor() - c.execute("""SELECT * FROM videotypes""") - types = [] - for row in c.fetchall(): - types.append(VideoTypes(db=db, raw=row)) - c.close() - return types + _table = 'videotypes' + _where = 'intid=%s' + _setwheredat = 'self.intid,' + _logmodule = 'Python VideoType' def __str__(self): return "" % self.extension def __repr__(self): return str(self).encode('utf-8') - def __init__(self, id=None, ext=None, db=None, raw=None): - if raw is not None: - DBData.__init__(self, db=db, raw=raw) - elif id is not None: + def __init__(self, id=None, ext=None, db=None): + if id is not None: DBData.__init__(self, data=(id,), db=db) elif ext is not None: - self.__dict__['where'] = 'extension=%s' - self.__dict__['wheredat'] = 'self.extension,' + self.__dict__['_where'] = 'extension=%s' + self.__dict__['_wheredat'] = 'self.extension,' DBData.__init__(self, data=(ext,), db=db) + else: + DBData.__init__(self, None, db=db) # end VideoTypes() def isValidPosixFilename(name, NAME_MAX=255): @@ -757,24 +752,23 @@ def getStorageGroups(): return nothing ''' records = mythdb.getStorageGroup(hostname=localhostname) - if records: - for record in records: - # Only include Video, coverfile, banner, fanart, screenshot and trailers storage groups - if record.groupname in storagegroupnames.keys(): - dirname = record.dirname - try: - dirname = unicode(record.dirname, 'utf8') - except (UnicodeDecodeError): - sys.stderr.write(u"\n! Error: The local Storage group (%s) directory contained\ncharacters that caused a UnicodeDecodeError. This storage group has been rejected.'\n" % (record['groupname'])) - continue # Skip any line that has non-utf8 characters in it - except (UnicodeEncodeError, TypeError): - pass - # Strip the trailing slash so it is consistent with all other directory paths in Jamu - if dirname[-1:] == u'/': - storagegroups[storagegroupnames[record.groupname]].append(dirname[:-1]) - else: - storagegroups[storagegroupnames[record.groupname]].append(dirname) - continue + for record in records: + # Only include Video, coverfile, banner, fanart, screenshot and trailers storage groups + if record.groupname in storagegroupnames.keys(): + dirname = record.dirname + try: + dirname = unicode(record.dirname, 'utf8') + except (UnicodeDecodeError): + sys.stderr.write(u"\n! Error: The local Storage group (%s) directory contained\ncharacters that caused a UnicodeDecodeError. This storage group has been rejected.'\n" % (record['groupname'])) + continue # Skip any line that has non-utf8 characters in it + except (UnicodeEncodeError, TypeError): + pass + # Strip the trailing slash so it is consistent with all other directory paths in Jamu + if dirname[-1:] == u'/': + storagegroups[storagegroupnames[record.groupname]].append(dirname[:-1]) + else: + storagegroups[storagegroupnames[record.groupname]].append(dirname) + continue any_storage_group = False tmp_storagegroups = dict(storagegroups) @@ -1165,18 +1159,16 @@ class Configuration(object): # regex strings to parse folder names for TV series title, season and episode numbers self.config['fullname_parse_season_episode_translation'] = {u'slash': u'\\', u'season': u'Season', u'episode': u'Episode'} self.config['fullname_parse_regex'] = [ - # Title/Season 1/01 Subtitle - u'''^.+?/(?P[^/]+)/%(season)s%(slash)s '''+ - u'''(?P[0-9]+)/(?P[0-9]+).+$''', # Title/Season 1/s01e01 Subtitle u'''^.+?/(?P[^/]+)/%(season)s%(slash)s '''+ u'''(?P[0-9]+)/[Ss][0-9]+[Ee](?P[0-9]+).+$''', + # ramsi # Title/Season 1/1x01 Subtitle -# u'''^.+?/(?P[^/]+)/%(season)s%(slash)s '''+ -# u'''(?P[0-9]+)/(?:(?P=seasno))[Xx](?P[0-9]+).+$''', - # Title [xx]/Season 1/1x01 Subtitle - u'''^.+?/(?P[^/]+?)(?:\[.*\])*/%(season)s%(slash)s '''+ - u'''(?P[0-9]+)/(?:(?P=seasno))[Xx](?P[0-9]+).+$''', + u'''^.+?/(?P[^/]+)/%(season)s%(slash)s '''+ + u'''[0-9]+/(?P[0-9]+)[Xx](?P[0-9]+).+$''', + # Title/Season 1/01 Subtitle + u'''^.+?/(?P[^/]+)/%(season)s%(slash)s '''+ + u'''(?P[0-9]+)/(?P[0-9]+).+$''', # Title/Season 1/Title s01e01 Subtitle u'''^.+?/(?P[^/]+)/%(season)s%(slash)s '''+ u'''(?P[0-9]+)/(?:(?P=seriesname))%(slash)s [Ss][0-9]+'''+ @@ -1255,7 +1247,7 @@ class Configuration(object): ) sys.exit(1) cfg = ConfigParser.SafeConfigParser() - cfg.read(useroptions) + cfg.readfp(codecs.open(useroptions, "r", "utf8")) for section in cfg.sections(): if section[:5] == 'File ': self.config['config_file'] = section[5:] @@ -1290,12 +1282,12 @@ class Configuration(object): if section == 'regex': # Change variables per user config file for option in cfg.options(section): - self.config['name_parse'].append(re.compile(unicode(cfg.get(section, option), 'utf8'), re.UNICODE)) + self.config['name_parse'].append(re.compile(cfg.get(section, option), re.UNICODE)) continue if section == 'ignore-directory': # Video directories to be excluded from Jamu processing for option in cfg.options(section): - self.config['ignore-directory'].append(unicode(cfg.get(section, option), 'utf8')) + self.config['ignore-directory'].append(cfg.get(section, option)) continue if section =='series_name_override': overrides = {} @@ -1658,22 +1650,21 @@ class Configuration(object): """ # Get videotypes table field names: try: - records = VideoTypes.getAll() + records = VideoTypes.getAllEntries(mythdb) except MythError, e: sys.stderr.write(u"\n! Error: Reading videotypes MythTV table: %s\n" % e.args[0]) return False - if records: - for record in records: - # Remove any extentions that are in Jamu's list but the user wants ignore - if record.f_ignore: - if record.extension in self.config['video_file_exts']: - self.config['video_file_exts'].remove(record.extension) - if record.extension.lower() in self.config['video_file_exts']: - self.config['video_file_exts'].remove(record.extension.lower()) - else: # Add extentions that are not in the Jamu list - if not record.extension in self.config['video_file_exts']: - self.config['video_file_exts'].append(record.extension) + for record in records: + # Remove any extentions that are in Jamu's list but the user wants ignore + if record.f_ignore: + if record.extension in self.config['video_file_exts']: + self.config['video_file_exts'].remove(record.extension) + if record.extension.lower() in self.config['video_file_exts']: + self.config['video_file_exts'].remove(record.extension.lower()) + else: # Add extentions that are not in the Jamu list + if not record.extension in self.config['video_file_exts']: + self.config['video_file_exts'].append(record.extension) # Make sure that all video file extensions are lower case for index in range(len(self.config['video_file_exts'])): self.config['video_file_exts'][index] = self.config['video_file_exts'][index].lower() @@ -2798,8 +2789,11 @@ class VideoFiles(Tvdatabase): continue ignore = False if os.path.isdir(cfile): + # ramsi allow regex in ignore-directory for directory in self.config['ignore-directory']: # ignore directory list - if not cfile.startswith(directory): + #if not cfile.startswith(directory): + if re.search(directory,cfile) is None: + print "yes" continue ignore = True if ignore: # Skip this directory @@ -2883,10 +2877,9 @@ class VideoFiles(Tvdatabase): #remove ._- characters from name (- removed only if next to end of line) seriesname = re.sub("[\._]|\-(?=$)", " ", seriesname).strip() - - # RAMSI - seriesname = re.sub("(?:\[.*\])+$", " ", seriesname).strip() - + # ramsi remove [en] tags + seriesname = re.sub("(?:\[.*\])+", " ", seriesname).strip() + seasno, epno = int(seasno), int(epno) if self.config['series_name_override']: @@ -2903,6 +2896,8 @@ class VideoFiles(Tvdatabase): movie = movie.replace(self.config['dvd'], '') categories+=u', DVD' movie = re.sub("[\._]|\-(?=$)", " ", movie).strip() + # ramsi remove [en] tags + movie = re.sub("(?:\[.*\])+", " ", movie).strip() try: allEps.append({ 'file_seriesname':movie, 'seasno':0, @@ -2955,6 +2950,8 @@ class VideoFiles(Tvdatabase): movie = movie.replace(self.config['dvd'], '') categories+=u', DVD' movie = re.sub("[\._]|\-(?=$)", " ", movie).strip() + # ramsi remove [en] tags + movie = re.sub("(?:\[.*\])+", " ", movie).strip() try: allEps.append({ 'file_seriesname':movie, 'seasno':0, @@ -3083,7 +3080,7 @@ class MythTvMetaData(VideoFiles): filename = self.rtnRelativePath(name, u'mythvideo') # Use the MythVideo hashing protocol when the video is in a storage groups if filename[0] != u'/': - hash_value = FileOps(mythbeconn.hostname).getHash(filename, u'Videos') + hash_value = mythbeconn.getHash(filename, u'Videos') if hash_value == u'NULL': return u'' else: @@ -3875,6 +3872,9 @@ class MythTvMetaData(VideoFiles): except: pass continue + if key == 'movierating': + meta_dict['rating'] = data + continue if meta_dict.has_key('rating'): if meta_dict['rating'] == '': meta_dict['rating'] = 'Unknown' @@ -4354,7 +4354,7 @@ class MythTvMetaData(VideoFiles): else: intid = result.intid if intid: - metadata = Video(id=intid, db=mythvideo) + metadata = Video(intid, db=mythvideo) if tmp_filename[0] == '/': host = u'' self.absolutepath = True @@ -4366,7 +4366,7 @@ class MythTvMetaData(VideoFiles): sys.stdout.write(u"Simulation Mythdb update for old file:\n(%s) new:\n(%s)\n" % (video_file, tmp_filename)) else: self._displayMessage(u"Mythdb update for old file:\n(%s) new:\n(%s)\n" % (video_file, tmp_filename)) - Video(id=intid, db=mythvideo).update({'filename': tmp_filename, 'host': host}) + Video(intid, db=mythvideo).update({'filename': tmp_filename, 'host': host}) num_mythdb_updates+=1 break else: @@ -4467,12 +4467,12 @@ class MythTvMetaData(VideoFiles): host = localhostname.lower() self.absolutepath = False if intid: - metadata = Video(id=intid, db=mythvideo) + metadata = Video(intid, db=mythvideo) if self.config['simulation']: sys.stdout.write(u"Simulation Mythdb update for renamed file(%s)\n" % (tmp_filename)) else: self._displayMessage(u"Mythdb update for renamed file(%s)\n" % (tmp_filename)) - Video(id=intid, db=mythvideo).update({'filename': tmp_filename, 'host': host}) + Video(intid, db=mythvideo).update({'filename': tmp_filename, 'host': host}) else: if self.config['simulation']: sys.stdout.write(u"Simulation Mythdb add for renamed file(%s)\n" % (tmp_filename)) @@ -4542,7 +4542,7 @@ class MythTvMetaData(VideoFiles): if intid == None: missing_list.append(cfile) else: - meta_dict = Video(id=intid, db=mythvideo) + meta_dict = Video(intid, db=mythvideo) if self.config['video_dir']: if not mythvideo.getVideo(exactfile=meta_dict[u'filename'], host=meta_dict[u'host']): missing_list.append(cfile) @@ -4592,7 +4592,7 @@ class MythTvMetaData(VideoFiles): repair[graphicstype] = u'No Cover' else: repair[graphicstype] = u'' - Video(id=vidintid, db=mythvideo).update(repair) + Video(vidintid, db=mythvideo).update(repair) return False # end _checkValidGraphicFile() @@ -4844,7 +4844,7 @@ class MythTvMetaData(VideoFiles): videometadatarecords=[] if len(intids): for intid in intids: - vidrec = Video(id=intid, db=mythvideo) + vidrec = Video(intid, db=mythvideo) if vidrec[u'host'] != u'' and vidrec[u'host'] != None: if vidrec[u'host'].lower() != localhostname.lower(): continue @@ -5010,18 +5010,19 @@ class MythTvMetaData(VideoFiles): sys.stdout.write( u"Simulation MythTV DB update for Miro video (%s)\n" % (program['title'],)) else: - Video(id=intid, db=mythvideo).update(changed_fields) + Video(intid, db=mythvideo).update(changed_fields) # end updateMiroVideo() def _getScheduledRecordedProgramList(self): '''Find all Scheduled and Recorded programs return array of found programs, if none then empty array is returned ''' + global localhostname programs=[] # Get pending recordings try: - progs = MythBE(backend=mythbeconn.hostname, db=mythbeconn.db).getUpcomingRecordings() + progs = mythbeconn.getUpcomingRecordings() except MythError, e: sys.stderr.write(u"\n! Error: Getting Upcoming Recordings list: %s\n" % e.args[0]) return programs @@ -5035,21 +5036,21 @@ class MythTvMetaData(VideoFiles): record['seriesid'] = prog.seriesid if record['subtitle'] and prog.airdate != None: - record['originalairdate'] = prog.airdate[:4] + record['originalairdate'] = prog.airdate.year else: if prog.year != '0': record['originalairdate'] = prog.year elif prog.airdate != None: - record['originalairdate'] = prog.airdate[:4] + record['originalairdate'] = prog.airdate.year for program in programs: # Skip duplicates if program['title'] == record['title']: break else: programs.append(record) - # Get recorded table field names: + # Get recorded records try: - recordedlist = MythBE(backend=mythbeconn.hostname, db=mythbeconn.db).getRecordings() + recordedlist = list(mythdb.searchRecorded(hostname=localhostname)) except MythError, e: sys.stderr.write(u"\n! Error: Getting recorded programs list: %s\n" % e.args[0]) return programs @@ -5058,12 +5059,7 @@ class MythTvMetaData(VideoFiles): return programs recordedprogram = {} - for recordedProgram in recordedlist: - try: - recordedRecord = recordedProgram.getRecorded() - except MythError, e: - sys.stderr.write(u"\n! Error: Getting recorded table record: %s\n" % e.args[0]) - return programs + for recordedRecord in recordedlist: if recordedRecord.recgroup == u'Deleted': continue recorded = {} @@ -5082,14 +5078,14 @@ class MythTvMetaData(VideoFiles): # Get Release year for recorded movies # Get Recorded videos recordedprogram / airdate try: - recordedDetails = recordedRecord.getRecordedProgram() + recordedDetails = dict(RecordedProgram.fromRecorded(recordedRecord)) except MythError, e: sys.stderr.write(u"\n! Error: Getting recordedprogram table record: %s\n" % e.args[0]) continue - if not recordedDetails: + if not len(recordedDetails): continue - if not recordedDetails.subtitle: - recordedprogram[recordedDetails.title]= u'%d' % recordedDetails.airdate + if not recordedDetails['subtitle']: + recordedprogram[recordedDetails['title']]= u'%d' % recordedDetails['airdate'] # Add release year to recorded movies for program in programs: @@ -5672,9 +5668,9 @@ class MythTvMetaData(VideoFiles): sys.stdout.write(u"\n\nEntry exists in MythDB but category is 0 and year is 1895 (default values).\nUpdating (%s).\n" % cfile['filename']) filename = self.rtnRelativePath(videopath, u'mythvideo') if filename[0] == u'/': - Video(id=intid, db=mythvideo).update({'filename': filename, u'host': u''}) + Video(intid, db=mythvideo).update({'filename': filename, u'host': u''}) else: - Video(id=intid, db=mythvideo).update({'filename': filename, u'host': localhostname.lower()}) + Video(intid, db=mythvideo).update({'filename': filename, u'host': localhostname.lower()}) if cfile['seasno'] == 0 and cfile['epno'] == 0: movie=True else: @@ -5682,7 +5678,7 @@ class MythTvMetaData(VideoFiles): # Get a dictionary of the existing meta data plus a copy for update comparison meta_dict={} - vim = Video(id=intid, db=mythvideo) + vim = Video(intid, db=mythvideo) for key in vim.keys(): meta_dict[key] = vim[key] @@ -5722,7 +5718,7 @@ class MythTvMetaData(VideoFiles): continue # Only update the reference number if self.config['mythtv_ref_num'] or inetref == '99999999': - Video(id=intid, db=mythvideo).update({'inetref': inetref}) + Video(intid, db=mythvideo).update({'inetref': inetref}) num_mythdb_updates+=1 videos_updated_metadata.append(cfile['filename']) self._displayMessage(u"\nReference number (%s) added for (%s) \n" % (inetref, cfile['filename'])) @@ -5742,9 +5738,9 @@ class MythTvMetaData(VideoFiles): # Only update the reference number and title if self.config['mythtv_ref_num'] or inetref == '99999999': if inetref == u'99999999': - Video(id=intid, db=mythvideo).update({'inetref': inetref}) + Video(intid, db=mythvideo).update({'inetref': inetref}) else: - Video(id=intid, db=mythvideo).update({'inetref': inetref, 'title': tmp_dict['title']}) + Video(intid, db=mythvideo).update({'inetref': inetref, 'title': tmp_dict['title']}) num_mythdb_updates+=1 videos_updated_metadata.append(cfile['filename']) self._displayMessage(u"\nReference number (%s) added for (%s) \n" % (inetref, cfile['filename'])) @@ -6185,14 +6181,14 @@ class MythTvMetaData(VideoFiles): # Clean up a few fields before updating Mythdb if available_metadata['showlevel'] == 0: # Allows mythvideo to display this video available_metadata['showlevel'] = 1 - Video(id=intid, db=mythvideo).update(available_metadata) + Video(intid, db=mythvideo).update(available_metadata) num_mythdb_updates+=1 videos_updated_metadata.append(cfile['filename']) for key in ['genres', 'cast', 'countries']: if key == 'genres' and len(cfile['categories']): genres_cast[key]+=cfile['categories'] if genres_cast.has_key(key): - self._addCastGenreCountry( genres_cast[key], Video(id=intid, db=mythvideo), key) + self._addCastGenreCountry( genres_cast[key], Video(intid, db=mythvideo), key) self._displayMessage( u"Updated Mythdb for video file(%s)\n" % cfile['filename'] )