From: Samir Benmendil <samir.benmendil@gmail.com>
Date: Wed, 7 Jan 2015 02:49:21 +0000 (+0000)
Subject: Merge branch 'flexget'
X-Git-Url: https://git.rmz.io/dotfiles.git/commitdiff_plain/ca89775db49fd0376f920e77487e0485e98e787f?hp=6c189118cecfab00dea01b7b516cb572babc4045

Merge branch 'flexget'
---

diff --git a/flexget/.gitignore b/flexget/.gitignore
index 9625a75..3d4e124 100644
--- a/flexget/.gitignore
+++ b/flexget/.gitignore
@@ -1,4 +1,7 @@
 db-config.sqlite
+test-config.sqlite
 flexget.log*
 received/
 temp/
+private.yml
+.config-lock
diff --git a/flexget/config.yml b/flexget/config.yml
index e306cf9..40b5b26 100644
--- a/flexget/config.yml
+++ b/flexget/config.yml
@@ -1,87 +1,193 @@
-email:
+secrets: private.yml
+
+email: # {{{1
   from: flexget@tardis
   to: ramsi@localhost
 
-presets:
-  tv:
+templates: # {{{1
+  torrents: # {{{2
+    inputs:
+      - rss: { url: 'https://ezrss.it/feed/', silent: yes }
+      - rss: { url: 'http://publichd.se/rss.php', silent: yes }
+      - rss: { url: 'http://rss.bt-chat.com/?group=3', silent: yes }
+      - rss: { url: 'http://kickass.to/tv/?rss=1', silent: yes }
+      - rss: { url: 'http://showrss.info/feeds/all.rss', silent: yes }
+      # - rss: { url: 'http://www.torlock.com/television/rss.xml', silent: yes }
+      # - rss: { url: 'http://torrentz.eu/feed_verified?q=tv', silent: yes }
+      # - rss: { url: 'http://rss.thepiratebay.org/208', silent: yes }
+
+    transmission:
+      host: localhost
+      port: 9091
+      path: /mnt/Skaro/torrents/done
+      ratio: 0.0
+
+  usenet: # {{{2
+    sabnzbd:
+      key: '{{secrets.sabnzbd.key}}'
+      url: '{{secrets.sabnzbd.url}}'
+
+  usenet_paused: # {{{2
+    sabnzbd:
+      key: '{{secrets.sabnzbd.key}}'
+      url: '{{secrets.sabnzbd.url}}'
+      priority: -2
+
+  usenet-movies: # {{{2
+    template: usenet
+    set:
+      category: 'to check'
+    inputs:
+      - rss: { url: 'https://www.usenet-crawler.com/rss?t=2040&dl=1&num=100{{secrets.crawler.api}}', silent: yes } # Movies HD
+
+  usenet-tv: # {{{2
+    template: usenet
+    inputs:
+      - rss: { url: 'https://www.usenet-crawler.com/rss?t=5030&dl=1&num=100{{secrets.crawler.api}}', silent: yes } # TV SD
+      - rss: { url: 'https://www.usenet-crawler.com/rss?t=5040&dl=1&num=100{{secrets.crawler.api}}', silent: yes } # TV HD
+
+  filters: # {{{2
     content_filter:
+      require:
+        - '*.mkv'
+        - '*.mp4'
       reject:
+        - 'filename.mkv'
         - '*.rar'
         - '*.r0*'
         - '*.wmv'
+        - '*.exe'
+    regexp:
+      reject:
+        - \b3-?D\b:       {from: title}
+        - \btrailer\b:    {from: title} # I've gotten a few trailers that matched as movies. VERY annoying
+        - \bR5\b:         {from: title} # The following are poor quality types that somehow got through the quality
+        - \bWEBSCR\b:     {from: title} # features usually due to the title including a keyword like "720p" as well
+        - \bscreener\b:   {from: title}
+        - \bTS\b:         {from: title}
+        - \bCam\b:        {from: title}
+        - '{C_P}':        {from: title} # The following are groups with poor quality releases
+        - TrTd TeaM:      {from: title}
+        - \[TNTVillage\]: {from: title}
+        - \[facepalm\]:   {from: title}
+        - \bASAP\b:       {from: title}
+        - \b Go\):        {from: title} # filter those misplaced french only movies
 
-    exists_series:
-      - "/mnt/Skaro/Series"
-
+  tv: # {{{2
     series:
-      settings:
-        normal:
-          timeframe: 12 hours
-          target: 1080p webdl
-          quality: 480p+
-          upgrade: yes
-
       normal:
         - 8 out of 10 cats
         - american dad
-        - breaking bad
+        - archer 2009
         - boardwalk empire
+        - breaking bad
+        - broadchurch
         - burn notice
         - chuck
+        - community
         - dexter
-        - doctor who
+        - doctor who 2005
         - downton abbey
         - eureka
         - family guy
+        - foyle's war
         - futurama
         - game of thrones
-        - house
+        - homeland
         - how i met your mother
+        - last week tonight with john oliver
+        - master of sex
         - my little pony
-        - qi
+        - mythbusters
+        - qi (xl)
         - ray donovan
         - red dwarf
         - sherlock
+        - suits
         - the big bang theory
+        - the blacklist
         - the cleveland show
         - the walking dead
         - two and a half men
-        - weeds
+        - the wire:
+            quality: 720p+
 
-    transmission:
-      host: localhost
-      port: 9091
-      path: /mnt/Skaro/Torrents.flex
-      removewhendone: yes
-      ratio: 0.0
+  movies: # {{{2
+    imdb:
+      min_score: 7.5
+      min_votes: 50000
+    imdb_required: on
+    my_exists_movie:
+      path:
+        - '/mnt/Skaro/Movies'
+      allow_different_qualities: better
+      type: files
+    seen_movies: strict
+
+tasks: # {{{1
+  tv-shows: # {{{2
+    series:
+      settings:
+        normal:
+          timeframe: 12 hours
+          target: 1080p webdl
+          quality: 480p+
+          upgrade: yes
 
-tasks:
-  tv-shows:
     priority: 1
-    inputs:
-      - rss: { url: 'https://ezrss.it/feed/', silent: yes }
-      - rss: { url: 'http://publichd.se/rss.php', silent: yes }
-      - rss: { url: 'http://rss.bt-chat.com/?group=3', silent: yes }
-      - rss: { url: 'http://kickass.to/tv/?rss=1', silent: yes }
-      - rss: { url: 'http://showrss.info/feeds/all.rss', silent: yes }
-#      - rss: { url: 'http://www.torlock.com/television/rss.xml', silent: yes }
-#      - rss: { url: 'http://torrentz.eu/feed_verified?q=tv', silent: yes }
-#      - rss: { url: 'http://rss.thepiratebay.org/208', silent: yes }
-    preset: tv
+    template:
+      - tv
+      - filters
+      - usenet-tv
+    exists_series:
+      path: '/mnt/Skaro/Series/{{tvdb_series_name}}'
+      allow_different_qualities: better
+    content_size:
+      min:  100
+      max: 6000
+      strict: yes
 
+  movies-1080p: # {{{2
+    priority: 1
+    template:
+      - movies
+      - filters
+      - usenet-movies
+    quality: 1080p webdl+
+    content_size:
+      min:  3000
+      max: 15000
+      strict: yes
 
-  move-episodes:
+  movies-720p: # {{{2
+    priority: 2
+    template:
+      - movies
+      - filters
+      - usenet-movies
+    quality: 720p hdrip+
+    delay: 7 days
+    content_size:
+      min:  1000
+      max: 10000
+      strict: yes
+
+  move-tv: # {{{2
     priority: 2
     thetvdb_lookup: yes
     metainfo_series: yes
-    accept_all: yes
-    # only reject files that have been seen by this task
     seen: local
+    disable_builtins: [retry_failed]
+    all_series:
+      parse_only: yes
+    accept_all: yes
 
     find:
-      path: /mnt/Skaro/Torrents.flex
+      path:
+        - /mnt/Skaro/usenet/done
+        - /mnt/Skaro/torrents/done
       recursive: yes
-      regexp: '.*\.(avi|mkv|mp4|mpg|divx)$'
+      regexp: '.*\.(avi|mkv|mp4|mpg|divx|webm)$'
 
     regexp:
       reject:
@@ -89,21 +195,65 @@ tasks:
 
     move:
       to: "/mnt/Skaro/Series/{{tvdb_series_name}}/{% if tvdb_season == 0 %}Specials{% else %}Season {{tvdb_season}}{% endif %}/"
-      filename: "{{tvdb_season}}x{{tvdb_episode|pad(2)}} {{tvdb_ep_name|replace('/','-')}} ({{tvdb_ep_air_date|formatdate('%Y')}}){{'{'}}{{quality|replace(' ','}{')}}{{'}'}}[en]{{location|pathext}}"
+      filename: "{{tvdb_season}}x{{tvdb_episode|pad(2)}} {{tvdb_ep_name|replace('/','-')}} ({{tvdb_ep_air_date|formatdate('%Y')}}){{'{'}}{{quality|replace(' ','}{')}}{{'}'}}[en]"
+      clean_source: 100   # removes dir if < [N] MB
 
-  move-movies:
+  move-movies: # {{{2
     priority: 3
-    tmdb_lookup: yes
+    imdb_lookup: yes
     accept_all: yes
     seen: local
+    disable_builtins: [retry_failed]
 
     find:
-      path: /mnt/Skaro/Torrents.flex
+      path:
+        - /mnt/Skaro/usenet/done
+        - /mnt/Skaro/torrents/done
       recursive: yes
       regexp: '.*\.(mkv|avi|mp4)$'
 
     move:
       to: "/mnt/Skaro/Movies"
-      filename: "{{tmdb_name}} ({{tmdb_year}}){{'{'}}{{quality|replace(' ','}{')}}{{'}'}}[en]{{location|pathext}}"
+      filename: "{{imdb_name}} ({{imdb_year}}){{'{'}}{{quality|replace(' ','}{')}}{{'}'}}[en]"
+      clean_source: 100   # removes dir if < [N] MB
+
+  seed-tv-db: # {{{2
+    manual: yes
+    find:
+      path: '/mnt/Skaro/Series'
+      regexp: '.*\.(avi|mkv|mp4|webm)$'
+      recursive: yes
+    require_field:
+      - series_name
+    manipulate:
+      - title:
+          from: location
+          replace:
+            regexp: '.*/([^/]+)/[^/]+/(\d+)x(\d+)([^/]*)\.[^.]+$'
+            format: '\1 S\2E\3\4'
+      - title:
+          replace:
+            regexp: '\[.*\]'
+            format: ''
+      - title:
+          replace:
+            regexp: '[\(\)\{\}]'
+            format: ' '
+    all_series: yes
+    assume_quality: 480p
+    # template: tv
+    # configure_series:
+    #   from:
+    #     listdir:
+    #       - /mnt/Skaro/Series
+    # series:
+    #   settings:
+    #     normal:
+    #       target: 480p
+
+schedules: # {{{1
+  - tasks: '*'
+    interval:
+      hours: 1
 
 # vim: sts=2 ts=2 sw=2 et
diff --git a/flexget/plugins/my_exists_movie.py b/flexget/plugins/my_exists_movie.py
new file mode 100644
index 0000000..92d6e12
--- /dev/null
+++ b/flexget/plugins/my_exists_movie.py
@@ -0,0 +1,156 @@
+from __future__ import unicode_literals, division, absolute_import
+import os
+import re
+import logging
+
+from flexget import plugin
+from flexget.event import event
+from flexget.config_schema import one_or_more
+from flexget.plugin import get_plugin_by_name
+from flexget.utils.tools import TimedDict
+
+log = logging.getLogger('my_exists_movie')
+
+
+class FilterExistsMovie(object):
+    """
+    Reject existing movies.
+
+    Example::
+
+      exists_movie: /storage/movies/
+    """
+
+    schema = {
+        'anyOf': [
+            one_or_more({'type': 'string', 'format': 'path'}),
+            {
+                'type': 'object',
+                'properties': {
+                    'path': one_or_more({'type': 'string', 'format': 'path'}),
+                    'allow_different_qualities': {'enum': ['better', True, False], 'default': False},
+                    'type': {'enum': ['files', 'dirs'], 'default': 'files'}
+                },
+                'required': ['path'],
+                'additionalProperties': False
+            }
+        ]
+    }
+
+    def __init__(self):
+        self.cache = TimedDict(cache_time='1 hour')
+
+    def prepare_config(self, config):
+        # if config is not a dict, assign value to 'path' key
+        if not isinstance(config, dict):
+            config = {'path': config}
+        # if only a single path is passed turn it into a 1 element list
+        if isinstance(config['path'], basestring):
+            config['path'] = [config['path']]
+        return config
+
+    @plugin.priority(-1)
+    def on_task_filter(self, task, config):
+        # if not task.accepted:
+        #     log.debug('nothing accepted, aborting')
+        #     return
+
+        config = self.prepare_config(config)
+        imdb_lookup = plugin.get_plugin_by_name('imdb_lookup').instance
+
+        incompatible_files = 0
+        incompatible_entries = 0
+        count_entries = 0
+        count_files = 0
+
+        # list of imdb ids gathered from paths / cache
+        qualities = {}
+
+        for folder in config['path']:
+            folder = os.path.expanduser(folder)
+            # see if this path has already been scanned
+            if folder in self.cache:
+                log.verbose('Using cached scan for %s ...' % folder)
+                qualities.update(self.cache[folder])
+                continue
+
+            path_ids = {}
+
+            if not os.path.isdir(folder):
+                log.critical('Path %s does not exist' % folder)
+                continue
+
+            log.verbose('Scanning path %s ...' % folder)
+
+            # Help debugging by removing a lot of noise
+            #logging.getLogger('movieparser').setLevel(logging.WARNING)
+            #logging.getLogger('imdb_lookup').setLevel(logging.WARNING)
+
+            # scan through
+            for root, dirs, files in os.walk(folder):
+                for item in eval(config['type']):
+                    log.debug("item: %s" % item)
+                    pattern = re.compile(".*\.(avi|mkv|mp4|mpg|webm)")
+                    if not re.search(pattern, item):
+                        continue
+                    count_files += 1
+
+                    movie = get_plugin_by_name('parsing').instance.parse_movie(item)
+
+                    try:
+                        imdb_id = imdb_lookup.imdb_id_lookup(movie_title=movie.name,
+                                                            raw_title=item,
+                                                            session=task.session)
+                        if imdb_id in path_ids:
+                            log.trace('duplicate %s' % item)
+                            continue
+                        if imdb_id is not None:
+                            log.trace('adding: %s' % imdb_id)
+                            path_ids[imdb_id] = movie.quality
+                    except plugin.PluginError as e:
+                        log.trace('%s lookup failed (%s)' % (item, e.value))
+                        incompatible_files += 1
+
+            # store to cache and extend to found list
+            self.cache[folder] = path_ids
+            qualities.update(path_ids)
+
+        log.debug('qualities: %s' % qualities)
+
+        log.debug('-- Start filtering entries ----------------------------------')
+
+        # do actual filtering
+        for entry in task.accepted:
+            count_entries += 1
+            if not entry.get('imdb_id', eval_lazy=False):
+                try:
+                    imdb_lookup.lookup(entry)
+                except plugin.PluginError as e:
+                    log.trace('entry %s imdb failed (%s)' % (entry['title'], e.value))
+                    incompatible_entries += 1
+                    continue
+
+            entry.trace("msg")
+            # actual filtering
+            if entry['imdb_id'] in qualities:
+                if config.get('allow_different_qualities') == 'better':
+                    if entry['quality'] > qualities[entry['imdb_id']]:
+                        log.trace('better quality')
+                        continue
+                elif config.get('allow_different_qualities'):
+                    if entry['quality'] != qualities[entry['imdb_id']]:
+                        log.trace('wrong quality')
+                        continue
+
+                entry.reject('movie exists')
+
+        if incompatible_files or incompatible_entries:
+            log.verbose('There were some incompatible items. %s of %s entries '
+                        'and %s of %s directories could not be verified.' %
+                (incompatible_entries, count_entries, incompatible_files, count_dirs))
+
+        log.debug('-- Finished filtering entries -------------------------------')
+
+@event('plugin.register')
+def register_plugin():
+    plugin.register(FilterExistsMovie, 'my_exists_movie', groups=['exists'], api_ver=2)