Implement support for multiple filters

Allows for adding more filters so regex or searching by thread number
will be much easier

src/rss_thread_watch/config.clj

@@ -78,6 +78,8 @@
             boards))))
 
 (defn config-fill-board-defaults
+  ;; TODO: must have check that if board is default, it's enabled, if it's not, give some big fat warning
+  ;;  that users must always specify board, maybe change the error?
   "Fills every enabled board with default config values"
   [config]
   (let [defaults (:boards-defaults config)]

src/rss_thread_watch/core.clj

@@ -72,6 +72,7 @@
     (let [config (conf/get-some-config (:config options))]
       ;; TODO: probably refactor to use separate config.clj file when validation will be added
       ;; Init the few globals we have
+      ;; TODO: this all needs to go in separate function so it doesnt have to duplicated in repl-main
       (reset! conf/GLOBAL-CONFIG config)
       (reset! feed/boards-enabled-cache (set (keys (get config :boards-enabled))))
       (reset! watcher/chod-threads-cache (watcher/generate-chod-cache-structure config))
@@ -83,6 +84,12 @@
 (defn repl-main
   "Development entry point"
   []
+  (let [config (conf/get-some-config nil)]
+      ;; TODO: probably refactor to use separate config.clj file when validation will be added
+      ;; Init the few globals we have
+      (reset! conf/GLOBAL-CONFIG config)
+      (reset! feed/boards-enabled-cache (set (keys (get config :boards-enabled))))
+      (reset! watcher/chod-threads-cache (watcher/generate-chod-cache-structure config)))
   (jetty/run-jetty (rp/wrap-params #'feed/http-handler)
                    {:port (:port conf/CONFIG-DEFAULT)
                     ;; Dont block REPL thread

src/rss_thread_watch/feed_generator.clj

@@ -35,8 +35,8 @@
    This is done by always making new GUID - (concat thread-number UNIX-time-of-data-update)"
   [thread time]
   (assoc thread :guid (str (:no thread)
-                          "-"
+                           "-"
-                          time)))
+                           time)))
 
 (defn new-guid-paranoid
   "Generate unique GUID on EVERY request to the feed.
@@ -64,14 +64,13 @@
   (let [filterable (select-keys query-string
                                 (keys known-filter-map))]
     (ut/fkmap (fn [k v]
-                {(get known-filter-map k) v})
+                {(get known-filter-map k) (ut/vectorize v)})
               filterable)))
 
 (defn filter-chod-posts
   "Return list of all threads with equal or higher ChoD than requested
-  ;;resume
   READS FROM GLOBALS: watcher.time-of-cache"
-  [query-vec chod-treshold repeat? board-cache]
+  [filters chod-treshold repeat? board-cache]
   (let [{time-of-generation :time
          cache :data} board-cache
         guid-fn (case repeat?
@@ -79,28 +78,24 @@
                   "true"     (fn [x] (new-guid-always x time-of-generation))
                   update-only-guid)
         cache-start-index (first (ut/indices (fn [x] (>= (:chod x) chod-treshold))
-                                          cache))
+                                             cache))
         ;; So we don't have to search thru everything we have cached
         needed-cache-part (subvec cache cache-start-index)
-        ;; Here we gonna run fmap but not really
+        actuall-matches (keep (fn [thread]
-        actuall-matches (keep (fn [t]
+                                (some
-                                (let [title (:title t)]
+                                 (fn [fun]
-                                  ;; Todo: Man, wouldn't it be cool to know which querry matched the thread?
+                                   (when (fun thread (get filters fun))
-                                  ;; Would be so much easier for user to figure out why is it showing
+                                     thread))
-                                  ;; and it would solve the problem of super long titles (or OPs instead of titles)
+                                 (keys filters)))
-                                  (when (some (fn [querry]
-                                                (s/includes? (s/lower-case title) (s/lower-case querry)))
-                                              query-vec)
-                                    t)))
                               (reverse needed-cache-part))]
     ;; Finally generate and append GUIDs
     (map guid-fn actuall-matches)))
 
 (defn thread-to-rss-item
   "Converts cached thread item to feed item which can be serialized into RSS"
-  [t host board]
+  [t host]
-  (let [link-url (s/replace host "{threadnum}" (str (:no t)))]            ;Hardcode emergency bugfix
+  (let [link-url (s/replace host "{threadnum}" (str (:no t)))]
-  {:title (format "%.2f%% - %s" (:chod t) (:title t))                  ;TODO: Generate link from the target somehow, or just include it from API response
+  {:title (format "%.2f%% - %s" (:chod t) (:title t))
    ;; :url link-url <- this is supposed to be for images according to: https://cyber.harvard.edu/rss/rss.html
    :description (format "The thread: '%s' has %.2f%% chance of dying" (:title t) (:chod t))
    :link link-url
@@ -108,9 +103,8 @@
 
 (defn generate-feed
   "Generates feed from matching items"
-  [query-vec chod-treshold repeat? cache board-config self-link]
+  [filters chod-treshold repeat? cache board-config self-link]
-  (let [items (filter-chod-posts query-vec chod-treshold repeat? cache)
+  (let [items (filter-chod-posts filters chod-treshold repeat? cache)
-        served-filename (get @conf/GLOBAL-CONFIG :served-filename)
         head {:title (str "RSS Thread watcher v" conf/VERSION)
               ;; :link is the homepage of the channel
               :link (get @conf/GLOBAL-CONFIG :homepage)
@@ -141,22 +135,17 @@
               query :query-string
               scheme :scheme
               server-name :server-name} rqst
-             filters (make-filters prms)
+             filters (make-filters prms f/known-filters)
-             ;; qrs (prms "q")
+             ;; BUG if local fileserver not running -> FileNotFound exception is thrown and it fucks up the feed generation
-             self-uri (str (s/replace-first scheme ":" "")
+             ;; Should be handled because wrong config and thus url generation could do the same
+             self-uri (str (s/replace-first scheme ":" "") ;
                            "://" server-name uri "?" query)
-             ;; queries (if (vector? qrs) qrs [qrs]) ; to always return vector
-             real-chod (if-let [ch (or (and (vector? chod)
-                                            (first chod))
-                                       chod)]
-                         (try             ;If we can't parse number from chod, use default 94
-                           (if (or (vector? chod)
-                                   ;; TODO: Do we seriously parse this twice?
-                                   (<= (Integer/parseInt chod) 60)) ; Never accept chod lower than 60 TODO: don't hardcode this
-                             60 (Integer/parseInt chod))
-                           (catch Exception e
-                             94)))
              board-config (get-in @conf/GLOBAL-CONFIG [:boards-enabled board])
+             real-chod (try (max (Integer/parseInt (or (and (vector? chod)
+                                                            (first chod))
+                                                       chod)) 60) ;HARDCODED CHoD
+                            (catch Exception _
+                              (get board-config :default-chod)))
              cache @watcher/chod-threads-cache]
          (println "\n\nRCVD: " rqst)
          ;; (println rqst)
@@ -193,7 +182,7 @@
           ;; There shouldn't be any problems with this mime type but if there are
           ;; replace with "text/xml", or even better, get RSS reader that is not utter shit
           :header {"Content-Type" "application/rss+xml"}
-          :body (generate-feed queries real-chod repeat? (watcher/get-thread-data board @conf/GLOBAL-CONFIG) board-config self-uri)})
+          :body (generate-feed filters real-chod repeat? (watcher/get-thread-data board @conf/GLOBAL-CONFIG) board-config self-uri)})
        (catch Exception e
          ;; Ex-info has been crafted to match HTTP response body so we can send it
          (if-let [caught (ex-data e)]

src/rss_thread_watch/filters.clj

@@ -20,15 +20,15 @@
 
 (defn case-sensitive-filter
   "Returns true if string [s] is matched by any query. It's case insensitive"
-  [s queries]
+  [{:keys [title]} queries]
   (some (fn [querry]
-          (cs/includes? s querry))
+          (cs/includes? title querry))
         queries))
 
 (defn case-insensitive-filter
   "Returns true if string [s] is case-matched by query"
-  [s queries]
+  [{:keys [title]} queries]
-  (case-sensitive-filter (cs/lower-case s) (map cs/lower-case queries)))
+  (case-sensitive-filter {:title (cs/lower-case title)} (map cs/lower-case queries)))
 
 (def known-filters
   {"Q" case-sensitive-filter

src/rss_thread_watch/utils.clj

@@ -74,6 +74,7 @@
               {k (map-apply-defaults conf-val default-val)}
               {k (nil?-else conf-val default-val)})))))
 
+;; This is a shitty version of reduce-kv
 (defn fmap
   "Applies function [f] to every key and value in map [m]
    Function signature should be (f [key value]).