Implement Cache generation

src/rss_thread_watch/watcher.clj

@@ -0,0 +1,70 @@
+;; Copyright (C) 2023  Felisp
+;;
+;; This program is free software: you can redistribute it and/or modify
+;; it under the terms of the GNU Affero General Public License as published by
+;; the Free Software Foundation, version 3 of the License.
+;;
+;; This program is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU Affero General Public License for more details.
+;;
+;; You should have received a copy of the GNU Affero General Public License
+;; along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+(ns rss-thread-watch.watcher
+  "Functions for monitoring catalog and maintaining list of near-death threads"
+  (:require [clojure.java.io :as io]
+            [clojure.data.json :as js])
+  (:gen-class))
+
+(def chod-threads-cache
+  "Cached vector of threads that have CHanceOfDeath > configured"
+  (atom []))
+
+(defn process-page
+  "Procesess every thread in page, leaving only relevant information
+   (title no chod)"
+  ([threads-to-index threads-total starting-index] (process-page threads-to-index threads-total starting-index (transient [])))
+  ([remaining-threads threads-total index ret]
+   (if (empty? remaining-threads)
+     (persistent! ret)
+     (let [thread (first remaining-threads)]
+       (recur (rest remaining-threads)
+              threads-total
+              (inc index)
+              (conj! ret {:title (:sub thread)
+                          :no (:no thread)
+                          :chod (* 100 (float (/ index threads-total)))}))))))
+
+(defn build-cache
+  "Build cache of near-death threads so the values don't have to be recalculated on each request.
+
+   Bugs: There is probably off-by-one somewhere in the calculation because thread at position 150 doesn't
+         have 100% chance of death which is weird, however, 99.333% is considered close enough for now"
+  [pages-to-index pages-total threads-per-page threads-total]
+  (vec (flatten (map (fn [single-page]
+                       ;; We have to (dec page-number) bcs otherwise we would get the total number of threads
+                       ;; including the whole page of threads
+                       (let [page-number (dec (:page single-page))] ;        inc to get to the actuall page
+                         (process-page (:threads single-page) threads-total (inc (* page-number threads-per-page)))))
+                     pages-to-index))))
+
+
+
+(defn update-thread-cache!
+  "Updates cache of near-death threads. Writes to chod-threads-cache as side effect.
+   [url] - Url to download data from
+   [starting-page] - From which page consider threads to be fit for near-death cache"
+  [url starting-page]
+  ;; Todo: surround with try so we can timeout and other stuff
+  (let [catalog (with-open [readr (io/reader url)]
+                  (js/read readr :key-fn keyword))
+        pages-total (count catalog)
+        ;; universal calculation for total number of threads:
+        ;; (pages-total-1) * threadsPerPage + threadsOnLastpage ;;accounts for boards which have stickied threads making them have 11pages
+        threads-per-page (count (:threads (first catalog)))
+        threads-total (+ (* threads-per-page (dec pages-total) (count (last catalog)))) ;; Todo: Yeah, maybe this calculation could be refactored into let
+        to-index (filter (fn [item]
+                           (<= starting-page (:page item))) catalog)]
+    (reset! chod-threads-cache (build-cache to-index pages-total threads-per-page threads-total))))