Treebrary.org/rss-thread-watcher
1
0
Fork 0
Code Issues 19 Pull requests Projects 2 Releases 2 Packages Wiki Activity

Refactor config utils into it's own file and namespace

Browse source 
Everything /seems/ to be working
This commit is contained in:
Felisp 2024-09-10 17:15:49 +02:00
parent 20752a3b1c
commit e871d1a6c4
5 changed files with 106 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
project.clj
View file

@ -1,4 +1,4 @@
(defproject rss-thread-watch "0.4.5-SNAPSHOT" (defproject rss-thread-watch "0.4.6-SNAPSHOT"
:description "RSS based thread watcher" :description "RSS based thread watcher"
:url "http://example.com/FIXME" :url "http://example.com/FIXME"
:license {:name "AGPL-3.0-only" :license {:name "AGPL-3.0-only"

88
src/rss_thread_watch/config.clj Normal file
View file

@ -0,0 +1,88 @@
(ns rss-thread-watch.config
"Functions for working with configuration"
(:require [clojure.java.io :as io]
[clojure.edn :as edn]
[clojure.string :as s]
[rss-thread-watch.utils :as u])
(:gen-class))
(def VERSION "0.4.6")
(def GLOBAL-CONFIG
"Global config with defaults for missing entires"
;; I know globals are ew in Clojure but I don't know any
;; better way of doing this
(atom nil))
;; Internal default config
(def CONFIG-DEFAULT
"Internal default config"
{:port 6969
:default-board "/mlp/"
:enable-board-listing true
:board-disabled-message "This board is not enabled for feed generation.\n\nYou can contact me here: [contact] and I may enable it for you"
:boards-defaults {:refresh-rate 300
:starting-page 7
:default-chod 94
:target "https://api.4chan.org/{board}/catalog.json"
:host "https://boards.4chan.org/{board}/thread/{threadnum}"
:lazy-load true}
:boards-enabled {"/mlp/" {:lazy-load false}
"/g/" {:starting-page 7}
"/po/" {:starting-page 8
:refresh-rate 86400}
"/p/" {:starting-page 8
:refresh-rate 1800}}})
(defn load-config
"Attempts to load config from file [f].
Returns loaded config map or nil if failed"
[f]
(let [fl (io/as-file f)]
(when (.exists fl)
(with-open [r (io/reader fl)]
(edn/read (java.io.PushbackReader. r))))))
(defn config-url-expand
"Expands substitution in :target and :host fields"
[filled-config]
(let [boards (get filled-config :boards-enabled)
selecting '(:target :host)
pattern "{board}"]
(assoc filled-config
:boards-enabled
(u/fmap (fn [board confs]
(->> (select-keys confs selecting)
(u/fmap (fn [k v]
(s/replace v pattern (s/replace board "/" ""))))
(merge confs)))
boards))))
(defn config-fill-board-defaults
"Fills every enabled board with default config values"
[config]
(let [defaults (:boards-defaults config)]
(as-> config conf
(update-in conf
'(:boards-enabled)
(fn [mp]
(u/fmap (fn [k v]
(assoc (u/map-apply-defaults v defaults) :name k))
mp)))
(dissoc conf :boards-defaults)
(config-url-expand conf))))
(defn get-some-config
"Attempts to get config somehow,
first from [custom-file], if it's nil,
then from ./config.edn file.
If is neither exists, default internal one is used."
[custom-file]
(config-fill-board-defaults
;; TODO: There has to be try/catch for when file is invalid edn
;; This is gonna be done when config validation comes in Beta 2
(let [file-to-try (u/nil?-else custom-file
"./config.edn")]
(u/when-else (load-config file-to-try)
CONFIG-DEFAULT))))

90
src/rss_thread_watch/core.clj
View file

@ -21,30 +21,10 @@
[ring.middleware.params :as rp] [ring.middleware.params :as rp]
[rss-thread-watch.watcher :as watcher] [rss-thread-watch.watcher :as watcher]
[rss-thread-watch.feed-generator :as feed] [rss-thread-watch.feed-generator :as feed]
[rss-thread-watch.utils :as u]) [rss-thread-watch.utils :as u]
[rss-thread-watch.config :as conf])
(:gen-class)) (:gen-class))
(def VERSION "0.4.5")
;; Internal default config
(def CONFIG-DEFAULT
"Internal default config"
{:port 6969
:default-board "/mlp/"
:enable-board-listing true
:board-disabled-message "This board is not enabled for feed generation.\n\nYou can contact me here: [contact] and I may enable it for you"
:boards-defaults {:refresh-rate 300
:starting-page 7
:default-chod 94
:target "https://api.4chan.org/{board}/catalog.json"
:host "https://boards.4chan.org/{board}/thread/{threadnum}"
:lazy-load true}
:boards-enabled {"/mlp/" {:lazy-load false}
"/g/" {:starting-page 7}
"/po/" {:starting-page 8
:refresh-rate 86400}
"/p/" {:starting-page 8
:refresh-rate 1800}}})
(def cli-options (def cli-options
"Configuration defining program arguments for cli.tools" "Configuration defining program arguments for cli.tools"
@ -68,58 +48,6 @@
(println "Error while updating cache: " e ", retrying in " (/ ms 1000 60) " minutes")))) (println "Error while updating cache: " e ", retrying in " (/ ms 1000 60) " minutes"))))
(Thread/sleep ms))))) (Thread/sleep ms)))))
(defn load-config
"Attempts to load config from file [f].
Returns loaded config map or nil if failed"
[f]
(let [fl (io/as-file f)]
(when (.exists fl)
(with-open [r (io/reader fl)]
(edn/read (java.io.PushbackReader. r))))))
(defn config-url-expand
"Expands substitution in :target and :host fields"
[filled-config]
(let [boards (get filled-config :boards-enabled)
selecting '(:target :host)
pattern "{board}"]
(assoc filled-config
:boards-enabled
(u/fmap (fn [board confs]
(->> (select-keys confs selecting)
(u/fmap (fn [k v]
(s/replace v pattern (s/replace board "/" ""))))
(merge confs)))
boards))))
(defn config-fill-board-defaults
"Fills every enabled board with default config values"
[config]
(let [defaults (:boards-defaults config)]
(as-> config conf
(update-in conf
'(:boards-enabled)
(fn [mp]
(u/fmap (fn [k v]
(assoc (u/map-apply-defaults v defaults) :name k))
mp)))
(dissoc conf :boards-defaults)
(config-url-expand conf))))
(defn get-some-config
"Attempts to get config somehow,
first from [custom-file], if it's nil,
then from ./config.edn file.
If is neither exists, default internal one is used."
[custom-file]
(config-fill-board-defaults
;; TODO: There has to be try/catch for when file is invalid edn
;; This is gonna be done when config validation comes in Beta 2
(let [file-to-try (u/nil?-else custom-file
"./config.edn")]
(u/when-else (load-config file-to-try)
CONFIG-DEFAULT))))
(defn -main (defn -main
"Entry point, starts webserver" "Entry point, starts webserver"
[& args] [& args]
@ -129,26 +57,26 @@
(println "Error: " err) (println "Error: " err)
(System/exit 1)) (System/exit 1))
(when (get options :version) (when (get options :version)
(println "RSS Thread Watcher " VERSION " Licensed under AGPL-3.0-only") (println "RSS Thread Watcher " conf/VERSION " Licensed under AGPL-3.0-only")
(System/exit 0)) (System/exit 0))
(when (get options :help) (when (get options :help)
(println "RSS Thread Watcher help:\n" (get parsed-args :summary)) (println "RSS Thread Watcher help:\n" (get parsed-args :summary))
(System/exit 0)) (System/exit 0))
(when (get options :print-default-config) (when (get options :print-default-config)
(println ";;Default internal config file from RSS Thread Watcher " VERSION) (println ";;Default internal config file from RSS Thread Watcher " conf/VERSION)
(clojure.pprint/pprint CONFIG-DEFAULT) (clojure.pprint/pprint conf/CONFIG-DEFAULT)
;; In case someone was copying by hand, this might be useful ;; In case someone was copying by hand, this might be useful
(println ";;END of Default internal config file") (println ";;END of Default internal config file")
(System/exit 0)) (System/exit 0))
(let [config (get-some-config (:config options))] (let [config (conf/get-some-config (:config options))]
;; TODO: probably refactor to use separate config.clj file when validation will be added ;; TODO: probably refactor to use separate config.clj file when validation will be added
;; Init the few globals we have ;; Init the few globals we have
(reset! watcher/GLOBAL-CONFIG config) (reset! conf/GLOBAL-CONFIG config)
(reset! feed/boards-enabled-cache (set (keys (get config :boards-enabled)))) (reset! feed/boards-enabled-cache (set (keys (get config :boards-enabled))))
(reset! watcher/chod-threads-cache (watcher/generate-chod-cache-structure config)) (reset! watcher/chod-threads-cache (watcher/generate-chod-cache-structure config))
(clojure.pprint/pprint config) (clojure.pprint/pprint config)
(jetty/run-jetty (rp/wrap-params feed/http-handler) {:port (:port CONFIG-DEFAULT) (jetty/run-jetty (rp/wrap-params feed/http-handler) {:port (:port conf/CONFIG-DEFAULT)
:join? true})))) :join? true}))))
;; Docs: https://github.com/ring-clojure/ring/wiki/Getting-Started ;; Docs: https://github.com/ring-clojure/ring/wiki/Getting-Started
@ -156,7 +84,7 @@
"Development entry point" "Development entry point"
[] []
(jetty/run-jetty (rp/wrap-params #'feed/http-handler) (jetty/run-jetty (rp/wrap-params #'feed/http-handler)
{:port (:port CONFIG-DEFAULT) {:port (:port conf/CONFIG-DEFAULT)
;; Dont block REPL thread ;; Dont block REPL thread
:join? false})) :join? false}))
;; (repl-main) ;; (repl-main)

15
src/rss_thread_watch/feed_generator.clj
View file

@ -19,7 +19,8 @@
[clj-rss.core :as rss] [clj-rss.core :as rss]
[clojure.string :as s] [clojure.string :as s]
[rss-thread-watch.watcher :as watcher] [rss-thread-watch.watcher :as watcher]
[rss-thread-watch.utils :as ut]) [rss-thread-watch.utils :as ut]
[rss-thread-watch.config :as conf])
(:gen-class)) (:gen-class))
(def boards-enabled-cache (def boards-enabled-cache
@ -98,7 +99,7 @@
"Generates feed from matching items" "Generates feed from matching items"
[query-vec chod-treshold repeat? cache board-config] [query-vec chod-treshold repeat? cache board-config]
(let [items (filter-chod-posts query-vec chod-treshold repeat? cache) (let [items (filter-chod-posts query-vec chod-treshold repeat? cache)
head {:title "RSS Thread watcher v0.4.5" ;TODO: hardcoded string here, remake to reference to config.clj head {:title (str "RSS Thread watcher v" conf/VERSION)
:link "https://tools.treebrary.org/thread-watcher/feed.xml" :link "https://tools.treebrary.org/thread-watcher/feed.xml"
:feed-url "https://tools.treebrary.org/thread-watcher/feed.xml" :feed-url "https://tools.treebrary.org/thread-watcher/feed.xml"
:description "RSS based thread watcher"} :description "RSS based thread watcher"}
@ -113,12 +114,12 @@
READS FROM GLOBALS: READS FROM GLOBALS:
rss-thread-watch.watcher.chod-threads-cache rss-thread-watch.watcher.chod-threads-cache
rss-thread-watch.watcher.GLOBAL-CONFIG" ;TODO: Update if it really reads from there anymore rss-thread-watch.config.GLOBAL-CONFIG" ;TODO: Update if it really reads from there anymore
[rqst] [rqst]
(try (let [{{chod "chod" (try (let [{{chod "chod"
board "board" board "board"
repeat? "repeat" :or {chod "94" repeat? "repeat" :or {chod "94"
board (get @watcher/GLOBAL-CONFIG :default-board) board (get @conf/GLOBAL-CONFIG :default-board)
repeat? false} repeat? false}
:as prms} :params :as prms} :params
uri :uri} rqst uri :uri} rqst
@ -134,7 +135,7 @@
60 (Integer/parseInt chod)) 60 (Integer/parseInt chod))
(catch Exception e (catch Exception e
94))) 94)))
board-config (get-in @watcher/GLOBAL-CONFIG [:boards-enabled board]) board-config (get-in @conf/GLOBAL-CONFIG [:boards-enabled board])
cache @watcher/chod-threads-cache] cache @watcher/chod-threads-cache]
(println "\n\nRCVD: " rqst) (println "\n\nRCVD: " rqst)
;; (println rqst) ;; (println rqst)
@ -147,7 +148,7 @@
(when-not (contains? @boards-enabled-cache board) (when-not (contains? @boards-enabled-cache board)
(throw (ex-info "403" {:status 403 (throw (ex-info "403" {:status 403
:header {"Content-Type" "text/plain"} :header {"Content-Type" "text/plain"}
:body (get @watcher/GLOBAL-CONFIG :board-disabled-message)}))) :body (get @conf/GLOBAL-CONFIG :board-disabled-message)})))
;; No url params -> we redirect to documentation about params ;; No url params -> we redirect to documentation about params
(when (empty? prms) (when (empty? prms)
(throw (ex-info "302" (throw (ex-info "302"
@ -171,7 +172,7 @@
;; There shouldn't be any problems with this mime type but if there are ;; There shouldn't be any problems with this mime type but if there are
;; replace with "text/xml", or even better, get RSS reader that is not utter shit ;; replace with "text/xml", or even better, get RSS reader that is not utter shit
:header {"Content-Type" "application/rss+xml"} :header {"Content-Type" "application/rss+xml"}
:body (generate-feed queries real-chod repeat? (watcher/get-thread-data board @watcher/GLOBAL-CONFIG) board-config)}) :body (generate-feed queries real-chod repeat? (watcher/get-thread-data board @conf/GLOBAL-CONFIG) board-config)})
(catch Exception e (catch Exception e
;; Ex-info has been crafted to match HTTP response body so we can send it ;; Ex-info has been crafted to match HTTP response body so we can send it
(if-let [caught (ex-data e)] (if-let [caught (ex-data e)]

6
src/rss_thread_watch/watcher.clj
View file

@ -18,12 +18,6 @@
[clojure.data.json :as js]) [clojure.data.json :as js])
(:gen-class)) (:gen-class))
(def GLOBAL-CONFIG
"Global config with defaults for missing entires"
;; I know globals are ew in Clojure but I don't know any
;; better way of doing this
(atom nil))
(def chod-threads-cache (def chod-threads-cache
"Cached map of threads that have CHanceOfDeath > configured" "Cached map of threads that have CHanceOfDeath > configured"
(atom {})) (atom {}))