dependencies
| (this space intentionally left almost blank) | ||||||||||||||||||||||||
(ns flycouchdb.migration (:use [flycouchdb.parser.parse-edn-structures :only (parse-edn-structures apply-functions)] [flycouchdb.parser.parse-migration-names :only (generate-migrations-structure)] [flycouchdb.scanner :only (extract-migrations)] [be.dsquare.clutch :only (couch up? exist? create-view! get-view)] [clojure.java.io :only (input-stream)] [clj-time.core :only (now)] [slingshot.slingshot :only [throw+ try+]]) (:require [com.ashafa.clutch :as clutch])) | |||||||||||||||||||||||||
(def migration-db "migration-db") | |||||||||||||||||||||||||
(def migration-counter (atom 0)) | |||||||||||||||||||||||||
(defrecord FlyCouchDB [^String location-folder ^String datasource-url ^String datasource-user ^String datasource-password]) | |||||||||||||||||||||||||
Validates the connection | (defn- validate-connection [] (let [db (couch migration-db)] (if (up? db) db (throw+ {:type :flycouchdb :fn "validate-connection" :message (str "Trying to connect to CouchDB. Connection is down!")})))) | ||||||||||||||||||||||||
Creates the migration database in CouchDB in case it does not exist. The migration database is named 'migration-db', and is a hardcoded name so far. | (defn- create-if-not-exists [] (let [db (couch migration-db)] (when-not (exist? db) (clutch/create! db) (create-view! db "migration-template" "order-migrations" "function(doc) {if (doc.counter) {emit(doc.counter, doc);}}")))) | ||||||||||||||||||||||||
(defmulti slurp-edn-structures :source) | |||||||||||||||||||||||||
(defmethod slurp-edn-structures :file [migration] (assoc migration :edn-structure (read-string (slurp (:file migration))))) | |||||||||||||||||||||||||
(defmethod slurp-edn-structures :jar [migration] (assoc migration :edn-structure (read-string (slurp (input-stream (:file migration)))))) | |||||||||||||||||||||||||
(defmethod slurp-edn-structures :vfs [migration] (assoc migration :edn-structure (read-string (slurp (.openStream (:file migration)))))) | |||||||||||||||||||||||||
(defn- columns [column-names] (fn [row] (vec (map row column-names)))) | |||||||||||||||||||||||||
We compare by :version and subversion | (defn- compare-by-version-subversion [{version-a :version subversion-a :subversion} {version-b :version subversion-b :subversion}] (cond (< version-a version-b) -1 (> version-a version-b) 1 (< subversion-a subversion-b) -1 (> subversion-a subversion-b) 1 :else 0)) | ||||||||||||||||||||||||
Update the counter with the last migration that was run | (defn- update-counter! [{counter :counter :or {:counter 0}}] (reset! migration-counter counter)) | ||||||||||||||||||||||||
Getting the last migration from CouchDB, ordered by the counter entry | (defn- get-last-migration [] (-> migration-db couch (get-view "migration-template" "order-migrations") last :value (#(if (nil? %) {:version -1 :subversion -1 :counter 0} %)))) | ||||||||||||||||||||||||
Start the migration process | (defn migrate [^FlyCouchDB flycouchdb] (do (validate-connection) (create-if-not-exists) (let [last-migration (get-last-migration)] (update-counter! last-migration) (->> flycouchdb extract-migrations generate-migrations-structure (sort-by (columns [:version :subversion])) (filter #(= (compare-by-version-subversion % last-migration) 1)) (map (fn [migration] (slurp-edn-structures migration))) (map (fn [migration] (assoc migration :edn-function (parse-edn-structures (:edn-structure migration))))) (map (fn [migration] (assoc migration :ts (str (now))))) (map (fn [migration] (assoc migration :counter (swap! migration-counter inc)))) (mapv (fn [migration] (do (apply-functions migration) (println (:name migration)) (let [db (couch migration-db) edn-migration (-> migration (dissoc :file :edn-function :file :edn-structure) (assoc :dbname (:dbname (:edn-structure migration)) :action (:action (:edn-structure migration))))] (clutch/assoc! db (:name migration) edn-migration))))))))) | ||||||||||||||||||||||||
Returns an instance of an implementation of FlyCouchDB | (defn flycouchdb [^String location-folder] (FlyCouchDB. location-folder nil nil nil)) | ||||||||||||||||||||||||
(ns flycouchdb.parser.parse-edn-structures (:use [be.dsquare.clutch :only (couch drop! up? exist? create-view! take-all get-view get-user-view create-user-view!)] [com.ashafa.clutch :only (create!)] [slingshot.slingshot :only [throw+ try+]]) (:require [com.ashafa.clutch :as clutch])) | |||||||||||||||||||||||||
(defmulti parse-edn-structures :action) | |||||||||||||||||||||||||
(defmethod parse-edn-structures :create [{dbname :dbname}] (fn [] (let [couchdb (couch dbname)] (create! couchdb)))) | |||||||||||||||||||||||||
(defmethod parse-edn-structures :delete [{dbname :dbname}] (fn [] (let [couchdb (couch dbname)] (drop! couchdb)))) | |||||||||||||||||||||||||
(defmethod parse-edn-structures :create-view [{dbname :dbname {:keys [view-design view-name view-function]} :create-view}] (fn [] (let [couchdb (couch dbname)] (create-view! couchdb view-design view-name view-function)))) | |||||||||||||||||||||||||
(defmethod parse-edn-structures :rename-keys [{dbname :dbname {:keys [filter-fn rename-fn]} :rename-keys}] (fn [] (let [couchdb (couch dbname)] (->> couchdb take-all (map #(second %)) (filter (eval filter-fn)) (map (fn [entry] {:old_id (:_id entry) :new_id ((eval rename-fn) entry) :document-body (dissoc entry :_id :_rev)})) (mapv (fn [{:keys [old_id new_id document-body]}] (do (clutch/dissoc! couchdb old_id) (clutch/assoc! couchdb new_id document-body)))))))) | |||||||||||||||||||||||||
(defmethod parse-edn-structures :edit-entries [{dbname :dbname {:keys [filter-fn edit-fn]} :edit-entries}] (fn [] (let [couchdb (couch dbname)] (->> couchdb take-all (map #(second %)) (filter (eval filter-fn)) (mapv (fn [entry] (clutch/assoc! couchdb (:_id entry) ((eval edit-fn) entry)))))))) | |||||||||||||||||||||||||
(defmethod parse-edn-structures :composite [{{composite-fn :composite-fn} :composite}] (let [migration-functions (->> ((eval composite-fn)) (mapv (fn [edn-structure] (parse-edn-structures edn-structure))))] (fn [] (->> migration-functions (mapv (fn [m-fn] ((eval m-fn)))))))) | |||||||||||||||||||||||||
(defmethod parse-edn-structures :insert-documents [{dbname :dbname {composite-fn :insert-documents-fn} :insert-documents}] (let [db (couch dbname) documents ((eval composite-fn))] (fn [] (->> documents (mapv (fn [document] (clutch/assoc! db (:_id document) document))))))) | |||||||||||||||||||||||||
Apply the anonymous functions that were created in this namespace so that there is no problem with the imports. | (defn apply-functions [{:keys [edn-function edn-structure file-name ts]}] (edn-function)) | ||||||||||||||||||||||||
(ns flycouchdb.parser.parse-migration-names (:use clojure.pprint [slingshot.slingshot :only [throw+ try+]])) | |||||||||||||||||||||||||
Checks if it's an edn file | (defn- edn-file? [file-name] (let [total-file (count file-name)] (= (subs file-name (- total-file 4) total-file) ".edn"))) | ||||||||||||||||||||||||
Remove the extension from the file name | (defn- remove-file-extension ([file-name] (remove-file-extension file-name "edn")) ([file-name extension] (subs file-name 0 (- (count file-name) (+ 1 (count extension)))))) | ||||||||||||||||||||||||
Validate if it's a correct edn file and retorn name | (defn- validate-edn-file [{file-name :file-name}] (if (edn-file? file-name) {:name (remove-file-extension file-name)} (throw+ {:type :flycouchdb :fn "validate-edn-file" :message (str "This file: " file-name " is not a correct edn file")}))) | ||||||||||||||||||||||||
Checks if the version number is V{d}{ddd}_ Example: V1101_ where this is gonna be the migration 101 from the first version | (defn- correct-version? [^String file-name] (let [find-version (re-find #"V(\d+)_" file-name)] (not (nil? find-version)))) | ||||||||||||||||||||||||
Extract version from migration name V1_ | (defn- extract-migration-number [^String name] (->> name (re-find #"V(\d+)_") second read-string)) | ||||||||||||||||||||||||
Checks if the version number is V{d}{ddd}_ Example: V1101_ where this is gonna be the migration 101 from the first version | (defn- correct-subversion-number? [^String name] (let [find-version (re-find #"V(\d+)_(\d+)__[a-zA-Z]+" name)] (not (nil? find-version)))) | ||||||||||||||||||||||||
Checks if the version number is V{d}{ddd}_ Example: V1101_ where this is gonna be the migration 101 from the first version | (defn- extract-migration-subversion-number [^String name] (->> name (re-find #"V(\d+)_(\d+)__[a-zA-Z]+") rest second read-string)) | ||||||||||||||||||||||||
Validate if the version is correctly formated and returns it | (defn- validate-migration-version [{name :name}] (if (correct-version? name) {:version (extract-migration-number name)} (throw+ {:type :flycouchdb :fn "validate-migration-version" :message (str "This version of " name " is not valid!" "\nExample: V1_131__Create_Database.edn")}))) | ||||||||||||||||||||||||
(defn- validate-subversion-number [{name :name}] (if (correct-subversion-number? name) {:subversion (extract-migration-subversion-number name)} (throw+ {:type :flycouchdb :fn "validate-subversion-number" :message (str "This sub-version of " name " is not valid!" "\nExample: V1_131__Create_Database.edn")}))) | |||||||||||||||||||||||||
This generates the following structure per each migration: {:version 1, :subversion 132, :name 'V1132DeleteDatabase', :file #<BufferedInputStream java.io.BufferedInputStream@2b46504>, :file-name 'V1132DeleteDatabase.edn'} | (defn generate-migrations-structure [folder-seq] (->> folder-seq (map #(merge % (validate-edn-file %))) (map #(merge % (validate-migration-version %))) (map #(merge % (validate-subversion-number %))))) | ||||||||||||||||||||||||
(ns flycouchdb.scanner (:use [clojure.java.io :only (file resource)] [slingshot.slingshot :only [throw+ try+]]) (:import [java.net URL URLDecoder] [java.io File] [java.util.jar JarFile] [org.jboss.vfs VFS VirtualFile] [clojure.lang ISeq])) | |||||||||||||||||||||||||
Check that the permissions are ok for reading it | (defn- validate-file-can-be-read [^ISeq file] (if (.canRead file) file (throw+ {:type :flycouchdb :fn "validate-file-can-be-read" :message (str "This file: " (.getName file) " can not be read")}))) | ||||||||||||||||||||||||
Create a map that contains all the properties from a migration file | (defn- generate-basic-structure-from-file [^ISeq file] {:file-name (.getName file) :file file :source :file}) | ||||||||||||||||||||||||
validates if it's a file | (defn- validate-is-a-file [^ISeq file] (try (when (.isFile file) file) (catch Exception _ (throw+ {:type :flycouchdb :fn "validate-is-a-file" :message "Is not a valid File"})))) | ||||||||||||||||||||||||
Extracting the migrations from a file/directory resource | (defn extract-file-migrations [^File location-folder] (let [folder-seq (file-seq location-folder)] (->> folder-seq rest (map #(validate-is-a-file %)) (map #(validate-file-can-be-read %)) (map #(generate-basic-structure-from-file %))))) | ||||||||||||||||||||||||
Create a map that contains all the properties from a Virtual File System | (defn generate-basic-structure-from-vfs [^VirtualFile v] {:file-name (.getName v) :file v :source :vfs}) | ||||||||||||||||||||||||
Extracting the migrations from a VFS resource | (defn- extract-vfs-migrations [^String location-folder] (->> location-folder .getPath (. URLDecoder decode) File. .getAbsolutePath (. VFS getChild) .getChildren (mapv generate-basic-structure-from-vfs))) | ||||||||||||||||||||||||
Create a map that contains all the properties from a JAR file | (defn- generate-basic-structure-from-jar [migration-folder jar-entry] {:file-name (subs (.getName jar-entry) (count migration-folder) (count (.getName jar-entry))) :file (resource (.getName jar-entry)) :source :jar}) | ||||||||||||||||||||||||
Extracting the migrations from a JAR resource | (defn- extract-jar-migrations [[^String jar-path ^String migration-folder]] (->> jar-path JarFile. .entries enumeration-seq (filter #(re-find (re-pattern (str migration-folder "*")) (.getName %))) (filter #(re-find #"\.edn{1}|\.clj{1}" (.getName %))) (mapv (partial generate-basic-structure-from-jar migration-folder)))) | ||||||||||||||||||||||||
It splits the jar path and the internal folder from the jar URL from jar:file:/Users/haduart/flycouchdb-example-0.1.0-SNAPSHOT-standalone.jar!/migrations/ to ['/Users/haduart/flycouchdb-example-0.1.0-SNAPSHOT-standalone.jar' 'migrations/'] | (defn- parse-jar-path-and-folder [^URL jar-url] (let [jar-file (str jar-url) jar-string (subs jar-file (count "jar:file:") (count jar-file)) jar-position (.indexOf jar-string ".jar!")] [(str (subs jar-string 0 jar-position) ".jar") (subs jar-string (+ jar-position (count ".jar!/")) (count jar-string))])) | ||||||||||||||||||||||||
Wraps around the Java call. Just to make the function testable | (defn- get-url-protocol [location-folder] (.getProtocol location-folder)) | ||||||||||||||||||||||||
(defmulti extract-migrations (fn [{location-folder :location-folder}] (get-url-protocol location-folder))) | |||||||||||||||||||||||||
(defmethod extract-migrations "jar" [{location-folder :location-folder}] (-> location-folder parse-jar-path-and-folder extract-jar-migrations)) | |||||||||||||||||||||||||
(defmethod extract-migrations "file" [{location-folder :location-folder}] (-> location-folder file extract-file-migrations)) | |||||||||||||||||||||||||
(defmethod extract-migrations "vfs" [{location-folder :location-folder}] (-> location-folder extract-vfs-migrations)) | |||||||||||||||||||||||||